From ab8c4420a90ccb492b1ec41690f23528cc6646f7 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Thu, 30 Oct 2025 19:53:43 +0200 Subject: [PATCH 001/176] Move defaulting of the Shoot networks from the `ShootValidator` to the `ShootMutator` admission plugin (part 3) (#13207) * Fetch the Seed resource in the `ShootMutator` admission plugin * Fix integration tests * Duplicate defaulting of Shoot networks in the `ShootMutator` admission plugin * Clean up defaulting of Shoot networks from the `ShootValidator` admission plugin * Update `ShootMutator` docs * Wait until the caches have been synced * Add nil check for Seed `.spec.networks.shootDefaults.{pods,services}` * Address PR review feedback from acumino --- docs/concepts/apiserver-admission-plugins.md | 1 + plugin/pkg/shoot/mutator/admission.go | 91 ++++++++++++ plugin/pkg/shoot/mutator/admission_test.go | 133 +++++++++++++++++- plugin/pkg/shoot/validator/admission.go | 28 +--- plugin/pkg/shoot/validator/admission_test.go | 22 --- .../shootvalidator_suite_test.go | 4 - .../shootvalidator/shootvalidator_test.go | 6 +- .../bastion/bastion_suite_test.go | 2 +- .../shoot/migration/migration_suite_test.go | 2 +- .../shootstate/shootstate_suite_test.go | 2 +- .../gardenlet/shoot/state/state_suite_test.go | 2 +- .../shoot/status/status_suite_test.go | 2 +- 12 files changed, 240 insertions(+), 55 deletions(-) diff --git a/docs/concepts/apiserver-admission-plugins.md b/docs/concepts/apiserver-admission-plugins.md index 2c45f1718c2..a181bdd197f 100644 --- a/docs/concepts/apiserver-admission-plugins.md +++ b/docs/concepts/apiserver-admission-plugins.md @@ -235,6 +235,7 @@ This admission controller reacts on `CREATE` and `UPDATE` operations for `Shoot` It mutates the `Shoot` in the following way: - It sets the `gardener.cloud/created-by=` annotation for newly created `Shoot` resources. - It maintains annotations used for Shoot lifecycle operations such as `shoot.gardener.cloud/tasks` and `maintenance.shoot.gardener.cloud/needs-retry-operation`. +- It defaults Shoot `.spec.networking.pods` and `.spec.networking.services` fields in case they are not provided and the Seed specifies the `.spec.networks.shootDefaults` field. Over time, the `ShootMutator` admission plugin will take over all the mutations that are performed by `ShootValidator`. For more details, see https://github.com/gardener/gardener/issues/2158. diff --git a/plugin/pkg/shoot/mutator/admission.go b/plugin/pkg/shoot/mutator/admission.go index ac8ee164eec..db3e660ebe7 100644 --- a/plugin/pkg/shoot/mutator/admission.go +++ b/plugin/pkg/shoot/mutator/admission.go @@ -6,8 +6,12 @@ package mutator import ( "context" + "errors" + "fmt" "io" + "net" "reflect" + "slices" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -17,7 +21,12 @@ import ( "k8s.io/utils/ptr" "github.com/gardener/gardener/pkg/apis/core" + "github.com/gardener/gardener/pkg/apis/core/helper" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + admissioninitializer "github.com/gardener/gardener/pkg/apiserver/admission/initializer" + gardencoreinformers "github.com/gardener/gardener/pkg/client/core/informers/externalversions" + gardencorev1beta1listers "github.com/gardener/gardener/pkg/client/core/listers/core/v1beta1" "github.com/gardener/gardener/pkg/controllerutils" plugin "github.com/gardener/gardener/plugin/pkg" ) @@ -32,8 +41,17 @@ func Register(plugins *admission.Plugins) { // MutateShoot is an implementation of admission.Interface. type MutateShoot struct { *admission.Handler + + seedLister gardencorev1beta1listers.SeedLister + readyFunc admission.ReadyFunc } +var ( + _ = admissioninitializer.WantsCoreInformerFactory(&MutateShoot{}) + + readyFuncs []admission.ReadyFunc +) + // New creates a new MutateShoot admission plugin. func New() (*MutateShoot, error) { return &MutateShoot{ @@ -41,10 +59,50 @@ func New() (*MutateShoot, error) { }, nil } +// AssignReadyFunc assigns the ready function to the admission handler. +func (m *MutateShoot) AssignReadyFunc(f admission.ReadyFunc) { + m.readyFunc = f + m.SetReadyFunc(f) +} + +// SetCoreInformerFactory gets Lister from SharedInformerFactory. +func (m *MutateShoot) SetCoreInformerFactory(f gardencoreinformers.SharedInformerFactory) { + seedInformer := f.Core().V1beta1().Seeds() + m.seedLister = seedInformer.Lister() + + readyFuncs = append( + readyFuncs, + seedInformer.Informer().HasSynced, + ) +} + +// ValidateInitialization checks whether the plugin was correctly initialized. +func (m *MutateShoot) ValidateInitialization() error { + if m.seedLister == nil { + return errors.New("missing seed lister") + } + return nil +} + var _ admission.MutationInterface = (*MutateShoot)(nil) // Admit mutates the Shoot. func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admission.ObjectInterfaces) error { + // Wait until the caches have been synced + if m.readyFunc == nil { + m.AssignReadyFunc(func() bool { + for _, readyFunc := range readyFuncs { + if !readyFunc() { + return false + } + } + return true + }) + } + if !m.WaitForReady() { + return admission.NewForbidden(a, errors.New("not yet ready to handle request")) + } + // Ignore all kinds other than Shoot if a.GetKind().GroupKind() != core.Kind("Shoot") { return nil @@ -67,7 +125,17 @@ func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admissi } } + var seed *gardencorev1beta1.Seed + if shoot.Spec.SeedName != nil { + var err error + seed, err = m.seedLister.Get(*shoot.Spec.SeedName) + if err != nil { + return apierrors.NewInternalError(fmt.Errorf("could not find referenced seed %q: %w", *shoot.Spec.SeedName, err)) + } + } + mutationContext := &mutationContext{ + seed: seed, shoot: shoot, oldShoot: oldShoot, } @@ -77,11 +145,13 @@ func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admissi } mutationContext.addMetadataAnnotations(a) + mutationContext.defaultShootNetworks(helper.IsWorkerless(shoot)) return nil } type mutationContext struct { + seed *gardencorev1beta1.Seed shoot *core.Shoot oldShoot *core.Shoot } @@ -157,3 +227,24 @@ func addDeploymentTasks(shoot *core.Shoot, tasks ...string) { } controllerutils.AddTasks(shoot.Annotations, tasks...) } + +func (c *mutationContext) defaultShootNetworks(workerless bool) { + if c.seed != nil { + if c.shoot.Spec.Networking.Pods == nil && !workerless && + c.seed.Spec.Networks.ShootDefaults != nil && c.seed.Spec.Networks.ShootDefaults.Pods != nil && + cidrMatchesIPFamily(*c.seed.Spec.Networks.ShootDefaults.Pods, c.shoot.Spec.Networking.IPFamilies) { + c.shoot.Spec.Networking.Pods = c.seed.Spec.Networks.ShootDefaults.Pods + } + + if c.shoot.Spec.Networking.Services == nil && + c.seed.Spec.Networks.ShootDefaults != nil && c.seed.Spec.Networks.ShootDefaults.Services != nil && + cidrMatchesIPFamily(*c.seed.Spec.Networks.ShootDefaults.Services, c.shoot.Spec.Networking.IPFamilies) { + c.shoot.Spec.Networking.Services = c.seed.Spec.Networks.ShootDefaults.Services + } + } +} + +func cidrMatchesIPFamily(cidr string, ipfamilies []core.IPFamily) bool { + ip, _, _ := net.ParseCIDR(cidr) + return ip != nil && (ip.To4() != nil && slices.Contains(ipfamilies, core.IPFamilyIPv4) || ip.To4() == nil && slices.Contains(ipfamilies, core.IPFamilyIPv6)) +} diff --git a/plugin/pkg/shoot/mutator/admission_test.go b/plugin/pkg/shoot/mutator/admission_test.go index 4c7fb303844..d99d88e1513 100644 --- a/plugin/pkg/shoot/mutator/admission_test.go +++ b/plugin/pkg/shoot/mutator/admission_test.go @@ -18,7 +18,9 @@ import ( "k8s.io/utils/ptr" "github.com/gardener/gardener/pkg/apis/core" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + gardencoreinformers "github.com/gardener/gardener/pkg/client/core/informers/externalversions" "github.com/gardener/gardener/pkg/controllerutils" . "github.com/gardener/gardener/pkg/utils/test/matchers" . "github.com/gardener/gardener/plugin/pkg/shoot/mutator" @@ -47,13 +49,34 @@ var _ = Describe("mutator", func() { }) }) + Describe("#ValidateInitialization", func() { + It("should return error if no SeedLister is set", func() { + admissionHandler, err := New() + Expect(err).NotTo(HaveOccurred()) + + err = admissionHandler.ValidateInitialization() + Expect(err).To(MatchError("missing seed lister")) + }) + + It("should not return error if all listers are set", func() { + admissionHandler, err := New() + Expect(err).NotTo(HaveOccurred()) + coreInformerFactory := gardencoreinformers.NewSharedInformerFactory(nil, 0) + admissionHandler.SetCoreInformerFactory(coreInformerFactory) + + Expect(admissionHandler.ValidateInitialization()).To(Succeed()) + }) + }) + Describe("#Admit", func() { var ( ctx context.Context userInfo = &user.DefaultInfo{Name: "foo"} + seed gardencorev1beta1.Seed + shoot core.Shoot - shoot core.Shoot + coreInformerFactory gardencoreinformers.SharedInformerFactory admissionHandler *MutateShoot ) @@ -61,16 +84,33 @@ var _ = Describe("mutator", func() { BeforeEach(func() { ctx = context.Background() + seed = gardencorev1beta1.Seed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "seed", + }, + } shoot = core.Shoot{ ObjectMeta: metav1.ObjectMeta{ Name: "shoot", Namespace: "garden-my-project", }, + Spec: core.ShootSpec{ + Provider: core.Provider{ + Workers: []core.Worker{ + { + Name: "worker-name", + }, + }, + }, + }, } var err error admissionHandler, err = New() Expect(err).NotTo(HaveOccurred()) + admissionHandler.AssignReadyFunc(func() bool { return true }) + coreInformerFactory = gardencoreinformers.NewSharedInformerFactory(nil, 0) + admissionHandler.SetCoreInformerFactory(coreInformerFactory) }) It("should ignore a kind other than shoot", func() { @@ -110,6 +150,16 @@ var _ = Describe("mutator", func() { Expect(err).To(MatchError("could not convert old object to Shoot")) }) + Context("reference checks", func() { + It("should reject because the referenced seed was not found", func() { + shoot.Spec.SeedName = ptr.To("seed") + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + + err := admissionHandler.Admit(ctx, attrs, nil) + Expect(err).To(BeInternalServerError()) + }) + }) + Context("created-by annotation", func() { It("should add the created-by annotation on shoot creation", func() { Expect(shoot.Annotations).NotTo(HaveKeyWithValue(v1beta1constants.GardenCreatedBy, userInfo.Name)) @@ -328,5 +378,86 @@ var _ = Describe("mutator", func() { ), ) }) + + Context("networking settings", func() { + var ( + podsCIDR = "100.96.0.0/11" + servicesCIDR = "100.64.0.0/13" + ) + + BeforeEach(func() { + seed.Spec.Networks.ShootDefaults = &gardencorev1beta1.ShootNetworks{ + Pods: &podsCIDR, + Services: &servicesCIDR, + } + shoot.Spec.SeedName = ptr.To(seed.Name) + shoot.Spec.Networking = &core.Networking{ + Pods: nil, + Services: nil, + IPFamilies: []core.IPFamily{core.IPFamilyIPv4}, + } + }) + + It("should not default shoot networks if shoot .spec.seedName is nil", func() { + shoot.Spec.SeedName = nil + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Networking.Pods).To(BeNil()) + Expect(shoot.Spec.Networking.Services).To(BeNil()) + }) + + It("should not default shoot networks if seed .spec.networks.shootDefaults is empty", func() { + seed.Spec.Networks.ShootDefaults = &gardencorev1beta1.ShootNetworks{} + + Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Networking.Pods).To(BeNil()) + Expect(shoot.Spec.Networking.Services).To(BeNil()) + }) + + It("should not default shoot pod network if shoot is workerless", func() { + shoot.Spec.Provider.Workers = nil + + Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Networking.Pods).To(BeNil()) + Expect(shoot.Spec.Networking.Services).To(Equal(&servicesCIDR)) + }) + + It("should not default shoot networks if shoot IP family does not match seed .spec.networks.shootDefaults", func() { + shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6} + + Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Networking.Pods).To(BeNil()) + Expect(shoot.Spec.Networking.Services).To(BeNil()) + }) + + It("should default shoot networks if all conditions are met", func() { + Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Networking.Pods).To(Equal(&podsCIDR)) + Expect(shoot.Spec.Networking.Services).To(Equal(&servicesCIDR)) + }) + }) }) }) diff --git a/plugin/pkg/shoot/validator/admission.go b/plugin/pkg/shoot/validator/admission.go index d2dcdaf75ac..a1d2c3a71fe 100644 --- a/plugin/pkg/shoot/validator/admission.go +++ b/plugin/pkg/shoot/validator/admission.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "io" - "net" "reflect" "slices" "strconv" @@ -859,11 +858,6 @@ func (c *validationContext) validateReferencedSecret(secretLister kubecorev1list return nil } -func cidrMatchesIPFamily(cidr string, ipfamilies []core.IPFamily) bool { - ip, _, _ := net.ParseCIDR(cidr) - return ip != nil && (ip.To4() != nil && slices.Contains(ipfamilies, core.IPFamilyIPv4) || ip.To4() == nil && slices.Contains(ipfamilies, core.IPFamilyIPv6)) -} - func (c *validationContext) validateShootNetworks(a admission.Attributes, workerless bool) field.ErrorList { var ( allErrs field.ErrorList @@ -875,24 +869,14 @@ func (c *validationContext) validateShootNetworks(a admission.Attributes, worker } if c.seed != nil { - if c.shoot.Spec.Networking.Pods == nil && !workerless { - if c.seed.Spec.Networks.ShootDefaults != nil { - if cidrMatchesIPFamily(*c.seed.Spec.Networks.ShootDefaults.Pods, c.shoot.Spec.Networking.IPFamilies) { - c.shoot.Spec.Networking.Pods = c.seed.Spec.Networks.ShootDefaults.Pods - } - } else if slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv4) { - allErrs = append(allErrs, field.Required(path.Child("pods"), "pods is required")) - } + if c.shoot.Spec.Networking.Pods == nil && !workerless && + slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv4) { + allErrs = append(allErrs, field.Required(path.Child("pods"), "pods is required")) } - if c.shoot.Spec.Networking.Services == nil { - if c.seed.Spec.Networks.ShootDefaults != nil { - if cidrMatchesIPFamily(*c.seed.Spec.Networks.ShootDefaults.Services, c.shoot.Spec.Networking.IPFamilies) { - c.shoot.Spec.Networking.Services = c.seed.Spec.Networks.ShootDefaults.Services - } - } else if slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv4) { - allErrs = append(allErrs, field.Required(path.Child("services"), "services is required")) - } + if c.shoot.Spec.Networking.Services == nil && + slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv4) { + allErrs = append(allErrs, field.Required(path.Child("services"), "services is required")) } if slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv4) { diff --git a/plugin/pkg/shoot/validator/admission_test.go b/plugin/pkg/shoot/validator/admission_test.go index eeb60556c73..42c64e7c8b3 100644 --- a/plugin/pkg/shoot/validator/admission_test.go +++ b/plugin/pkg/shoot/validator/admission_test.go @@ -2322,28 +2322,6 @@ var _ = Describe("validator", func() { }) - It("should default shoot networks if seed provides ShootDefaults", func() { - seed.Spec.Networks.ShootDefaults = &gardencorev1beta1.ShootNetworks{ - Pods: &podsCIDR, - Services: &servicesCIDR, - } - shoot.Spec.Networking.Pods = nil - shoot.Spec.Networking.Services = nil - - Expect(coreInformerFactory.Core().V1beta1().Projects().Informer().GetStore().Add(&project)).To(Succeed()) - Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) - Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) - Expect(coreInformerFactory.Core().V1beta1().SecretBindings().Informer().GetStore().Add(&secretBinding)).To(Succeed()) - Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(shoot.Spec.Networking.Pods).To(Equal(&podsCIDR)) - Expect(shoot.Spec.Networking.Services).To(Equal(&servicesCIDR)) - }) - It("should reject because the shoot node and the seed node networks intersect (HA control plane)", func() { shoot.Spec.Networking.Nodes = &seedNodesCIDR shoot.Spec.ControlPlane = &core.ControlPlane{HighAvailability: &core.HighAvailability{FailureTolerance: core.FailureTolerance{Type: core.FailureToleranceTypeZone}}} diff --git a/test/integration/apiserver/admissionplugins/shootvalidator/shootvalidator_suite_test.go b/test/integration/apiserver/admissionplugins/shootvalidator/shootvalidator_suite_test.go index c4487b374f9..eb582ec7fd8 100644 --- a/test/integration/apiserver/admissionplugins/shootvalidator/shootvalidator_suite_test.go +++ b/test/integration/apiserver/admissionplugins/shootvalidator/shootvalidator_suite_test.go @@ -322,10 +322,6 @@ var _ = BeforeSuite(func() { Pods: "10.0.0.0/16", Services: "10.1.0.0/16", Nodes: ptr.To("10.2.0.0/16"), - ShootDefaults: &gardencorev1beta1.ShootNetworks{ - Pods: ptr.To("100.128.0.0/11"), - Services: ptr.To("100.72.0.0/13"), - }, }, }, } diff --git a/test/integration/apiserver/admissionplugins/shootvalidator/shootvalidator_test.go b/test/integration/apiserver/admissionplugins/shootvalidator/shootvalidator_test.go index 9617c45746d..3c05f9e8b7a 100644 --- a/test/integration/apiserver/admissionplugins/shootvalidator/shootvalidator_test.go +++ b/test/integration/apiserver/admissionplugins/shootvalidator/shootvalidator_test.go @@ -49,7 +49,11 @@ var _ = Describe("ShootValidator tests", func() { }, }, Kubernetes: gardencorev1beta1.Kubernetes{Version: "1.31.1"}, - Networking: &gardencorev1beta1.Networking{Type: ptr.To("foo-networking")}, + Networking: &gardencorev1beta1.Networking{ + Type: ptr.To("foo-networking"), + Pods: ptr.To("100.128.0.0/11"), + Services: ptr.To("100.72.0.0/13"), + }, }, } }) diff --git a/test/integration/controllermanager/bastion/bastion_suite_test.go b/test/integration/controllermanager/bastion/bastion_suite_test.go index 9d38837a036..24faf9f0a5b 100644 --- a/test/integration/controllermanager/bastion/bastion_suite_test.go +++ b/test/integration/controllermanager/bastion/bastion_suite_test.go @@ -69,7 +69,7 @@ var _ = BeforeSuite(func() { testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ Args: []string{ - "--disable-admission-plugins=Bastion,DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator", + "--disable-admission-plugins=Bastion,DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ShootMutator", }, }, } diff --git a/test/integration/controllermanager/shoot/migration/migration_suite_test.go b/test/integration/controllermanager/shoot/migration/migration_suite_test.go index f049c4817fa..88435c39f7d 100644 --- a/test/integration/controllermanager/shoot/migration/migration_suite_test.go +++ b/test/integration/controllermanager/shoot/migration/migration_suite_test.go @@ -56,7 +56,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/shootstate/shootstate_suite_test.go b/test/integration/controllermanager/shootstate/shootstate_suite_test.go index 3d386b5a2e7..ae6ce7f3154 100644 --- a/test/integration/controllermanager/shootstate/shootstate_suite_test.go +++ b/test/integration/controllermanager/shootstate/shootstate_suite_test.go @@ -63,7 +63,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/gardenlet/shoot/state/state_suite_test.go b/test/integration/gardenlet/shoot/state/state_suite_test.go index 943ba851706..403528d2964 100644 --- a/test/integration/gardenlet/shoot/state/state_suite_test.go +++ b/test/integration/gardenlet/shoot/state/state_suite_test.go @@ -91,7 +91,7 @@ var _ = BeforeSuite(func() { }, GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ Args: []string{ - "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator", + "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ShootMutator", }, }, } diff --git a/test/integration/gardenlet/shoot/status/status_suite_test.go b/test/integration/gardenlet/shoot/status/status_suite_test.go index bfe17b5de49..f6ecfc54a43 100644 --- a/test/integration/gardenlet/shoot/status/status_suite_test.go +++ b/test/integration/gardenlet/shoot/status/status_suite_test.go @@ -77,7 +77,7 @@ var _ = BeforeSuite(func() { }, GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ Args: []string{ - "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator", + "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ShootMutator", }, }, } From c4d228456f95d54608320fd102737fc3fa8e480a Mon Sep 17 00:00:00 2001 From: Johannes Scheerer Date: Thu, 30 Oct 2025 18:53:50 +0100 Subject: [PATCH 002/176] Upgrade `k8s.io/*` to `v0.34.1`, `sigs.k8s.io/controller-runtime` to `v0.22.3`, `sigs.k8s.io/controller-tools` to `v0.19.0` (#13238) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Upgrade k8s.io/* to `v0.34.1` * Run `make tidy` and `make generate` * Upgrade controller-runtime to `v0.22.3` * Upgrade controller-tools to `v0.19.0` * Run `make generate` * Use `ReconciliationTimeout` in controller options instead of custom context handling * Removed seedmanagement/applyconfiguration and settings/applyconfiguration. In the past, the corresponding packages were autogenerated, but since PR#8851 they are no longer updated. It does not look like we actually need them. Hence, removing them seems like the better option compared to re-enabling autogeneration. As a plus, this enables us to get rid of a direct dependency. * Adapt tests to `controller-runtime` changes. `.metadata.creationTimestamp` is no longer necessary to be set. `.preferences` is no longer necessary for kubeconfigs to be set. * Adapt tests for leader election validation changes (`ErrorTypeInvalid` => `ErrorTypeRequired`) * Adapt webhook tests to include new caching options * Adapt test to new validation error messages * Adapt fake client usage with custom resources including structs as pointer. Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. The resulting error looked like the following: ``` [38;5;13m• [PANICKED] [0.025 seconds][0m [0mController [38;5;13m[1m#MachineConditionChangedPredicate [BeforeEach] [0m#Create [38;5;243mshould return false when object is not machine[0m [38;5;13m[BeforeEach][0m [38;5;243m/home/prow/go/src/github.com/gardener/gardener/extensions/pkg/controller/worker/controller_test.go:38[0m [38;5;243m[It] /home/prow/go/src/github.com/gardener/gardener/extensions/pkg/controller/worker/controller_test.go:90[0m [38;5;13m[PANICKED] Test Panicked[0m [38;5;13mIn [1m[BeforeEach][0m[38;5;13m at: [1m/usr/local/go/src/reflect/value.go:1261[0m [38;5;243m@ 10/22/25 18:55:14.308[0m [38;5;13mreflect: call of reflect.Value.Field on ptr Value[0m ``` Changing the inclusion from pointer to non-pointer inclusion solves the issue, but just using a different object tracker seems less invasive. * Use utility functions for `TypeMeta` access. The fake client now clears `TypeMeta`. Hence, we need to use utility functions if we need to get the actual group/kind of resource objects. * Adapt tests to fake client clearing `TypeMeta` * Adapt test to fake client panicing on `Raw` structs being set together with `Object` --- charts/gardener/gardenlet/test/chart_test.go | 8 +- charts/gardener/gardenlet/test/test.go | 8 - .../operator/templates/crd-extensions.yaml | 2 +- .../operator/templates/crd-gardens.yaml | 2 +- ...rd-operator.gardener.cloud_extensions.yaml | 2 +- ...0-crd-operator.gardener.cloud_gardens.yaml | 2 +- ...urces.gardener.cloud_managedresources.yaml | 2 +- ...s.io_verticalpodautoscalercheckpoints.yaml | 2 +- ...scaling.k8s.io_verticalpodautoscalers.yaml | 2 +- ...tensions.gardener.cloud_backupbuckets.yaml | 2 +- ...tensions.gardener.cloud_backupentries.yaml | 2 +- ...rd-extensions.gardener.cloud_bastions.yaml | 2 +- ...rd-extensions.gardener.cloud_clusters.yaml | 2 +- ...ions.gardener.cloud_containerruntimes.yaml | 2 +- ...tensions.gardener.cloud_controlplanes.yaml | 2 +- ...-extensions.gardener.cloud_dnsrecords.yaml | 2 +- ...-extensions.gardener.cloud_extensions.yaml | 2 +- ...nsions.gardener.cloud_infrastructures.yaml | 2 +- ...rd-extensions.gardener.cloud_networks.yaml | 2 +- ...gardener.cloud_operatingsystemconfigs.yaml | 2 +- ...crd-extensions.gardener.cloud_workers.yaml | 7 +- ...rd-fluentbit.fluent.io_clusterfilters.yaml | 2 +- ...bit.fluent.io_clusterfluentbitconfigs.yaml | 2 +- ...crd-fluentbit.fluent.io_clusterinputs.yaml | 2 +- ...bit.fluent.io_clustermultilineparsers.yaml | 2 +- ...rd-fluentbit.fluent.io_clusteroutputs.yaml | 2 +- ...rd-fluentbit.fluent.io_clusterparsers.yaml | 2 +- ...10-crd-fluentbit.fluent.io_collectors.yaml | 140 ++++- .../10-crd-fluentbit.fluent.io_filters.yaml | 2 +- ...-fluentbit.fluent.io_fluentbitconfigs.yaml | 2 +- ...10-crd-fluentbit.fluent.io_fluentbits.yaml | 405 +++++++++++++-- ...-fluentbit.fluent.io_multilineparsers.yaml | 2 +- .../10-crd-fluentbit.fluent.io_outputs.yaml | 2 +- .../10-crd-fluentbit.fluent.io_parsers.yaml | 2 +- ...rd-machine.sapcloud.io_machineclasses.yaml | 2 +- ...achine.sapcloud.io_machinedeployments.yaml | 7 +- .../10-crd-machine.sapcloud.io_machines.yaml | 7 +- ...0-crd-machine.sapcloud.io_machinesets.yaml | 7 +- ...toring.coreos.com_alertmanagerconfigs.yaml | 2 +- ...d-monitoring.coreos.com_alertmanagers.yaml | 372 +++++++++++-- ...crd-monitoring.coreos.com_podmonitors.yaml | 2 +- .../10-crd-monitoring.coreos.com_probes.yaml | 2 +- ...onitoring.coreos.com_prometheusagents.yaml | 372 +++++++++++-- ...rd-monitoring.coreos.com_prometheuses.yaml | 374 +++++++++++-- ...monitoring.coreos.com_prometheusrules.yaml | 2 +- ...d-monitoring.coreos.com_scrapeconfigs.yaml | 2 +- ...monitoring.coreos.com_servicemonitors.yaml | 2 +- ...rd-monitoring.coreos.com_thanosrulers.yaml | 372 +++++++++++-- .../seed-crds/10-crd-perses.dev_perses.yaml | 6 +- .../10-crd-perses.dev_persesdashboards.yaml | 2 +- .../10-crd-perses.dev_persesdatasources.yaml | 2 +- ...urces.gardener.cloud_managedresources.yaml | 2 +- .../genericactuator/actuator_test.go | 3 +- .../pkg/controller/worker/controller_test.go | 7 +- .../actuator_reconcile_test.go | 8 + go.mod | 59 +-- go.sum | 148 +++--- pkg/api/extensions/accessor_test.go | 4 +- .../v1alpha1/zz_generated.defaults.go | 33 ++ pkg/apiserver/openapi/openapi_generated.go | 412 +++++++++++++-- pkg/client/kubernetes/applier_test.go | 2 + pkg/client/kubernetes/chartapplier_test.go | 7 - .../applyconfiguration/internal/internal.go | 50 -- .../seedmanagement/v1alpha1/gardenlet.go | 59 --- .../v1alpha1/gardenletdeployment.go | 137 ----- .../seedmanagement/v1alpha1/image.go | 49 -- .../seedmanagement/v1alpha1/managedseed.go | 206 -------- .../seedmanagement/v1alpha1/managedseedset.go | 205 -------- .../v1alpha1/managedseedsetspec.go | 77 --- .../v1alpha1/managedseedsetstatus.go | 123 ----- .../v1alpha1/managedseedspec.go | 36 -- .../v1alpha1/managedseedstatus.go | 42 -- .../v1alpha1/managedseedtemplate.go | 174 ------- .../seedmanagement/v1alpha1/pendingreplica.go | 59 --- .../v1alpha1/rollingupdatestrategy.go | 27 - .../seedmanagement/v1alpha1/shoot.go | 27 - .../seedmanagement/v1alpha1/updatestrategy.go | 40 -- .../applyconfiguration/utils.go | 51 -- .../applyconfiguration/internal/internal.go | 50 -- .../v1alpha1/clusteropenidconnectpreset.go | 195 ------- .../clusteropenidconnectpresetspec.go | 64 --- .../v1alpha1/kubeapiserveropenidconnect.go | 107 ---- .../openidconnectclientauthentication.go | 42 -- .../settings/v1alpha1/openidconnectpreset.go | 196 ------- .../v1alpha1/openidconnectpresetspec.go | 58 --- .../settings/applyconfiguration/utils.go | 35 -- ...s.io_verticalpodautoscalercheckpoints.yaml | 2 +- ...scaling.k8s.io_verticalpodautoscalers.yaml | 2 +- ...tensions.gardener.cloud_backupbuckets.yaml | 2 +- ...tensions.gardener.cloud_backupentries.yaml | 2 +- ...rd-extensions.gardener.cloud_bastions.yaml | 2 +- ...rd-extensions.gardener.cloud_clusters.yaml | 2 +- ...ions.gardener.cloud_containerruntimes.yaml | 2 +- ...tensions.gardener.cloud_controlplanes.yaml | 2 +- ...-extensions.gardener.cloud_dnsrecords.yaml | 2 +- ...-extensions.gardener.cloud_extensions.yaml | 2 +- ...nsions.gardener.cloud_infrastructures.yaml | 2 +- ...rd-extensions.gardener.cloud_networks.yaml | 2 +- ...gardener.cloud_operatingsystemconfigs.yaml | 2 +- ...crd-extensions.gardener.cloud_workers.yaml | 7 +- .../components/nodeagent/rbac_test.go | 3 - .../components/nodeagent/secrets_test.go | 5 +- .../opentelemetrycollector/config_test.go | 1 - pkg/component/gardener/access/access_test.go | 2 - .../gardener/apiserver/apiserver_test.go | 9 +- ...rd-dashboard.gardener.cloud_terminals.yaml | 4 +- ...urces.gardener.cloud_managedresources.yaml | 2 +- .../resourcemanager/resource_manager_test.go | 2 - .../kubernetes/apiserver/apiserver_test.go | 9 +- .../networking/coredns/coredns_test.go | 19 +- .../istio/test_charts/ingress_namespace.yaml | 1 - .../test_charts/ingress_servicemonitor.yaml | 1 - .../istio/test_charts/ingress_telemetry.yaml | 1 - .../test_charts/istiod_servicemonitor.yaml | 1 - .../nginxingress/nginxingress_test.go | 35 -- .../nodelocaldns/nodelocaldns_test.go | 10 - .../dependencywatchdog/access_test.go | 13 +- .../dependencywatchdog/bootstrap_test.go | 10 - .../machine_controller_manager_test.go | 4 - ...rd-machine.sapcloud.io_machineclasses.yaml | 2 +- ...achine.sapcloud.io_machinedeployments.yaml | 7 +- .../crd-machine.sapcloud.io_machines.yaml | 7 +- .../crd-machine.sapcloud.io_machinesets.yaml | 7 +- .../node_problem_detector_test.go | 7 - ...rd-fluentbit.fluent.io_clusterfilters.yaml | 2 +- ...bit.fluent.io_clusterfluentbitconfigs.yaml | 2 +- ...crd-fluentbit.fluent.io_clusterinputs.yaml | 2 +- ...bit.fluent.io_clustermultilineparsers.yaml | 2 +- ...rd-fluentbit.fluent.io_clusteroutputs.yaml | 2 +- ...rd-fluentbit.fluent.io_clusterparsers.yaml | 2 +- .../crd-fluentbit.fluent.io_collectors.yaml | 140 ++++- .../crd-fluentbit.fluent.io_filters.yaml | 2 +- ...-fluentbit.fluent.io_fluentbitconfigs.yaml | 2 +- .../crd-fluentbit.fluent.io_fluentbits.yaml | 405 +++++++++++++-- ...-fluentbit.fluent.io_multilineparsers.yaml | 2 +- .../crd-fluentbit.fluent.io_outputs.yaml | 2 +- .../crd-fluentbit.fluent.io_parsers.yaml | 2 +- .../blackboxexporter/blackboxexporter_test.go | 9 - .../metricsserver/metricsserver_test.go | 11 - .../nodeexporter/nodeexporter_test.go | 5 - .../templates/crd-perses.dev_perses.yaml | 6 +- .../crd-perses.dev_persesdashboards.yaml | 2 +- .../crd-perses.dev_persesdatasources.yaml | 2 +- ...toring.coreos.com_alertmanagerconfigs.yaml | 2 +- ...d-monitoring.coreos.com_alertmanagers.yaml | 372 +++++++++++-- ...crd-monitoring.coreos.com_podmonitors.yaml | 2 +- .../crd-monitoring.coreos.com_probes.yaml | 2 +- ...onitoring.coreos.com_prometheusagents.yaml | 372 +++++++++++-- ...rd-monitoring.coreos.com_prometheuses.yaml | 374 +++++++++++-- ...monitoring.coreos.com_prometheusrules.yaml | 2 +- ...d-monitoring.coreos.com_scrapeconfigs.yaml | 2 +- ...monitoring.coreos.com_servicemonitors.yaml | 2 +- ...rd-monitoring.coreos.com_thanosrulers.yaml | 372 +++++++++++-- ...crd-opentelemetry.io_instrumentations.yaml | 491 +++++++++++++++--- .../crd-opentelemetry.io_opampbridges.yaml | 176 ++++++- ...ntelemetry.io_opentelemetrycollectors.yaml | 460 ++++++++++++++-- ...crd-opentelemetry.io_targetallocators.yaml | 400 ++++++++++++-- .../observability/plutono/plutono_test.go | 4 - pkg/component/seed/system/system_test.go | 5 - pkg/component/shared/kubeapiserver_test.go | 2 + pkg/controller/networkpolicy/add.go | 2 + pkg/controller/networkpolicy/reconciler.go | 3 - pkg/controller/reference/add.go | 2 + pkg/controller/reference/reconciler.go | 3 - pkg/controller/tokenrequestor/add.go | 2 + pkg/controller/tokenrequestor/reconciler.go | 8 +- .../tokenrequestor/reconciler_test.go | 1 - pkg/controller/vpaevictionrequirements/add.go | 2 + .../vpaevictionrequirements/reconciler.go | 3 - .../v1alpha1/validation/validation_test.go | 2 +- .../controller/bastion/add.go | 2 + .../controller/bastion/reconciler.go | 4 - .../certificatesigningrequest/add.go | 2 + .../certificatesigningrequest/reconciler.go | 4 - .../controller/cloudprofile/add.go | 2 + .../controller/cloudprofile/reconciler.go | 3 - .../controller/controllerdeployment/add.go | 2 + .../controllerdeployment/reconciler.go | 3 - .../controllerregistrationfinalizer/add.go | 2 + .../reconciler.go | 3 - .../extensionclusterrole/add.go | 6 +- .../extensionclusterrole/reconciler.go | 3 - .../controllerregistration/seed/add.go | 2 + .../controllerregistration/seed/reconciler.go | 3 - .../seedfinalizer/add.go | 2 + .../seedfinalizer/reconciler.go | 3 - .../controller/credentialsbinding/add.go | 2 + .../credentialsbinding/reconciler.go | 15 +- pkg/controllermanager/controller/event/add.go | 2 + .../controller/event/reconciler.go | 4 - .../controller/exposureclass/add.go | 2 + .../controller/exposureclass/reconciler.go | 3 - .../controller/managedseedset/add.go | 1 + .../controller/managedseedset/reconciler.go | 3 - .../controller/namespacedcloudprofile/add.go | 1 + .../namespacedcloudprofile/reconciler.go | 3 - .../controller/project/activity/add.go | 2 + .../controller/project/activity/reconciler.go | 4 - .../controller/project/project/add.go | 2 + .../controller/project/project/reconciler.go | 3 - .../controller/project/stale/add.go | 1 + .../controller/project/stale/reconciler.go | 4 - pkg/controllermanager/controller/quota/add.go | 2 + .../controller/quota/reconciler.go | 3 - .../controller/secretbinding/add.go | 2 + .../controller/secretbinding/reconciler.go | 3 - .../controller/seed/backupbucketscheck/add.go | 1 + .../seed/backupbucketscheck/reconciler.go | 4 - .../controller/seed/extensionscheck/add.go | 1 + .../seed/extensionscheck/reconciler.go | 4 - .../controller/seed/lifecycle/add.go | 1 + .../controller/seed/lifecycle/reconciler.go | 4 - .../controller/seed/secrets/add.go | 2 + .../controller/seed/secrets/reconciler.go | 3 - .../controller/shoot/conditions/add.go | 2 + .../controller/shoot/conditions/reconciler.go | 4 - .../controller/shoot/hibernation/add.go | 2 + .../shoot/hibernation/reconciler.go | 4 - .../controller/shoot/maintenance/add.go | 2 + .../shoot/maintenance/reconciler.go | 3 - .../controller/shoot/migration/add.go | 2 + .../controller/shoot/migration/reconciler.go | 4 - .../controller/shoot/quota/add.go | 1 + .../controller/shoot/quota/reconciler.go | 4 - .../controller/shoot/retry/add.go | 2 + .../controller/shoot/retry/reconciler.go | 4 - .../controller/shoot/statuslabel/add.go | 2 + .../shoot/statuslabel/reconciler.go | 4 - pkg/controllerutils/miscellaneous.go | 8 - pkg/extensions/cluster_test.go | 7 +- pkg/gardenadm/staticpod/translator_test.go | 9 - .../v1alpha1/validation/validation_test.go | 2 +- pkg/gardenlet/controller/backupbucket/add.go | 2 + .../controller/backupbucket/reconciler.go | 17 +- pkg/gardenlet/controller/backupentry/add.go | 2 + .../controller/backupentry/reconciler.go | 17 +- pkg/gardenlet/controller/bastion/add.go | 2 + .../controller/bastion/reconciler.go | 3 +- .../controllerinstallation/care/add.go | 3 +- .../controllerinstallation/care/reconciler.go | 3 +- .../controllerinstallation/add.go | 2 + .../controllerinstallation/reconciler.go | 3 +- .../controllerinstallation/required/add.go | 1 + .../required/reconciler.go | 4 - pkg/gardenlet/controller/gardenlet/add.go | 1 + .../controller/gardenlet/reconciler.go | 4 - pkg/gardenlet/controller/managedseed/add.go | 1 + .../controller/managedseed/reconciler.go | 3 - pkg/gardenlet/controller/seed/care/add.go | 3 +- .../controller/seed/care/reconciler.go | 5 - pkg/gardenlet/controller/seed/lease/add.go | 1 + .../controller/seed/lease/reconciler.go | 3 - pkg/gardenlet/controller/shoot/care/add.go | 3 +- .../controller/shoot/care/health_test.go | 7 +- .../controller/shoot/care/reconciler.go | 5 - .../controller/shoot/shoot/cleaner_test.go | 7 +- pkg/gardenlet/controller/shoot/state/add.go | 6 +- .../controller/shoot/state/reconciler.go | 4 - pkg/gardenlet/controller/shoot/status/add.go | 6 +- .../controller/shoot/status/reconciler.go | 4 - .../tokenrequestor/workloadidentity/add.go | 2 + .../workloadidentity/reconciler.go | 8 +- .../operation/botanist/kubeapiserver_test.go | 1 - .../operation/botanist/kubeproxy_test.go | 1 - .../botanist/machinecontrollermanager_test.go | 7 +- .../operation/botanist/migration_test.go | 7 +- pkg/gardenlet/operation/botanist/secrets.go | 10 +- .../kubelet_bootstrap_kubeconfig_test.go | 1 - pkg/nodeagent/controller/node/add.go | 6 +- pkg/nodeagent/controller/node/reconciler.go | 4 - .../controller/operatingsystemconfig/add.go | 6 +- .../operatingsystemconfig/reconciler.go | 11 +- .../operatingsystemconfig/reconciler_test.go | 2 - pkg/nodeagent/controller/token/add.go | 7 +- pkg/nodeagent/controller/token/reconciler.go | 4 - .../v1alpha1/validation/validation_test.go | 2 +- pkg/operator/controller/extension/care/add.go | 1 + .../controller/extension/care/reconciler.go | 5 - .../controller/extension/extension/add.go | 2 + .../extension/extension/reconciler_delete.go | 6 +- .../extension/reconciler_reconcile.go | 7 +- .../extension/required/runtime/add.go | 1 + .../extension/required/runtime/reconciler.go | 4 - .../extension/required/virtual/add.go | 2 + .../extension/required/virtual/reconciler.go | 4 - pkg/operator/controller/garden/care/add.go | 1 + .../controller/garden/care/reconciler.go | 5 - pkg/operator/controller/gardenlet/add.go | 2 + .../controller/gardenlet/reconciler.go | 4 - .../v1alpha1/validation/validation_test.go | 2 +- .../controller/csrapprover/add.go | 2 + .../controller/csrapprover/reconciler.go | 4 - .../controller/garbagecollector/add.go | 1 + .../controller/garbagecollector/reconciler.go | 8 +- .../controller/health/health/add.go | 1 + .../controller/health/health/reconciler.go | 6 - .../controller/health/progressing/add.go | 1 + .../health/progressing/reconciler.go | 6 - .../controller/managedresource/add.go | 1 + .../controller/managedresource/reconciler.go | 14 +- .../controller/networkpolicy/add.go | 2 + .../controller/networkpolicy/reconciler.go | 3 - .../node/agentreconciliationdelay/add.go | 5 +- .../agentreconciliationdelay/reconciler.go | 8 +- .../controller/node/criticalcomponents/add.go | 2 + .../node/criticalcomponents/reconciler.go | 8 +- .../nodeagentauthorizer/authorizer_test.go | 7 +- .../v1alpha1/validation/validation_test.go | 2 +- pkg/scheduler/controller/shoot/add.go | 2 + pkg/scheduler/controller/shoot/reconciler.go | 4 - pkg/utils/gardener/machines_test.go | 7 +- .../gardener/shootstate/shootstate_test.go | 9 +- pkg/utils/managedresources/registry_test.go | 2 - .../validation/componentbaseconfig_test.go | 2 +- pkg/webhook/authorizer/handler_test.go | 12 +- .../managedseed/validator/admission_test.go | 4 +- .../operatingsystemconfig_test.go | 3 +- ...gardener.cloud_certificaterevocations.yaml | 2 +- ...-crd-cert.gardener.cloud_certificates.yaml | 2 +- .../10-crd-cert.gardener.cloud_issuers.yaml | 2 +- .../mock/client-go/kubernetes/mocks.go | 72 +-- .../mock/controller-runtime/client/mocks.go | 38 ++ 322 files changed, 5932 insertions(+), 3571 deletions(-) delete mode 100644 pkg/client/seedmanagement/applyconfiguration/internal/internal.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/gardenlet.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/gardenletdeployment.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/image.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseed.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedset.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedsetspec.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedsetstatus.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedspec.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedstatus.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedtemplate.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/pendingreplica.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/rollingupdatestrategy.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/shoot.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/updatestrategy.go delete mode 100644 pkg/client/seedmanagement/applyconfiguration/utils.go delete mode 100644 pkg/client/settings/applyconfiguration/internal/internal.go delete mode 100644 pkg/client/settings/applyconfiguration/settings/v1alpha1/clusteropenidconnectpreset.go delete mode 100644 pkg/client/settings/applyconfiguration/settings/v1alpha1/clusteropenidconnectpresetspec.go delete mode 100644 pkg/client/settings/applyconfiguration/settings/v1alpha1/kubeapiserveropenidconnect.go delete mode 100644 pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectclientauthentication.go delete mode 100644 pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectpreset.go delete mode 100644 pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectpresetspec.go delete mode 100644 pkg/client/settings/applyconfiguration/utils.go diff --git a/charts/gardener/gardenlet/test/chart_test.go b/charts/gardener/gardenlet/test/chart_test.go index 9456b59afc6..849c41d3a29 100644 --- a/charts/gardener/gardenlet/test/chart_test.go +++ b/charts/gardener/gardenlet/test/chart_test.go @@ -356,7 +356,7 @@ var _ = Describe("#Gardenlet Chart Test", func() { Provider: gardencorev1beta1.SeedProvider{}, }, }, - }, nil, nil, nil, nil, map[string]string{"gardenlet-configmap": "gardenlet-configmap-babc313f"}), + }, nil, nil, nil, nil, map[string]string{"gardenlet-configmap": "gardenlet-configmap-f3c6b5a9"}), Entry("verify deployment with two replica and three zones", nil, nil, nil, nil, nil, &gardenletconfigv1alpha1.SeedConfig{ SeedTemplate: gardencorev1beta1.SeedTemplate{ @@ -371,7 +371,7 @@ var _ = Describe("#Gardenlet Chart Test", func() { }, }, &seedmanagement.GardenletDeployment{ ReplicaCount: ptr.To[int32](2), - }, nil, nil, nil, map[string]string{"gardenlet-configmap": "gardenlet-configmap-39425435"}), + }, nil, nil, nil, map[string]string{"gardenlet-configmap": "gardenlet-configmap-7ce49ea8"}), Entry("verify deployment with only one replica", nil, nil, nil, nil, nil, &gardenletconfigv1alpha1.SeedConfig{ SeedTemplate: gardencorev1beta1.SeedTemplate{ @@ -386,7 +386,7 @@ var _ = Describe("#Gardenlet Chart Test", func() { }, }, &seedmanagement.GardenletDeployment{ ReplicaCount: ptr.To[int32](1), - }, nil, nil, nil, map[string]string{"gardenlet-configmap": "gardenlet-configmap-39425435"}), + }, nil, nil, nil, map[string]string{"gardenlet-configmap": "gardenlet-configmap-7ce49ea8"}), Entry("verify deployment with only one zone", nil, nil, nil, nil, nil, &gardenletconfigv1alpha1.SeedConfig{ SeedTemplate: gardencorev1beta1.SeedTemplate{ @@ -399,7 +399,7 @@ var _ = Describe("#Gardenlet Chart Test", func() { }, }, }, - }, nil, nil, nil, nil, map[string]string{"gardenlet-configmap": "gardenlet-configmap-8e8038ec"}), + }, nil, nil, nil, nil, map[string]string{"gardenlet-configmap": "gardenlet-configmap-3a4b364a"}), Entry("verify deployment with image vector override", nil, nil, nil, nil, nil, nil, nil, ptr.To("dummy-override-content"), nil, nil, map[string]string{ "gardenlet-configmap": "gardenlet-configmap-e03f6438", "gardenlet-imagevector-overwrite": "gardenlet-imagevector-overwrite-32ecb769", diff --git a/charts/gardener/gardenlet/test/test.go b/charts/gardener/gardenlet/test/test.go index 9ea34600356..925bdbad7f8 100644 --- a/charts/gardener/gardenlet/test/test.go +++ b/charts/gardener/gardenlet/test/test.go @@ -120,7 +120,6 @@ func ValidateGardenletChartRBAC(ctx context.Context, c client.Client, expectedLa func getGardenletClusterRole(labels map[string]string) *rbacv1.ClusterRole { return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: rbacv1.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{ Name: "gardener.cloud:system:gardenlet", Labels: labels, @@ -379,7 +378,6 @@ func getGardenletClusterRole(labels map[string]string) *rbacv1.ClusterRole { func getAPIServerSNIClusterRole(labels map[string]string) *rbacv1.ClusterRole { return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: rbacv1.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{ Name: "gardener.cloud:system:gardenlet:apiserver-sni", Labels: labels, @@ -409,7 +407,6 @@ func getAPIServerSNIClusterRole(labels map[string]string) *rbacv1.ClusterRole { func getManagedIstioClusterRole(labels map[string]string) *rbacv1.ClusterRole { return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: rbacv1.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{ Name: "gardener.cloud:system:gardenlet:managed-istio", Labels: labels, @@ -444,7 +441,6 @@ func getManagedIstioClusterRole(labels map[string]string) *rbacv1.ClusterRole { func getGardenletClusterRoleBinding(labels map[string]string, serviceAccountName string) *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: rbacv1.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{ Name: "gardener.cloud:system:gardenlet", Labels: labels, @@ -467,7 +463,6 @@ func getGardenletClusterRoleBinding(labels map[string]string, serviceAccountName func getAPIServerSNIClusterRoleBinding(labels map[string]string, serviceAccountName string) *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: rbacv1.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{ Name: "gardener.cloud:system:gardenlet:apiserver-sni", Labels: labels, @@ -490,7 +485,6 @@ func getAPIServerSNIClusterRoleBinding(labels map[string]string, serviceAccountN func getManagedIstioClusterRoleBinding(labels map[string]string, serviceAccountName string) *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: rbacv1.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{ Name: "gardener.cloud:system:gardenlet:managed-istio", Labels: labels, @@ -513,7 +507,6 @@ func getManagedIstioClusterRoleBinding(labels map[string]string, serviceAccountN func getGardenGardenletRole(labels map[string]string) *rbacv1.Role { return &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: rbacv1.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{ Name: "gardener.cloud:system:gardenlet", Namespace: "garden", @@ -532,7 +525,6 @@ func getGardenGardenletRole(labels map[string]string) *rbacv1.Role { func getGardenGardenletRoleBinding(labels map[string]string, serviceAccountName string) *rbacv1.RoleBinding { return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: rbacv1.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{ Name: "gardener.cloud:system:gardenlet", Namespace: "garden", diff --git a/charts/gardener/operator/templates/crd-extensions.yaml b/charts/gardener/operator/templates/crd-extensions.yaml index a8161032712..02f3f6f9872 100644 --- a/charts/gardener/operator/templates/crd-extensions.yaml +++ b/charts/gardener/operator/templates/crd-extensions.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: extensions.operator.gardener.cloud spec: group: operator.gardener.cloud diff --git a/charts/gardener/operator/templates/crd-gardens.yaml b/charts/gardener/operator/templates/crd-gardens.yaml index 927de52082f..1892a6751c3 100644 --- a/charts/gardener/operator/templates/crd-gardens.yaml +++ b/charts/gardener/operator/templates/crd-gardens.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: gardens.operator.gardener.cloud spec: group: operator.gardener.cloud diff --git a/example/operator/10-crd-operator.gardener.cloud_extensions.yaml b/example/operator/10-crd-operator.gardener.cloud_extensions.yaml index a8161032712..02f3f6f9872 100644 --- a/example/operator/10-crd-operator.gardener.cloud_extensions.yaml +++ b/example/operator/10-crd-operator.gardener.cloud_extensions.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: extensions.operator.gardener.cloud spec: group: operator.gardener.cloud diff --git a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml index 927de52082f..1892a6751c3 100644 --- a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml +++ b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: gardens.operator.gardener.cloud spec: group: operator.gardener.cloud diff --git a/example/resource-manager/10-crd-resources.gardener.cloud_managedresources.yaml b/example/resource-manager/10-crd-resources.gardener.cloud_managedresources.yaml index e24239b49bd..6ce5eeccc11 100644 --- a/example/resource-manager/10-crd-resources.gardener.cloud_managedresources.yaml +++ b/example/resource-manager/10-crd-resources.gardener.cloud_managedresources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: managedresources.resources.gardener.cloud spec: group: resources.gardener.cloud diff --git a/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalercheckpoints.yaml b/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalercheckpoints.yaml index 3adc7ba1ec4..cc28dc5ab71 100644 --- a/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalercheckpoints.yaml +++ b/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalercheckpoints.yaml @@ -4,7 +4,7 @@ kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes/kubernetes/pull/63797 - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: verticalpodautoscalercheckpoints.autoscaling.k8s.io spec: group: autoscaling.k8s.io diff --git a/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml b/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml index 6d1aad35d39..ee3a12bbe6a 100644 --- a/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml +++ b/example/seed-crds/10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml @@ -4,7 +4,7 @@ kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes/kubernetes/pull/63797 - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: verticalpodautoscalers.autoscaling.k8s.io spec: group: autoscaling.k8s.io diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_backupbuckets.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_backupbuckets.yaml index 708d9153fe6..1a8fa9ec50c 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_backupbuckets.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_backupbuckets.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: backupbuckets.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_backupentries.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_backupentries.yaml index 4302eaf29e8..1a0ce89ac74 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_backupentries.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_backupentries.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: backupentries.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_bastions.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_bastions.yaml index aa7f591ea91..00138957b66 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_bastions.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_bastions.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: bastions.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_clusters.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_clusters.yaml index 9b5611899d0..c670d2421b3 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_clusters.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusters.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_containerruntimes.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_containerruntimes.yaml index 1bad7e86aaf..84e534b215d 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_containerruntimes.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_containerruntimes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: containerruntimes.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_controlplanes.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_controlplanes.yaml index 601ef45e4d1..3acaee0b676 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_controlplanes.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_controlplanes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: controlplanes.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_dnsrecords.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_dnsrecords.yaml index 85315243dac..3021991231e 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_dnsrecords.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_dnsrecords.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: dnsrecords.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_extensions.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_extensions.yaml index db5adfb520d..12a7041fc8d 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_extensions.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_extensions.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: extensions.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_infrastructures.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_infrastructures.yaml index 59f37187695..a829864c14e 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_infrastructures.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_infrastructures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: infrastructures.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_networks.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_networks.yaml index 48e36d6074b..062cb41bec7 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_networks.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_networks.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: networks.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_operatingsystemconfigs.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_operatingsystemconfigs.yaml index fd702609742..d6147b312bf 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_operatingsystemconfigs.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_operatingsystemconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: operatingsystemconfigs.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/example/seed-crds/10-crd-extensions.gardener.cloud_workers.yaml b/example/seed-crds/10-crd-extensions.gardener.cloud_workers.yaml index 0f06d223d26..b7cf4e013db 100644 --- a/example/seed-crds/10-crd-extensions.gardener.cloud_workers.yaml +++ b/example/seed-crds/10-crd-extensions.gardener.cloud_workers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: workers.extensions.gardener.cloud spec: group: extensions.gardener.cloud @@ -648,9 +648,8 @@ spec: a node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which the + taint was added. format: date-time type: string value: diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfilters.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfilters.yaml index a5789bad004..624039566a0 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfilters.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfilters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusterfilters.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 8506b8bcccc..2aaeb309652 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusterfluentbitconfigs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterinputs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterinputs.yaml index 7931598ceb0..f3813ba4343 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterinputs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterinputs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusterinputs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clustermultilineparsers.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clustermultilineparsers.yaml index e6fe12dea7d..e56ff886a2d 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clustermultilineparsers.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clustermultilineparsers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clustermultilineparsers.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusteroutputs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusteroutputs.yaml index 79e5b28708d..f53fd251fb4 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusteroutputs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusteroutputs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusteroutputs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterparsers.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterparsers.yaml index e36b427f0b9..0e2983036e5 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterparsers.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterparsers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusterparsers.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_collectors.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_collectors.yaml index 0f7bc38be2f..c99f8c00ad4 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_collectors.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_collectors.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: collectors.fluentbit.fluent.io spec: group: fluentbit.fluent.io @@ -610,8 +610,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -1252,15 +1252,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -1420,13 +1418,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -1514,7 +1510,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2545,15 +2541,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -2735,12 +2729,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -2819,7 +2811,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -3239,6 +3231,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -3373,7 +3470,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_filters.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_filters.yaml index fcb3d3d56c4..55135835718 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_filters.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_filters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: filters.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbitconfigs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbitconfigs.yaml index 55ec28f4b8d..aafe67c640f 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbitconfigs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbitconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: fluentbitconfigs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbits.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbits.yaml index 506efe9bfa1..99e95fff5a2 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbits.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbits.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: fluentbits.fluentbit.fluent.io spec: group: fluentbit.fluent.io @@ -610,8 +610,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -1183,7 +1183,9 @@ spec: a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1241,6 +1243,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1391,8 +1430,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1450,6 +1490,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1510,8 +1587,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -1538,8 +1615,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -2210,7 +2288,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2264,10 +2342,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -2279,6 +2357,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -3707,15 +3838,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -3896,12 +4025,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details Glusterfs + topology. type: string path: description: |- @@ -3980,7 +4107,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -4392,6 +4519,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be addressed + to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -4526,7 +4758,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- @@ -5019,7 +5250,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -6055,15 +6286,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6245,12 +6474,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -6329,7 +6556,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -6749,6 +6976,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -6883,7 +7215,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_multilineparsers.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_multilineparsers.yaml index ec684d1414c..be64ff720a4 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_multilineparsers.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_multilineparsers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: multilineparsers.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_outputs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_outputs.yaml index 830a88f558e..7da1e0b4203 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_outputs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_outputs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: outputs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_parsers.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_parsers.yaml index 9d6c5136586..6bfe5e95284 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_parsers.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_parsers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: parsers.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/example/seed-crds/10-crd-machine.sapcloud.io_machineclasses.yaml b/example/seed-crds/10-crd-machine.sapcloud.io_machineclasses.yaml index fa3dffa06ec..9f7c5944e5f 100644 --- a/example/seed-crds/10-crd-machine.sapcloud.io_machineclasses.yaml +++ b/example/seed-crds/10-crd-machine.sapcloud.io_machineclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: machineclasses.machine.sapcloud.io spec: group: machine.sapcloud.io diff --git a/example/seed-crds/10-crd-machine.sapcloud.io_machinedeployments.yaml b/example/seed-crds/10-crd-machine.sapcloud.io_machinedeployments.yaml index 3f7b108ce82..de78891f050 100644 --- a/example/seed-crds/10-crd-machine.sapcloud.io_machinedeployments.yaml +++ b/example/seed-crds/10-crd-machine.sapcloud.io_machinedeployments.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: machinedeployments.machine.sapcloud.io spec: group: machine.sapcloud.io @@ -397,9 +397,8 @@ spec: to a node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at + which the taint was added. format: date-time type: string value: diff --git a/example/seed-crds/10-crd-machine.sapcloud.io_machines.yaml b/example/seed-crds/10-crd-machine.sapcloud.io_machines.yaml index cb88e5b4bf9..60a719acf87 100644 --- a/example/seed-crds/10-crd-machine.sapcloud.io_machines.yaml +++ b/example/seed-crds/10-crd-machine.sapcloud.io_machines.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: machines.machine.sapcloud.io spec: group: machine.sapcloud.io @@ -194,9 +194,8 @@ spec: a node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which + the taint was added. format: date-time type: string value: diff --git a/example/seed-crds/10-crd-machine.sapcloud.io_machinesets.yaml b/example/seed-crds/10-crd-machine.sapcloud.io_machinesets.yaml index 531232ba1c2..185ff6f7e68 100644 --- a/example/seed-crds/10-crd-machine.sapcloud.io_machinesets.yaml +++ b/example/seed-crds/10-crd-machine.sapcloud.io_machinesets.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: machinesets.machine.sapcloud.io spec: group: machine.sapcloud.io @@ -279,9 +279,8 @@ spec: to a node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at + which the taint was added. format: date-time type: string value: diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagerconfigs.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagerconfigs.yaml index 0298bac64ca..88429306f60 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagerconfigs.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagerconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: alertmanagerconfigs.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagers.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagers.yaml index c1319eb26a8..96fad4d0d5d 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagers.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: alertmanagers.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -674,8 +674,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -2887,8 +2887,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2946,6 +2947,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3006,8 +3044,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -3034,8 +3072,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -3706,7 +3745,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -3760,10 +3799,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -3775,6 +3814,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -4476,8 +4568,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4535,6 +4628,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4595,8 +4725,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -4623,8 +4753,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -5295,7 +5426,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -5349,10 +5480,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -5364,6 +5495,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -6009,7 +6193,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -6597,15 +6781,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6853,15 +7035,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -7023,13 +7203,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -8034,15 +8212,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -8224,12 +8400,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -8308,7 +8482,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -8728,6 +8902,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -8862,7 +9141,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_podmonitors.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_podmonitors.yaml index 5aac7b91cce..c888967e7c4 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_podmonitors.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_podmonitors.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: podmonitors.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_probes.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_probes.yaml index 0cc48a5bab3..4ac47e50aa1 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_probes.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_probes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: probes.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_prometheusagents.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_prometheusagents.yaml index 5d17a634635..74e741011b3 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_prometheusagents.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_prometheusagents.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: prometheusagents.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -705,8 +705,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -1510,8 +1510,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1569,6 +1570,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1629,8 +1667,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -1657,8 +1695,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -2329,7 +2368,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2383,10 +2422,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -2398,6 +2437,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -3337,8 +3429,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -3396,6 +3489,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3456,8 +3586,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -3484,8 +3614,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -4156,7 +4287,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -4210,10 +4341,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -4225,6 +4356,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -6354,7 +6538,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -7736,15 +7920,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -7992,15 +8174,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -8162,13 +8342,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -9432,15 +9610,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -9622,12 +9798,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -9706,7 +9880,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -10126,6 +10300,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -10260,7 +10539,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_prometheuses.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_prometheuses.yaml index 00768cd7ecc..c5420605b20 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_prometheuses.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_prometheuses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: prometheuses.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -783,8 +783,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -2248,8 +2248,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2307,6 +2308,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2367,8 +2405,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -2395,8 +2433,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -3067,7 +3106,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -3121,10 +3160,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -3136,6 +3175,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -4117,8 +4209,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4176,6 +4269,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4236,8 +4366,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -4264,8 +4394,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -4936,7 +5067,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -4990,10 +5121,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -5005,6 +5136,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -7920,7 +8104,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -9485,15 +9669,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -9741,15 +9923,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -9911,13 +10091,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -10304,7 +10482,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -11698,15 +11876,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -11888,12 +12064,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -11972,7 +12146,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -12392,6 +12566,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -12526,7 +12805,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_prometheusrules.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_prometheusrules.yaml index 6e41ee48a8d..f4800ee9ed9 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_prometheusrules.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_prometheusrules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: prometheusrules.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_scrapeconfigs.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_scrapeconfigs.yaml index e9157384362..b6a61bb8bad 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_scrapeconfigs.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_scrapeconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: scrapeconfigs.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_servicemonitors.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_servicemonitors.yaml index 0d868a6074e..5913993c581 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_servicemonitors.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_servicemonitors.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: servicemonitors.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_thanosrulers.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_thanosrulers.yaml index be079491bcd..2285730da9d 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_thanosrulers.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_thanosrulers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: thanosrulers.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -671,8 +671,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -1165,8 +1165,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1224,6 +1225,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1284,8 +1322,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -1312,8 +1350,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -1984,7 +2023,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2038,10 +2077,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -2053,6 +2092,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -2974,8 +3066,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -3033,6 +3126,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3093,8 +3223,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -3121,8 +3251,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -3793,7 +3924,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -3847,10 +3978,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -3862,6 +3993,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -5680,7 +5864,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -6374,15 +6558,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6630,15 +6812,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6800,13 +6980,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -7850,15 +8028,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -8040,12 +8216,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -8124,7 +8298,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -8544,6 +8718,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -8678,7 +8957,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/example/seed-crds/10-crd-perses.dev_perses.yaml b/example/seed-crds/10-crd-perses.dev_perses.yaml index 78ded4e5638..3a9e93b42ce 100644 --- a/example/seed-crds/10-crd-perses.dev_perses.yaml +++ b/example/seed-crds/10-crd-perses.dev_perses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: perses.perses.dev spec: group: perses.dev @@ -610,8 +610,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm diff --git a/example/seed-crds/10-crd-perses.dev_persesdashboards.yaml b/example/seed-crds/10-crd-perses.dev_persesdashboards.yaml index e88d62e47f1..301ea4eb0e6 100644 --- a/example/seed-crds/10-crd-perses.dev_persesdashboards.yaml +++ b/example/seed-crds/10-crd-perses.dev_persesdashboards.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: persesdashboards.perses.dev spec: group: perses.dev diff --git a/example/seed-crds/10-crd-perses.dev_persesdatasources.yaml b/example/seed-crds/10-crd-perses.dev_persesdatasources.yaml index 07c4e8a0fef..4e4838af27b 100644 --- a/example/seed-crds/10-crd-perses.dev_persesdatasources.yaml +++ b/example/seed-crds/10-crd-perses.dev_persesdatasources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: persesdatasources.perses.dev spec: group: perses.dev diff --git a/example/seed-crds/10-crd-resources.gardener.cloud_managedresources.yaml b/example/seed-crds/10-crd-resources.gardener.cloud_managedresources.yaml index e24239b49bd..6ce5eeccc11 100644 --- a/example/seed-crds/10-crd-resources.gardener.cloud_managedresources.yaml +++ b/example/seed-crds/10-crd-resources.gardener.cloud_managedresources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: managedresources.resources.gardener.cloud spec: group: resources.gardener.cloud diff --git a/extensions/pkg/controller/controlplane/genericactuator/actuator_test.go b/extensions/pkg/controller/controlplane/genericactuator/actuator_test.go index 31924d909e8..97a49951d69 100644 --- a/extensions/pkg/controller/controlplane/genericactuator/actuator_test.go +++ b/extensions/pkg/controller/controlplane/genericactuator/actuator_test.go @@ -317,8 +317,7 @@ var _ = Describe("Actuator", func() { if webhookConfig != nil { compressedData, err := test.BrotliCompressionForManifests(`apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration -metadata: - creationTimestamp: null +metadata: {} webhooks: - admissionReviewVersions: null clientConfig: {} diff --git a/extensions/pkg/controller/worker/controller_test.go b/extensions/pkg/controller/worker/controller_test.go index bde440cbad9..278ab079d5f 100644 --- a/extensions/pkg/controller/worker/controller_test.go +++ b/extensions/pkg/controller/worker/controller_test.go @@ -13,6 +13,8 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/event" @@ -37,7 +39,10 @@ var _ = Describe("Controller", func() { BeforeEach(func() { ctx = context.Background() - c = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder()) + c = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithObjectTracker(objectTracker).Build() machineDeployment = &machinev1alpha1.MachineDeployment{ ObjectMeta: metav1.ObjectMeta{ diff --git a/extensions/pkg/controller/worker/genericactuator/actuator_reconcile_test.go b/extensions/pkg/controller/worker/genericactuator/actuator_reconcile_test.go index 896c26f0807..828155b4e47 100644 --- a/extensions/pkg/controller/worker/genericactuator/actuator_reconcile_test.go +++ b/extensions/pkg/controller/worker/genericactuator/actuator_reconcile_test.go @@ -12,6 +12,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -39,9 +41,12 @@ var _ = Describe("ActuatorReconcile", func() { ) BeforeEach(func() { + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. seedClient = fakeclient.NewClientBuilder(). WithScheme(kubernetes.SeedScheme). WithStatusSubresource(&extensionsv1alpha1.Worker{}, &machinev1alpha1.MachineDeployment{}). + WithObjectTracker(testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder())). Build() ctx = context.Background() @@ -171,9 +176,12 @@ var _ = Describe("ActuatorReconcile", func() { ) BeforeEach(func() { + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. seedClient = fakeclient.NewClientBuilder(). WithScheme(kubernetes.SeedScheme). WithStatusSubresource(&extensionsv1alpha1.Worker{}, &machinev1alpha1.MachineDeployment{}). + WithObjectTracker(testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder())). Build() ctx = context.Background() diff --git a/go.mod b/go.mod index 2cd653a7fba..6402cf360d3 100644 --- a/go.mod +++ b/go.mod @@ -62,31 +62,30 @@ require ( helm.sh/helm/v3 v3.18.6 istio.io/api v1.27.3 istio.io/client-go v1.27.2 - k8s.io/api v0.33.5 - k8s.io/apiextensions-apiserver v0.33.5 - k8s.io/apimachinery v0.33.5 - k8s.io/apiserver v0.33.5 + k8s.io/api v0.34.1 + k8s.io/apiextensions-apiserver v0.34.1 + k8s.io/apimachinery v0.34.1 + k8s.io/apiserver v0.34.1 // TODO(vitanovs): Update k8s.io/autoscaler/vertical-pod-autoscaler to v1.5.1 once https://github.com/gardener/gardener/issues/13162 is resolved. k8s.io/autoscaler/vertical-pod-autoscaler v1.4.2 - k8s.io/cli-runtime v0.33.5 - k8s.io/client-go v0.33.5 - k8s.io/cluster-bootstrap v0.33.5 - k8s.io/code-generator v0.33.5 - k8s.io/component-base v0.33.5 - k8s.io/component-helpers v0.33.5 + k8s.io/cli-runtime v0.34.1 + k8s.io/client-go v0.34.1 + k8s.io/cluster-bootstrap v0.34.1 + k8s.io/code-generator v0.34.1 + k8s.io/component-base v0.34.1 + k8s.io/component-helpers v0.34.1 k8s.io/klog/v2 v2.130.1 - k8s.io/kube-aggregator v0.33.5 - k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 - k8s.io/kube-proxy v0.33.5 + k8s.io/kube-aggregator v0.34.1 + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b + k8s.io/kube-proxy v0.34.1 k8s.io/kube-state-metrics/v2 v2.13.0 - k8s.io/kubelet v0.33.5 - k8s.io/metrics v0.33.5 - k8s.io/pod-security-admission v0.33.5 + k8s.io/kubelet v0.34.1 + k8s.io/metrics v0.34.1 + k8s.io/pod-security-admission v0.34.1 k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 - sigs.k8s.io/controller-runtime v0.21.0 - sigs.k8s.io/controller-tools v0.18.0 - sigs.k8s.io/structured-merge-diff/v4 v4.7.0 - sigs.k8s.io/yaml v1.5.0 + sigs.k8s.io/controller-runtime v0.22.3 + sigs.k8s.io/controller-tools v0.19.0 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -136,7 +135,7 @@ require ( github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/fxamacker/cbor/v2 v2.8.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect @@ -152,7 +151,7 @@ require ( github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/cel-go v0.25.0 // indirect + github.com/google/cel-go v0.26.0 // indirect github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/mux v1.8.1 // indirect @@ -191,7 +190,7 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/muhlemmer/gu v0.3.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect @@ -204,6 +203,7 @@ require ( github.com/perses/common v0.27.1-0.20250326140707-96e439b14e0e // indirect github.com/perses/perses v0.51.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/otlptranslator v0.0.2 // indirect github.com/prometheus/procfs v0.17.0 // indirect @@ -226,15 +226,15 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/zitadel/oidc/v3 v3.38.1 // indirect github.com/zitadel/schema v1.3.1 // indirect - go.etcd.io/etcd/api/v3 v3.5.21 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.21 // indirect - go.etcd.io/etcd/client/v3 v3.5.21 // indirect + go.etcd.io/etcd/api/v3 v3.6.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect + go.etcd.io/etcd/client/v3 v3.6.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/collector/featuregate v1.37.0 // indirect go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 // indirect go.opentelemetry.io/contrib/exporters/autoexport v0.57.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect go.opentelemetry.io/contrib/otelconf v0.18.0 // indirect go.opentelemetry.io/otel v1.38.0 // indirect @@ -276,12 +276,13 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect - k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 // indirect + k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f // indirect k8s.io/klog v1.0.0 // indirect - k8s.io/kms v0.33.5 // indirect + k8s.io/kms v0.34.1 // indirect k8s.io/sample-controller v0.30.3 // indirect oras.land/oras-go/v2 v2.6.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect ) diff --git a/go.sum b/go.sum index 3cb4f90db88..3b8452fd296 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU= -github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gardener/cert-management v0.18.0 h1:s2YhkN8z7lXe9En52GCeqQ9be10uEbLtH/FFAh6BVgQ= github.com/gardener/cert-management v0.18.0/go.mod h1:9+JT+EBJB2OIX65EG+P1p/DZ/UJ3W8WR0h40ZjKbw+Q= github.com/gardener/dependency-watchdog v1.6.0 h1:ARCIbcNmhjefmV7ex8ADReeD2MPsEawwT/MoZKTQV/M= @@ -310,8 +310,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= -github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -342,8 +340,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY= -github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -396,11 +394,12 @@ github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrR github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/hashicorp/consul/api v1.30.0 h1:ArHVMMILb1nQv8vZSGIwwQd2gtc+oSQZ6CalyiyH2XQ= @@ -451,8 +450,8 @@ github.com/ironcore-dev/vgopath v0.1.5 h1:+I46zEFfbmNIGIGylqedT2bMXw8V7yVP16GJkG github.com/ironcore-dev/vgopath v0.1.5/go.mod h1:qbSUA7Eg0SO97OYfkG0DH+DxaPrH6XCiAQHqqs9R63Q= github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY= github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= -github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= @@ -547,8 +546,9 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/muhlemmer/gu v0.3.1 h1:7EAqmFrW7n3hETvuAdmFmn4hS8W+z3LgKtrnow+YzNM= github.com/muhlemmer/gu v0.3.1/go.mod h1:YHtHR+gxM+bKEIIs7Hmi9sPT3ZDUvTN/i88wQpZkrdM= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -749,22 +749,20 @@ github.com/zitadel/oidc/v3 v3.38.1 h1:VTf1Bv/33UbSwJnIWbfEIdpUGYKfoHetuBNIqVTcjv github.com/zitadel/oidc/v3 v3.38.1/go.mod h1:muukzAasaWmn3vBwEVMglJfuTE0PKCvLJGombPwXIRw= github.com/zitadel/schema v1.3.1 h1:QT3kwiRIRXXLVAs6gCK/u044WmUVh6IlbLXUsn6yRQU= github.com/zitadel/schema v1.3.1/go.mod h1:071u7D2LQacy1HAN+YnMd/mx1qVE2isb0Mjeqg46xnU= -go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= -go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.etcd.io/etcd/api/v3 v3.5.21 h1:A6O2/JDb3tvHhiIz3xf9nJ7REHvtEFJJ3veW3FbCnS8= -go.etcd.io/etcd/api/v3 v3.5.21/go.mod h1:c3aH5wcvXv/9dqIw2Y810LDXJfhSYdHQ0vxmP3CCHVY= -go.etcd.io/etcd/client/pkg/v3 v3.5.21 h1:lPBu71Y7osQmzlflM9OfeIV2JlmpBjqBNlLtcoBqUTc= -go.etcd.io/etcd/client/pkg/v3 v3.5.21/go.mod h1:BgqT/IXPjK9NkeSDjbzwsHySX3yIle2+ndz28nVsjUs= -go.etcd.io/etcd/client/v2 v2.305.21 h1:eLiFfexc2mE+pTLz9WwnoEsX5JTTpLCYVivKkmVXIRA= -go.etcd.io/etcd/client/v2 v2.305.21/go.mod h1:OKkn4hlYNf43hpjEM3Ke3aRdUkhSl8xjKjSf8eCq2J8= -go.etcd.io/etcd/client/v3 v3.5.21 h1:T6b1Ow6fNjOLOtM0xSoKNQt1ASPCLWrF9XMHcH9pEyY= -go.etcd.io/etcd/client/v3 v3.5.21/go.mod h1:mFYy67IOqmbRf/kRUvsHixzo3iG+1OF2W2+jVIQRAnU= -go.etcd.io/etcd/pkg/v3 v3.5.21 h1:jUItxeKyrDuVuWhdh0HtjUANwyuzcb7/FAeUfABmQsk= -go.etcd.io/etcd/pkg/v3 v3.5.21/go.mod h1:wpZx8Egv1g4y+N7JAsqi2zoUiBIUWznLjqJbylDjWgU= -go.etcd.io/etcd/raft/v3 v3.5.21 h1:dOmE0mT55dIUsX77TKBLq+RgyumsQuYeiRQnW/ylugk= -go.etcd.io/etcd/raft/v3 v3.5.21/go.mod h1:fmcuY5R2SNkklU4+fKVBQi2biVp5vafMrWUEj4TJ4Cs= -go.etcd.io/etcd/server/v3 v3.5.21 h1:9w0/k12majtgarGmlMVuhwXRI2ob3/d1Ik3X5TKo0yU= -go.etcd.io/etcd/server/v3 v3.5.21/go.mod h1:G1mOzdwuzKT1VRL7SqRchli/qcFrtLBTAQ4lV20sXXo= +go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= +go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= +go.etcd.io/etcd/api/v3 v3.6.4 h1:7F6N7toCKcV72QmoUKa23yYLiiljMrT4xCeBL9BmXdo= +go.etcd.io/etcd/api/v3 v3.6.4/go.mod h1:eFhhvfR8Px1P6SEuLT600v+vrhdDTdcfMzmnxVXXSbk= +go.etcd.io/etcd/client/pkg/v3 v3.6.4 h1:9HBYrjppeOfFjBjaMTRxT3R7xT0GLK8EJMVC4xg6ok0= +go.etcd.io/etcd/client/pkg/v3 v3.6.4/go.mod h1:sbdzr2cl3HzVmxNw//PH7aLGVtY4QySjQFuaCgcRFAI= +go.etcd.io/etcd/client/v3 v3.6.4 h1:YOMrCfMhRzY8NgtzUsHl8hC2EBSnuqbR3dh84Uryl7A= +go.etcd.io/etcd/client/v3 v3.6.4/go.mod h1:jaNNHCyg2FdALyKWnd7hxZXZxZANb0+KGY+YQaEMISo= +go.etcd.io/etcd/pkg/v3 v3.6.4 h1:fy8bmXIec1Q35/jRZ0KOes8vuFxbvdN0aAFqmEfJZWA= +go.etcd.io/etcd/pkg/v3 v3.6.4/go.mod h1:kKcYWP8gHuBRcteyv6MXWSN0+bVMnfgqiHueIZnKMtE= +go.etcd.io/etcd/server/v3 v3.6.4 h1:LsCA7CzjVt+8WGrdsnh6RhC0XqCsLkBly3ve5rTxMAU= +go.etcd.io/etcd/server/v3 v3.6.4/go.mod h1:aYCL/h43yiONOv0QIR82kH/2xZ7m+IWYjzRmyQfnCAg= +go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= +go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -778,8 +776,8 @@ go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 h1:UW0+QyeyBVhn+COBec3nGh go.opentelemetry.io/contrib/bridges/prometheus v0.57.0/go.mod h1:ppciCHRLsyCio54qbzQv0E4Jyth/fLWDTJYfvWpcSVk= go.opentelemetry.io/contrib/exporters/autoexport v0.57.0 h1:jmTVJ86dP60C01K3slFQa2NQ/Aoi7zA+wy7vMOKD9H4= go.opentelemetry.io/contrib/exporters/autoexport v0.57.0/go.mod h1:EJBheUMttD/lABFyLXhce47Wr6DPWYReCzaZiXadH7g= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 h1:PS8wXpbyaDJQ2VDHHncMe9Vct0Zn1fEjpsjrLxGJoSc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0/go.mod h1:HDBUsEjOuRC0EzKZ1bSaRGZWUBAzo+MhAcUUORSr4D0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY= go.opentelemetry.io/contrib/otelconf v0.18.0 h1:ciF2Gf00BWs0DnexKFZXcxg9kJ8r3SUW1LOzW3CsKA8= @@ -1087,38 +1085,38 @@ istio.io/api v1.27.3/go.mod h1:DTVGH6CLXj5W8FF9JUD3Tis78iRgT1WeuAnxfTz21Wg= istio.io/client-go v1.27.2 h1:4IsF7UAdV5Yg0iq6ONyWZpjFr3z2ahkIbLWyzOHCAwA= istio.io/client-go v1.27.2/go.mod h1:zgT5R1USl6rwYK1eb2kisPuiji05TQJE7CQHU253iAg= k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= -k8s.io/api v0.33.5 h1:YR+uhYj05jdRpcksv8kjSliW+v9hwXxn6Cv10aR8Juw= -k8s.io/api v0.33.5/go.mod h1:2gzShdwXKT5yPGiqrTrn/U/nLZ7ZyT4WuAj3XGDVgVs= -k8s.io/apiextensions-apiserver v0.33.5 h1:93NZh6rmrcamX/tfv/dZrTsMiQX69ufANmDcKPEgSeA= -k8s.io/apiextensions-apiserver v0.33.5/go.mod h1:JIbyQnNlu6nQa7b1vgFi51pmlXOk8mdn0WJwUJnz/7U= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= +k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.33.5 h1:NiT64hln4TQXeYR18/ES39OrNsjGz8NguxsBgp+6QIo= -k8s.io/apimachinery v0.33.5/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/apiserver v0.33.5 h1:X1Gy33r4YkRLRqTjGjofk7X1/EjSLEVSJ/A+1qjoj60= -k8s.io/apiserver v0.33.5/go.mod h1:Q+b5Btbc8x0PqOCeh/xBTesKk+cXQRN+PF2wdrTKDeg= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apiserver v0.34.1 h1:U3JBGdgANK3dfFcyknWde1G6X1F4bg7PXuvlqt8lITA= +k8s.io/apiserver v0.34.1/go.mod h1:eOOc9nrVqlBI1AFCvVzsob0OxtPZUCPiUJL45JOTBG0= k8s.io/autoscaler/vertical-pod-autoscaler v1.4.2 h1:47RLgLhrxXfdBchTeNT2S9Xe9o+Y4kThExLfcGGUQMk= k8s.io/autoscaler/vertical-pod-autoscaler v1.4.2/go.mod h1:rIBiAf+sK2mw8ryeHIZuY5juhJ4e2rNLwo59SDRXF7I= -k8s.io/cli-runtime v0.33.5 h1:wM7DoglOkrJDmddla864mVpueaEDX7/XGAkHGMWQkpc= -k8s.io/cli-runtime v0.33.5/go.mod h1:ZmUR+ybq97SqxSkkqGQdIhzCfk/+ETUhwKQq5EguaCw= +k8s.io/cli-runtime v0.34.1 h1:btlgAgTrYd4sk8vJTRG6zVtqBKt9ZMDeQZo2PIzbL7M= +k8s.io/cli-runtime v0.34.1/go.mod h1:aVA65c+f0MZiMUPbseU/M9l1Wo2byeaGwUuQEQVVveE= k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= -k8s.io/client-go v0.33.5 h1:I8BdmQGxInpkMEnJvV6iG7dqzP3JRlpZZlib3OMFc3o= -k8s.io/client-go v0.33.5/go.mod h1:W8PQP4MxbM4ypgagVE65mUUqK1/ByQkSALF9tzuQ6u0= -k8s.io/cluster-bootstrap v0.33.5 h1:VO0BhAwtAa6PYbBTkpK6mZwkGs8dlbeAN8pXbEovWWw= -k8s.io/cluster-bootstrap v0.33.5/go.mod h1:jPhTDgjG8RfxGkNJQzt3Qffq/KcYGRJI+UbKeJvEFyw= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/cluster-bootstrap v0.34.1 h1:lyCwJKoeYzGI93vk5Sn/Gz2rzfTRXkRuZYOk2rUsHfA= +k8s.io/cluster-bootstrap v0.34.1/go.mod h1:9EJfkp7Fu4YBU0F6ysvrI5TndWLo8zufmDSjIWBNd94= k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/code-generator v0.33.5 h1:KwkOvhwAaorjSwF2MQhhdhL3i8bBmAal/TWhX67kdHw= -k8s.io/code-generator v0.33.5/go.mod h1:Ra+sdZquRakeTGcEnQAPw6BmlZ92IvxwQQTX/XOvOIE= -k8s.io/component-base v0.33.5 h1:4D3kxjEx1pJRy3WHAZsmX3+LCpmd4ftE+2J4v6naTnQ= -k8s.io/component-base v0.33.5/go.mod h1:Zma1YjBVuuGxIbspj1vGR3/5blzo2ARf1v0QTtog1to= -k8s.io/component-helpers v0.33.5 h1:1LDSMzn7YTreVLPaOBJK36ase/FWi2sDpeJJvbEBO2s= -k8s.io/component-helpers v0.33.5/go.mod h1:C3HsDU2lANSLgTTgMJ0TFnG5xZrVrxR3Ss9n7Wrsw4s= +k8s.io/code-generator v0.34.1 h1:WpphT26E+j7tEgIUfFr5WfbJrktCGzB3JoJH9149xYc= +k8s.io/code-generator v0.34.1/go.mod h1:DeWjekbDnJWRwpw3s0Jat87c+e0TgkxoR4ar608yqvg= +k8s.io/component-base v0.34.1 h1:v7xFgG+ONhytZNFpIz5/kecwD+sUhVE6HU7qQUiRM4A= +k8s.io/component-base v0.34.1/go.mod h1:mknCpLlTSKHzAQJJnnHVKqjxR7gBeHRv0rPXA7gdtQ0= +k8s.io/component-helpers v0.34.1 h1:gWhH3CCdwAx5P3oJqZKb4Lg5FYZTWVbdWtOI8n9U4XY= +k8s.io/component-helpers v0.34.1/go.mod h1:4VgnUH7UA/shuBur+OWoQC0xfb69sy/93ss0ybZqm3c= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201203183100-97869a43a9d9/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 h1:pWEwq4Asjm4vjW7vcsmijwBhOr1/shsbSYiWXmNGlks= k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 h1:2OX19X59HxDprNCVrWi6jb7LW1PoqTlYqEq5H2oetog= -k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f h1:SLb+kxmzfA87x4E4brQzB33VBbT2+x7Zq9ROIHmGn9Q= +k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= @@ -1126,23 +1124,23 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kms v0.33.5 h1:u60sDBe4Fz6PLeiNF13sEahjIU428ajiWuIw7NSVBlg= -k8s.io/kms v0.33.5/go.mod h1:C1I8mjFFBNzfUZXYt9FZVJ8MJl7ynFbGgZFbBzkBJ3E= -k8s.io/kube-aggregator v0.33.5 h1:5libMG9e4m9lwhNBT89bBCd9x/rZebMahw5CHq9DE/Q= -k8s.io/kube-aggregator v0.33.5/go.mod h1:mHmmDqxY2ZkInu7eSAXb1ecaKV/U9DqPTQWBV2O84go= +k8s.io/kms v0.34.1 h1:iCFOvewDPzWM9fMTfyIPO+4MeuZ0tcZbugxLNSHFG4w= +k8s.io/kms v0.34.1/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= +k8s.io/kube-aggregator v0.34.1 h1:WNLV0dVNoFKmuyvdWLd92iDSyD/TSTjqwaPj0U9XAEU= +k8s.io/kube-aggregator v0.34.1/go.mod h1:RU8j+5ERfp0h+gIvWtxRPfsa5nK7rboDm8RST8BJfYQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 h1:gAXU86Fmbr/ktY17lkHwSjw5aoThQvhnstGGIYKlKYc= -k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911/go.mod h1:GLOk5B+hDbRROvt0X2+hqX64v/zO3vXN7J78OUmBSKw= -k8s.io/kube-proxy v0.33.5 h1:wgzKB5pkDl8hlRQTE+Va5t9DTu40UZJGhIrWod/id90= -k8s.io/kube-proxy v0.33.5/go.mod h1:w7tesj0OkcF8UVFRSv8Ja1ohYrGgGcAmandGqB+orz0= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-proxy v0.34.1 h1:cIriNCJY5XmRhXCCyQiazyqi47lbwcBQf0H76fVOpkw= +k8s.io/kube-proxy v0.34.1/go.mod h1:syed9c5+gUVFMo6p24SnlTHzsp+BMd4ACcTw2dbArw0= k8s.io/kube-state-metrics/v2 v2.13.0 h1:g5OV0cwrDKvyrQApYxI7Ny+vFUgU3X6MYrnIzdCAdNU= k8s.io/kube-state-metrics/v2 v2.13.0/go.mod h1:sGt/NFkZkA4hqb4cVd/xG2G17dzZ72TQXqSpHn8rF/U= -k8s.io/kubelet v0.33.5 h1:PYV+O8B6ZoQMDaQdYTSCzNdQTLdjAAWTspS2KkNfjLQ= -k8s.io/kubelet v0.33.5/go.mod h1:8SQ/0fgyNwm6zjHktBhPAUlgtji19YMa2lhSlYKUhrA= -k8s.io/metrics v0.33.5 h1:dOG+Yh4SZ6TJ7LcjvVXrYFvZ9G+sogCpTWyhoOVgQCE= -k8s.io/metrics v0.33.5/go.mod h1:YvBlRD01hos/j7dmjdtSBaZbmGyCJrxVS977x5+Y1kk= -k8s.io/pod-security-admission v0.33.5 h1:wThuFFa3ykpClGqtjRKEn/7zaIuinvvqgpzPdHPVwOs= -k8s.io/pod-security-admission v0.33.5/go.mod h1:hlFjlok9N9X94rqbG3/xNU1m4V4hiuANvlIk/lL9hXE= +k8s.io/kubelet v0.34.1 h1:doAaTA9/Yfzbdq/u/LveZeONp96CwX9giW6b+oHn4m4= +k8s.io/kubelet v0.34.1/go.mod h1:PtV3Ese8iOM19gSooFoQT9iyRisbmJdAPuDImuccbbA= +k8s.io/metrics v0.34.1 h1:374Rexmp1xxgRt64Bi0TsjAM8cA/Y8skwCoPdjtIslE= +k8s.io/metrics v0.34.1/go.mod h1:Drf5kPfk2NJrlpcNdSiAAHn/7Y9KqxpRNagByM7Ei80= +k8s.io/pod-security-admission v0.34.1 h1:XsP5eh8qCj69hK0a5TBMU4Ed7Ckn8JEmmbk/iepj+XM= +k8s.io/pod-security-admission v0.34.1/go.mod h1:87yY36Gxc8Hjx24FxqAD5zMY4k0tP0u7Mu/XuwXEbmg= k8s.io/sample-controller v0.30.3 h1:oZTxERF8U3gANT2H5VxpkW32asgmW0IYGyUv9Opspvs= k8s.io/sample-controller v0.30.3/go.mod h1:yhy/cWCzevQLa2+7Gvj0J9+xzmNExypunffSNANBy7o= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1153,22 +1151,20 @@ oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= -sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= -sigs.k8s.io/controller-tools v0.18.0 h1:rGxGZCZTV2wJreeRgqVoWab/mfcumTMmSwKzoM9xrsE= -sigs.k8s.io/controller-tools v0.18.0/go.mod h1:gLKoiGBriyNh+x1rWtUQnakUYEujErjXs9pf+x/8n1U= +sigs.k8s.io/controller-runtime v0.22.3 h1:I7mfqz/a/WdmDCEnXmSPm8/b/yRTy6JsKKENTijTq8Y= +sigs.k8s.io/controller-runtime v0.22.3/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= +sigs.k8s.io/controller-tools v0.19.0 h1:OU7jrPPiZusryu6YK0jYSjPqg8Vhf8cAzluP9XGI5uk= +sigs.k8s.io/controller-tools v0.19.0/go.mod h1:y5HY/iNDFkmFla2CfQoVb2AQXMsBk4ad84iR1PLANB0= sigs.k8s.io/gateway-api v1.3.0 h1:q6okN+/UKDATola4JY7zXzx40WO4VISk7i9DIfOvr9M= sigs.k8s.io/gateway-api v1.3.0/go.mod h1:d8NV8nJbaRbEKem+5IuxkL8gJGOZ+FJ+NvOIltV8gDk= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI= -sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= -sigs.k8s.io/yaml v1.5.0 h1:M10b2U7aEUY6hRtU870n2VTPgR5RZiL/I6Lcc2F4NUQ= -sigs.k8s.io/yaml v1.5.0/go.mod h1:wZs27Rbxoai4C0f8/9urLZtZtF3avA3gKvGyPdDqTO4= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/pkg/api/extensions/accessor_test.go b/pkg/api/extensions/accessor_test.go index 29bfc756b0f..299a286dc76 100644 --- a/pkg/api/extensions/accessor_test.go +++ b/pkg/api/extensions/accessor_test.go @@ -92,7 +92,7 @@ var _ = Describe("Accessor", func() { ) Expect(acc.GetProviderConfig()).To(Equal(&runtime.RawExtension{ - Raw: []byte(`{"metadata":{"creationTimestamp":null}}`), + Raw: []byte(`{"metadata":{}}`), })) }) @@ -115,7 +115,7 @@ var _ = Describe("Accessor", func() { ) Expect(acc.GetProviderStatus()).To(Equal(&runtime.RawExtension{ - Raw: []byte(`{"metadata":{"creationTimestamp":null}}`), + Raw: []byte(`{"metadata":{}}`), })) }) diff --git a/pkg/apis/seedmanagement/v1alpha1/zz_generated.defaults.go b/pkg/apis/seedmanagement/v1alpha1/zz_generated.defaults.go index d58cab329b4..021fdabe1ec 100644 --- a/pkg/apis/seedmanagement/v1alpha1/zz_generated.defaults.go +++ b/pkg/apis/seedmanagement/v1alpha1/zz_generated.defaults.go @@ -78,6 +78,17 @@ func SetObjectDefaults_Gardenlet(in *Gardenlet) { } } } + for i := range in.Spec.Deployment.GardenletDeployment.Env { + a := &in.Spec.Deployment.GardenletDeployment.Env[i] + if a.ValueFrom != nil { + if a.ValueFrom.FileKeyRef != nil { + if a.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + a.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } + } } func SetObjectDefaults_GardenletList(in *GardenletList) { @@ -139,6 +150,17 @@ func SetObjectDefaults_ManagedSeed(in *ManagedSeed) { } } } + for i := range in.Spec.Gardenlet.Deployment.Env { + a := &in.Spec.Gardenlet.Deployment.Env[i] + if a.ValueFrom != nil { + if a.ValueFrom.FileKeyRef != nil { + if a.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + a.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } + } } } @@ -201,6 +223,17 @@ func SetObjectDefaults_ManagedSeedSet(in *ManagedSeedSet) { } } } + for i := range in.Spec.Template.Spec.Gardenlet.Deployment.Env { + a := &in.Spec.Template.Spec.Gardenlet.Deployment.Env[i] + if a.ValueFrom != nil { + if a.ValueFrom.FileKeyRef != nil { + if a.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + a.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } + } } if in.Spec.UpdateStrategy != nil { SetDefaults_UpdateStrategy(in.Spec.UpdateStrategy) diff --git a/pkg/apiserver/openapi/openapi_generated.go b/pkg/apiserver/openapi/openapi_generated.go index 290f0a1f49f..a8e2442480a 100644 --- a/pkg/apiserver/openapi/openapi_generated.go +++ b/pkg/apiserver/openapi/openapi_generated.go @@ -329,9 +329,12 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/api/core/v1.ConfigMapProjection": schema_k8sio_api_core_v1_ConfigMapProjection(ref), "k8s.io/api/core/v1.ConfigMapVolumeSource": schema_k8sio_api_core_v1_ConfigMapVolumeSource(ref), "k8s.io/api/core/v1.Container": schema_k8sio_api_core_v1_Container(ref), + "k8s.io/api/core/v1.ContainerExtendedResourceRequest": schema_k8sio_api_core_v1_ContainerExtendedResourceRequest(ref), "k8s.io/api/core/v1.ContainerImage": schema_k8sio_api_core_v1_ContainerImage(ref), "k8s.io/api/core/v1.ContainerPort": schema_k8sio_api_core_v1_ContainerPort(ref), "k8s.io/api/core/v1.ContainerResizePolicy": schema_k8sio_api_core_v1_ContainerResizePolicy(ref), + "k8s.io/api/core/v1.ContainerRestartRule": schema_k8sio_api_core_v1_ContainerRestartRule(ref), + "k8s.io/api/core/v1.ContainerRestartRuleOnExitCodes": schema_k8sio_api_core_v1_ContainerRestartRuleOnExitCodes(ref), "k8s.io/api/core/v1.ContainerState": schema_k8sio_api_core_v1_ContainerState(ref), "k8s.io/api/core/v1.ContainerStateRunning": schema_k8sio_api_core_v1_ContainerStateRunning(ref), "k8s.io/api/core/v1.ContainerStateTerminated": schema_k8sio_api_core_v1_ContainerStateTerminated(ref), @@ -360,6 +363,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/api/core/v1.EventSource": schema_k8sio_api_core_v1_EventSource(ref), "k8s.io/api/core/v1.ExecAction": schema_k8sio_api_core_v1_ExecAction(ref), "k8s.io/api/core/v1.FCVolumeSource": schema_k8sio_api_core_v1_FCVolumeSource(ref), + "k8s.io/api/core/v1.FileKeySelector": schema_k8sio_api_core_v1_FileKeySelector(ref), "k8s.io/api/core/v1.FlexPersistentVolumeSource": schema_k8sio_api_core_v1_FlexPersistentVolumeSource(ref), "k8s.io/api/core/v1.FlexVolumeSource": schema_k8sio_api_core_v1_FlexVolumeSource(ref), "k8s.io/api/core/v1.FlockerVolumeSource": schema_k8sio_api_core_v1_FlockerVolumeSource(ref), @@ -435,10 +439,12 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/api/core/v1.PodAffinityTerm": schema_k8sio_api_core_v1_PodAffinityTerm(ref), "k8s.io/api/core/v1.PodAntiAffinity": schema_k8sio_api_core_v1_PodAntiAffinity(ref), "k8s.io/api/core/v1.PodAttachOptions": schema_k8sio_api_core_v1_PodAttachOptions(ref), + "k8s.io/api/core/v1.PodCertificateProjection": schema_k8sio_api_core_v1_PodCertificateProjection(ref), "k8s.io/api/core/v1.PodCondition": schema_k8sio_api_core_v1_PodCondition(ref), "k8s.io/api/core/v1.PodDNSConfig": schema_k8sio_api_core_v1_PodDNSConfig(ref), "k8s.io/api/core/v1.PodDNSConfigOption": schema_k8sio_api_core_v1_PodDNSConfigOption(ref), "k8s.io/api/core/v1.PodExecOptions": schema_k8sio_api_core_v1_PodExecOptions(ref), + "k8s.io/api/core/v1.PodExtendedResourceClaimStatus": schema_k8sio_api_core_v1_PodExtendedResourceClaimStatus(ref), "k8s.io/api/core/v1.PodIP": schema_k8sio_api_core_v1_PodIP(ref), "k8s.io/api/core/v1.PodList": schema_k8sio_api_core_v1_PodList(ref), "k8s.io/api/core/v1.PodLogOptions": schema_k8sio_api_core_v1_PodLogOptions(ref), @@ -13888,6 +13894,7 @@ func schema_k8sio_api_autoscaling_v1_ScaleSpec(ref common.ReferenceCallback) com "replicas": { SchemaProps: spec.SchemaProps{ Description: "replicas is the desired number of instances for the scaled object.", + Default: 0, Type: []string{"integer"}, Format: "int32", }, @@ -15410,7 +15417,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope }, }, SchemaProps: spec.SchemaProps{ - Description: "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", + Description: "List of sources to populate environment variables in the container. The keys defined within a source may consist of any printable ASCII characters except '='. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -15474,11 +15481,30 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope }, "restartPolicy": { SchemaProps: spec.SchemaProps{ - Description: "RestartPolicy defines the restart behavior of individual containers in a pod. This field may only be set for init containers, and the only allowed value is \"Always\". For non-init containers or when this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. Setting the RestartPolicy as \"Always\" for the init container will have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy \"Always\" will be shut down. This lifecycle differs from normal init containers and is often referred to as a \"sidecar\" container. Although this init container still starts in the init container sequence, it does not wait for the container to complete before proceeding to the next init container. Instead, the next init container starts immediately after this init container is started, or after any startupProbe has successfully completed.", + Description: "RestartPolicy defines the restart behavior of individual containers in a pod. This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. Additionally, setting the RestartPolicy as \"Always\" for the init container will have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy \"Always\" will be shut down. This lifecycle differs from normal init containers and is often referred to as a \"sidecar\" container. Although this init container still starts in the init container sequence, it does not wait for the container to complete before proceeding to the next init container. Instead, the next init container starts immediately after this init container is started, or after any startupProbe has successfully completed.", Type: []string{"string"}, Format: "", }, }, + "restartPolicyRules": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Represents a list of rules to be checked to determine if the container should be restarted on exit. The rules are evaluated in order. Once a rule matches a container exit condition, the remaining rules are ignored. If no rule matches the container exit condition, the Container-level restart policy determines the whether the container is restarted or not. Constraints on the rules: - At most 20 rules are allowed. - Rules can have the same action. - Identical rules are not forbidden in validations. When rules are specified, container MUST set RestartPolicy explicitly even it if matches the Pod's RestartPolicy.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.ContainerRestartRule"), + }, + }, + }, + }, + }, "volumeMounts": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -15606,7 +15632,45 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, + "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.ContainerRestartRule", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, + } +} + +func schema_k8sio_api_core_v1_ContainerExtendedResourceRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ContainerExtendedResourceRequest has the mapping of container name, extended resource name to the device request name.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "containerName": { + SchemaProps: spec.SchemaProps{ + Description: "The name of the container requesting resources.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "resourceName": { + SchemaProps: spec.SchemaProps{ + Description: "The name of the extended resource in that container which gets backed by DRA.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "requestName": { + SchemaProps: spec.SchemaProps{ + Description: "The name of the request in the special ResourceClaim which corresponds to the extended resource.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"containerName", "resourceName", "requestName"}, + }, + }, } } @@ -15732,6 +15796,76 @@ func schema_k8sio_api_core_v1_ContainerResizePolicy(ref common.ReferenceCallback } } +func schema_k8sio_api_core_v1_ContainerRestartRule(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ContainerRestartRule describes how a container exit is handled.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "action": { + SchemaProps: spec.SchemaProps{ + Description: "Specifies the action taken on a container exit if the requirements are satisfied. The only possible value is \"Restart\" to restart the container.", + Type: []string{"string"}, + Format: "", + }, + }, + "exitCodes": { + SchemaProps: spec.SchemaProps{ + Description: "Represents the exit codes to check on container exits.", + Ref: ref("k8s.io/api/core/v1.ContainerRestartRuleOnExitCodes"), + }, + }, + }, + Required: []string{"action"}, + }, + }, + Dependencies: []string{ + "k8s.io/api/core/v1.ContainerRestartRuleOnExitCodes"}, + } +} + +func schema_k8sio_api_core_v1_ContainerRestartRuleOnExitCodes(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ContainerRestartRuleOnExitCodes describes the condition for handling an exited container based on its exit codes.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "operator": { + SchemaProps: spec.SchemaProps{ + Description: "Represents the relationship between the container exit code(s) and the specified values. Possible values are: - In: the requirement is satisfied if the container exit code is in the\n set of specified values.\n- NotIn: the requirement is satisfied if the container exit code is\n not in the set of specified values.", + Type: []string{"string"}, + Format: "", + }, + }, + "values": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Specifies the set of values to check for container exit codes. At most 255 elements are allowed.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + }, + Required: []string{"operator"}, + }, + }, + } +} + func schema_k8sio_api_core_v1_ContainerState(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -16517,7 +16651,7 @@ func schema_k8sio_api_core_v1_EnvFromSource(ref common.ReferenceCallback) common Properties: map[string]spec.Schema{ "prefix": { SchemaProps: spec.SchemaProps{ - Description: "Optional text to prepend to the name of each environment variable. Must be a C_IDENTIFIER.", + Description: "Optional text to prepend to the name of each environment variable. May consist of any printable ASCII characters except '='.", Type: []string{"string"}, Format: "", }, @@ -16551,7 +16685,7 @@ func schema_k8sio_api_core_v1_EnvVar(ref common.ReferenceCallback) common.OpenAP Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the environment variable. Must be a C_IDENTIFIER.", + Description: "Name of the environment variable. May consist of any printable ASCII characters except '='.", Default: "", Type: []string{"string"}, Format: "", @@ -16610,11 +16744,17 @@ func schema_k8sio_api_core_v1_EnvVarSource(ref common.ReferenceCallback) common. Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), }, }, + "fileKeyRef": { + SchemaProps: spec.SchemaProps{ + Description: "FileKeyRef selects a key of the env file. Requires the EnvFiles feature gate to be enabled.", + Ref: ref("k8s.io/api/core/v1.FileKeySelector"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ConfigMapKeySelector", "k8s.io/api/core/v1.ObjectFieldSelector", "k8s.io/api/core/v1.ResourceFieldSelector", "k8s.io/api/core/v1.SecretKeySelector"}, + "k8s.io/api/core/v1.ConfigMapKeySelector", "k8s.io/api/core/v1.FileKeySelector", "k8s.io/api/core/v1.ObjectFieldSelector", "k8s.io/api/core/v1.ResourceFieldSelector", "k8s.io/api/core/v1.SecretKeySelector"}, } } @@ -16719,7 +16859,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c }, }, SchemaProps: spec.SchemaProps{ - Description: "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", + Description: "List of sources to populate environment variables in the container. The keys defined within a source may consist of any printable ASCII characters except '='. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -16783,11 +16923,30 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c }, "restartPolicy": { SchemaProps: spec.SchemaProps{ - Description: "Restart policy for the container to manage the restart behavior of each container within a pod. This may only be set for init containers. You cannot set this field on ephemeral containers.", + Description: "Restart policy for the container to manage the restart behavior of each container within a pod. You cannot set this field on ephemeral containers.", Type: []string{"string"}, Format: "", }, }, + "restartPolicyRules": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Represents a list of rules to be checked to determine if the container should be restarted on exit. You cannot set this field on ephemeral containers.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.ContainerRestartRule"), + }, + }, + }, + }, + }, "volumeMounts": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -16922,7 +17081,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, + "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.ContainerRestartRule", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, } } @@ -17027,7 +17186,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb }, }, SchemaProps: spec.SchemaProps{ - Description: "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", + Description: "List of sources to populate environment variables in the container. The keys defined within a source may consist of any printable ASCII characters except '='. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -17091,11 +17250,30 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb }, "restartPolicy": { SchemaProps: spec.SchemaProps{ - Description: "Restart policy for the container to manage the restart behavior of each container within a pod. This may only be set for init containers. You cannot set this field on ephemeral containers.", + Description: "Restart policy for the container to manage the restart behavior of each container within a pod. You cannot set this field on ephemeral containers.", Type: []string{"string"}, Format: "", }, }, + "restartPolicyRules": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Represents a list of rules to be checked to determine if the container should be restarted on exit. You cannot set this field on ephemeral containers.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.ContainerRestartRule"), + }, + }, + }, + }, + }, "volumeMounts": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -17223,7 +17401,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, + "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.ContainerRestartRule", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, } } @@ -17593,6 +17771,57 @@ func schema_k8sio_api_core_v1_FCVolumeSource(ref common.ReferenceCallback) commo } } +func schema_k8sio_api_core_v1_FileKeySelector(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "FileKeySelector selects a key of the env file.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "volumeName": { + SchemaProps: spec.SchemaProps{ + Description: "The name of the volume mount containing the env file.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "path": { + SchemaProps: spec.SchemaProps{ + Description: "The path within the volume from which to select the file. Must be relative and may not contain the '..' path or start with '..'.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "key": { + SchemaProps: spec.SchemaProps{ + Description: "The key within the env file. An invalid key will prevent the pod from starting. The keys defined within a source may consist of any printable ASCII characters except '='. During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "optional": { + SchemaProps: spec.SchemaProps{ + Description: "Specify whether the file or its key must be defined. If the file or key does not exist, then the env var is not published. If optional is set to true and the specified key does not exist, the environment variable will not be set in the Pod's containers.\n\nIf optional is set to false and the specified key does not exist, an error will be returned during Pod creation.", + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"volumeName", "path", "key"}, + }, + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-map-type": "atomic", + }, + }, + }, + } +} + func schema_k8sio_api_core_v1_FlexPersistentVolumeSource(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -17902,7 +18131,7 @@ func schema_k8sio_api_core_v1_GlusterfsVolumeSource(ref common.ReferenceCallback Properties: map[string]spec.Schema{ "endpoints": { SchemaProps: spec.SchemaProps{ - Description: "endpoints is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", + Description: "endpoints is the endpoint name that details Glusterfs topology.", Default: "", Type: []string{"string"}, Format: "", @@ -20612,7 +20841,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimSpec(ref common.ReferenceCall }, "volumeAttributesClassName": { SchemaProps: spec.SchemaProps{ - Description: "volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass will be applied to the claim but it's not allowed to reset this field to empty string once it is set. If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass will be set by the persistentvolume controller if it exists. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default).", + Description: "volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, it can be changed after the claim is created. An empty string or nil value indicates that no VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/", Type: []string{"string"}, Format: "", }, @@ -20737,14 +20966,14 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimStatus(ref common.ReferenceCa }, "currentVolumeAttributesClassName": { SchemaProps: spec.SchemaProps{ - Description: "currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim This is a beta field and requires enabling VolumeAttributesClass feature (off by default).", + Description: "currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim", Type: []string{"string"}, Format: "", }, }, "modifyVolumeStatus": { SchemaProps: spec.SchemaProps{ - Description: "ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. This is a beta field and requires enabling VolumeAttributesClass feature (off by default).", + Description: "ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted.", Ref: ref("k8s.io/api/core/v1.ModifyVolumeStatus"), }, }, @@ -21249,7 +21478,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeSpec(ref common.ReferenceCallback) }, "volumeAttributesClassName": { SchemaProps: spec.SchemaProps{ - Description: "Name of VolumeAttributesClass to which this persistent volume belongs. Empty value is not allowed. When this field is not set, it indicates that this volume does not belong to any VolumeAttributesClass. This field is mutable and can be changed by the CSI driver after a volume has been updated successfully to a new class. For an unbound PersistentVolume, the volumeAttributesClassName will be matched with unbound PersistentVolumeClaims during the binding process. This is a beta field and requires enabling VolumeAttributesClass feature (off by default).", + Description: "Name of VolumeAttributesClass to which this persistent volume belongs. Empty value is not allowed. When this field is not set, it indicates that this volume does not belong to any VolumeAttributesClass. This field is mutable and can be changed by the CSI driver after a volume has been updated successfully to a new class. For an unbound PersistentVolume, the volumeAttributesClassName will be matched with unbound PersistentVolumeClaims during the binding process.", Type: []string{"string"}, Format: "", }, @@ -21566,7 +21795,7 @@ func schema_k8sio_api_core_v1_PodAntiAffinity(ref common.ReferenceCallback) comm }, }, SchemaProps: spec.SchemaProps{ - Description: "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", + Description: "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and subtracting \"weight\" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -21648,6 +21877,62 @@ func schema_k8sio_api_core_v1_PodAttachOptions(ref common.ReferenceCallback) com } } +func schema_k8sio_api_core_v1_PodCertificateProjection(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodCertificateProjection provides a private key and X.509 certificate in the pod filesystem.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "signerName": { + SchemaProps: spec.SchemaProps{ + Description: "Kubelet's generated CSRs will be addressed to this signer.", + Type: []string{"string"}, + Format: "", + }, + }, + "keyType": { + SchemaProps: spec.SchemaProps{ + Description: "The type of keypair Kubelet will generate for the pod.\n\nValid values are \"RSA3072\", \"RSA4096\", \"ECDSAP256\", \"ECDSAP384\", \"ECDSAP521\", and \"ED25519\".", + Type: []string{"string"}, + Format: "", + }, + }, + "maxExpirationSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "maxExpirationSeconds is the maximum lifetime permitted for the certificate.\n\nKubelet copies this value verbatim into the PodCertificateRequests it generates for this projection.\n\nIf omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver will reject values shorter than 3600 (1 hour). The maximum allowable value is 7862400 (91 days).\n\nThe signer implementation is then free to issue a certificate with any lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 seconds (1 hour). This constraint is enforced by kube-apiserver. `kubernetes.io` signers will never issue certificates with a lifetime longer than 24 hours.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "credentialBundlePath": { + SchemaProps: spec.SchemaProps{ + Description: "Write the credential bundle at this path in the projected volume.\n\nThe credential bundle is a single file that contains multiple PEM blocks. The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private key.\n\nThe remaining blocks are CERTIFICATE blocks, containing the issued certificate chain from the signer (leaf and any intermediates).\n\nUsing credentialBundlePath lets your Pod's application code make a single atomic read that retrieves a consistent key and certificate chain. If you project them to separate files, your application code will need to additionally check that the leaf certificate was issued to the key.", + Type: []string{"string"}, + Format: "", + }, + }, + "keyPath": { + SchemaProps: spec.SchemaProps{ + Description: "Write the key at this path in the projected volume.\n\nMost applications should use credentialBundlePath. When using keyPath and certificateChainPath, your application needs to check that the key and leaf certificate are consistent, because it is possible to read the files mid-rotation.", + Type: []string{"string"}, + Format: "", + }, + }, + "certificateChainPath": { + SchemaProps: spec.SchemaProps{ + Description: "Write the certificate chain at this path in the projected volume.\n\nMost applications should use credentialBundlePath. When using keyPath and certificateChainPath, your application needs to check that the key and leaf certificate are consistent, because it is possible to read the files mid-rotation.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"signerName", "keyType"}, + }, + }, + } +} + func schema_k8sio_api_core_v1_PodCondition(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -21897,6 +22182,49 @@ func schema_k8sio_api_core_v1_PodExecOptions(ref common.ReferenceCallback) commo } } +func schema_k8sio_api_core_v1_PodExtendedResourceClaimStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodExtendedResourceClaimStatus is stored in the PodStatus for the extended resource requests backed by DRA. It stores the generated name for the corresponding special ResourceClaim created by the scheduler.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "requestMappings": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "RequestMappings identifies the mapping of to device request in the generated ResourceClaim.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.ContainerExtendedResourceRequest"), + }, + }, + }, + }, + }, + "resourceClaimName": { + SchemaProps: spec.SchemaProps{ + Description: "ResourceClaimName is the name of the ResourceClaim that was generated for the Pod in the namespace of the Pod.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"requestMappings", "resourceClaimName"}, + }, + }, + Dependencies: []string{ + "k8s.io/api/core/v1.ContainerExtendedResourceRequest"}, + } +} + func schema_k8sio_api_core_v1_PodIP(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -22614,7 +22942,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "hostNetwork": { SchemaProps: spec.SchemaProps{ - Description: "Host networking requested for this pod. Use the host's network namespace. If this option is set, the ports that will be used must be specified. Default to false.", + Description: "Host networking requested for this pod. Use the host's network namespace. When using HostNetwork you should specify ports so the scheduler is aware. When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. Default to false.", Type: []string{"boolean"}, Format: "", }, @@ -22849,7 +23177,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "os": { SchemaProps: spec.SchemaProps{ - Description: "Specifies the OS of the containers in the pod. Some pod and container fields are restricted if this is set.\n\nIf the OS field is set to linux, the following fields must be unset: -securityContext.windowsOptions\n\nIf the OS field is set to windows, following fields must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers - spec.securityContext.appArmorProfile - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy - spec.securityContext.sysctls - spec.shareProcessNamespace - spec.securityContext.runAsUser - spec.securityContext.runAsGroup - spec.securityContext.supplementalGroups - spec.securityContext.supplementalGroupsPolicy - spec.containers[*].securityContext.appArmorProfile - spec.containers[*].securityContext.seLinuxOptions - spec.containers[*].securityContext.seccompProfile - spec.containers[*].securityContext.capabilities - spec.containers[*].securityContext.readOnlyRootFilesystem - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - spec.containers[*].securityContext.runAsGroup", + Description: "Specifies the OS of the containers in the pod. Some pod and container fields are restricted if this is set.\n\nIf the OS field is set to linux, the following fields must be unset: -securityContext.windowsOptions\n\nIf the OS field is set to windows, following fields must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers - spec.resources - spec.securityContext.appArmorProfile - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy - spec.securityContext.sysctls - spec.shareProcessNamespace - spec.securityContext.runAsUser - spec.securityContext.runAsGroup - spec.securityContext.supplementalGroups - spec.securityContext.supplementalGroupsPolicy - spec.containers[*].securityContext.appArmorProfile - spec.containers[*].securityContext.seLinuxOptions - spec.containers[*].securityContext.seccompProfile - spec.containers[*].securityContext.capabilities - spec.containers[*].securityContext.readOnlyRootFilesystem - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - spec.containers[*].securityContext.runAsGroup", Ref: ref("k8s.io/api/core/v1.PodOS"), }, }, @@ -22910,10 +23238,17 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "resources": { SchemaProps: spec.SchemaProps{ - Description: "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\" and \"memory\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate.", + Description: "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate.", Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), }, }, + "hostnameOverride": { + SchemaProps: spec.SchemaProps{ + Description: "HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. This field only specifies the pod's hostname and does not affect its DNS records. When this field is set to a non-empty string: - It takes precedence over the values set in `hostname` and `subdomain`. - The Pod's hostname will be set to this value. - `setHostnameAsFQDN` must be nil or set to false. - `hostNetwork` must be set to false.\n\nThis field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. Requires the HostnameOverride feature gate to be enabled.", + Type: []string{"string"}, + Format: "", + }, + }, }, Required: []string{"containers"}, }, @@ -23151,11 +23486,17 @@ func schema_k8sio_api_core_v1_PodStatus(ref common.ReferenceCallback) common.Ope }, }, }, + "extendedResourceClaimStatus": { + SchemaProps: spec.SchemaProps{ + Description: "Status of extended resource claim backed by DRA.", + Ref: ref("k8s.io/api/core/v1.PodExtendedResourceClaimStatus"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ContainerStatus", "k8s.io/api/core/v1.HostIP", "k8s.io/api/core/v1.PodCondition", "k8s.io/api/core/v1.PodIP", "k8s.io/api/core/v1.PodResourceClaimStatus", "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, + "k8s.io/api/core/v1.ContainerStatus", "k8s.io/api/core/v1.HostIP", "k8s.io/api/core/v1.PodCondition", "k8s.io/api/core/v1.PodExtendedResourceClaimStatus", "k8s.io/api/core/v1.PodIP", "k8s.io/api/core/v1.PodResourceClaimStatus", "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, } } @@ -24552,7 +24893,7 @@ func schema_k8sio_api_core_v1_ResourceRequirements(ref common.ReferenceCallback) }, }, SchemaProps: spec.SchemaProps{ - Description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container.\n\nThis is an alpha field and requires enabling the DynamicResourceAllocation feature gate.\n\nThis field is immutable. It can only be set for containers.", + Description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container.\n\nThis field depends on the DynamicResourceAllocation feature gate.\n\nThis field is immutable. It can only be set for containers.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -26330,7 +26671,7 @@ func schema_k8sio_api_core_v1_Taint(ref common.ReferenceCallback) common.OpenAPI }, "timeAdded": { SchemaProps: spec.SchemaProps{ - Description: "TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints.", + Description: "TimeAdded represents the time at which the taint was added.", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), }, }, @@ -26709,13 +27050,13 @@ func schema_k8sio_api_core_v1_Volume(ref common.ReferenceCallback) common.OpenAP }, "iscsi": { SchemaProps: spec.SchemaProps{ - Description: "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md", + Description: "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi", Ref: ref("k8s.io/api/core/v1.ISCSIVolumeSource"), }, }, "glusterfs": { SchemaProps: spec.SchemaProps{ - Description: "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. More info: https://examples.k8s.io/volumes/glusterfs/README.md", + Description: "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported.", Ref: ref("k8s.io/api/core/v1.GlusterfsVolumeSource"), }, }, @@ -26727,7 +27068,7 @@ func schema_k8sio_api_core_v1_Volume(ref common.ReferenceCallback) common.OpenAP }, "rbd": { SchemaProps: spec.SchemaProps{ - Description: "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. More info: https://examples.k8s.io/volumes/rbd/README.md", + Description: "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported.", Ref: ref("k8s.io/api/core/v1.RBDVolumeSource"), }, }, @@ -27052,11 +27393,17 @@ func schema_k8sio_api_core_v1_VolumeProjection(ref common.ReferenceCallback) com Ref: ref("k8s.io/api/core/v1.ClusterTrustBundleProjection"), }, }, + "podCertificate": { + SchemaProps: spec.SchemaProps{ + Description: "Projects an auto-rotating credential bundle (private key and certificate chain) that the pod can use either as a TLS client or server.\n\nKubelet generates a private key and uses it to send a PodCertificateRequest to the named signer. Once the signer approves the request and issues a certificate chain, Kubelet writes the key and certificate chain to the pod filesystem. The pod does not start until certificates have been issued for each podCertificate projected volume source in its spec.\n\nKubelet will begin trying to rotate the certificate at the time indicated by the signer using the PodCertificateRequest.Status.BeginRefreshAt timestamp.\n\nKubelet can write a single file, indicated by the credentialBundlePath field, or separate files, indicated by the keyPath and certificateChainPath fields.\n\nThe credential bundle is a single file in PEM format. The first PEM entry is the private key (in PKCS#8 format), and the remaining PEM entries are the certificate chain issued by the signer (typically, signers will return their certificate chain in leaf-to-root order).\n\nPrefer using the credential bundle format, since your application code can read it atomically. If you use keyPath and certificateChainPath, your application must make two separate file reads. If these coincide with a certificate rotation, it is possible that the private key and leaf certificate you read may not correspond to each other. Your application will need to check for this condition, and re-read until they are consistent.\n\nThe named signer controls chooses the format of the certificate it issues; consult the signer implementation's documentation to learn how to use the certificates it issues.", + Ref: ref("k8s.io/api/core/v1.PodCertificateProjection"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ClusterTrustBundleProjection", "k8s.io/api/core/v1.ConfigMapProjection", "k8s.io/api/core/v1.DownwardAPIProjection", "k8s.io/api/core/v1.SecretProjection", "k8s.io/api/core/v1.ServiceAccountTokenProjection"}, + "k8s.io/api/core/v1.ClusterTrustBundleProjection", "k8s.io/api/core/v1.ConfigMapProjection", "k8s.io/api/core/v1.DownwardAPIProjection", "k8s.io/api/core/v1.PodCertificateProjection", "k8s.io/api/core/v1.SecretProjection", "k8s.io/api/core/v1.ServiceAccountTokenProjection"}, } } @@ -27154,13 +27501,13 @@ func schema_k8sio_api_core_v1_VolumeSource(ref common.ReferenceCallback) common. }, "iscsi": { SchemaProps: spec.SchemaProps{ - Description: "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md", + Description: "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi", Ref: ref("k8s.io/api/core/v1.ISCSIVolumeSource"), }, }, "glusterfs": { SchemaProps: spec.SchemaProps{ - Description: "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. More info: https://examples.k8s.io/volumes/glusterfs/README.md", + Description: "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported.", Ref: ref("k8s.io/api/core/v1.GlusterfsVolumeSource"), }, }, @@ -27172,7 +27519,7 @@ func schema_k8sio_api_core_v1_VolumeSource(ref common.ReferenceCallback) common. }, "rbd": { SchemaProps: spec.SchemaProps{ - Description: "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. More info: https://examples.k8s.io/volumes/rbd/README.md", + Description: "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported.", Ref: ref("k8s.io/api/core/v1.RBDVolumeSource"), }, }, @@ -28549,7 +28896,7 @@ func schema_k8sio_api_networking_v1_NetworkPolicySpec(ref common.ReferenceCallba Properties: map[string]spec.Schema{ "podSelector": { SchemaProps: spec.SchemaProps{ - Description: "podSelector selects the pods to which this NetworkPolicy object applies. The array of ingress rules is applied to any pods selected by this field. Multiple network policies can select the same set of pods. In this case, the ingress rules for each are combined additively. This field is NOT optional and follows standard label selector semantics. An empty podSelector matches all pods in this namespace.", + Description: "podSelector selects the pods to which this NetworkPolicy object applies. The array of rules is applied to any pods selected by this field. An empty selector matches all pods in the policy's namespace. Multiple network policies can select the same set of pods. In this case, the ingress rules for each are combined additively. This field is optional. If it is not specified, it defaults to an empty selector.", Default: map[string]interface{}{}, Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"), }, @@ -28614,7 +28961,6 @@ func schema_k8sio_api_networking_v1_NetworkPolicySpec(ref common.ReferenceCallba }, }, }, - Required: []string{"podSelector"}, }, }, Dependencies: []string{ diff --git a/pkg/client/kubernetes/applier_test.go b/pkg/client/kubernetes/applier_test.go index 158065c2ac0..99ee2863204 100644 --- a/pkg/client/kubernetes/applier_test.go +++ b/pkg/client/kubernetes/applier_test.go @@ -204,6 +204,7 @@ spec: newService.Spec.ClusterIP = "" newService.Annotations = map[string]string{} expected = oldService.DeepCopy() + expected.TypeMeta = metav1.TypeMeta{} expected.ResourceVersion = "2" }) @@ -627,6 +628,7 @@ spec: newDeployment = oldDeployment.DeepCopy() expected = oldDeployment.DeepCopy() + expected.TypeMeta = metav1.TypeMeta{} expected.ResourceVersion = "2" }) diff --git a/pkg/client/kubernetes/chartapplier_test.go b/pkg/client/kubernetes/chartapplier_test.go index bbcf5691c33..7cbcd4dc8dd 100644 --- a/pkg/client/kubernetes/chartapplier_test.go +++ b/pkg/client/kubernetes/chartapplier_test.go @@ -69,7 +69,6 @@ var _ = Describe("chart applier", func() { mapper.Add(corev1.SchemeGroupVersion.WithKind("ConfigMap"), meta.RESTScopeNamespace) expectedCM = &corev1.ConfigMap{ - TypeMeta: configMapTypeMeta, ObjectMeta: metav1.ObjectMeta{ Name: configMapName, Namespace: namespace, @@ -105,7 +104,6 @@ var _ = Describe("chart applier", func() { const newNS = "other-namespace" existingCM := &corev1.ConfigMap{ - TypeMeta: configMapTypeMeta, ObjectMeta: metav1.ObjectMeta{ Name: configMapName, Namespace: newNS, @@ -164,7 +162,6 @@ var _ = Describe("chart applier", func() { test := func(chartPath string) { It("deletes the chart with default values", func() { existingCM := &corev1.ConfigMap{ - TypeMeta: configMapTypeMeta, ObjectMeta: metav1.ObjectMeta{ Name: configMapName, Namespace: namespace, @@ -181,7 +178,6 @@ var _ = Describe("chart applier", func() { const newNS = "other-namespace" existingCM := &corev1.ConfigMap{ - TypeMeta: configMapTypeMeta, ObjectMeta: metav1.ObjectMeta{ Name: configMapName, Namespace: newNS, @@ -267,7 +263,6 @@ var _ = Describe("chart applier", func() { const newNS = "other-namespace" existingCM := &corev1.ConfigMap{ - TypeMeta: configMapTypeMeta, ObjectMeta: metav1.ObjectMeta{ Name: configMapName, Namespace: newNS, @@ -325,7 +320,6 @@ var _ = Describe("chart applier", func() { test := func(archive []byte) { It("deletes the chart with default values", func() { existingCM := &corev1.ConfigMap{ - TypeMeta: configMapTypeMeta, ObjectMeta: metav1.ObjectMeta{ Name: configMapName, Namespace: namespace, @@ -342,7 +336,6 @@ var _ = Describe("chart applier", func() { const newNS = "other-namespace" existingCM := &corev1.ConfigMap{ - TypeMeta: configMapTypeMeta, ObjectMeta: metav1.ObjectMeta{ Name: configMapName, Namespace: newNS, diff --git a/pkg/client/seedmanagement/applyconfiguration/internal/internal.go b/pkg/client/seedmanagement/applyconfiguration/internal/internal.go deleted file mode 100644 index 6b06d7c5835..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/internal/internal.go +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package internal - -import ( - "fmt" - "sync" - - typed "sigs.k8s.io/structured-merge-diff/v4/typed" -) - -func Parser() *typed.Parser { - parserOnce.Do(func() { - var err error - parser, err = typed.NewParser(schemaYAML) - if err != nil { - panic(fmt.Sprintf("Failed to parse schema: %v", err)) - } - }) - return parser -} - -var parserOnce sync.Once -var parser *typed.Parser -var schemaYAML = typed.YAMLObject(`types: -- name: __untyped_atomic_ - scalar: untyped - list: - elementType: - namedType: __untyped_atomic_ - elementRelationship: atomic - map: - elementType: - namedType: __untyped_atomic_ - elementRelationship: atomic -- name: __untyped_deduced_ - scalar: untyped - list: - elementType: - namedType: __untyped_atomic_ - elementRelationship: atomic - map: - elementType: - namedType: __untyped_deduced_ - elementRelationship: separable -`) diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/gardenlet.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/gardenlet.go deleted file mode 100644 index 47c1e2261e5..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/gardenlet.go +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - seedmanagementv1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1" - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// GardenletApplyConfiguration represents an declarative configuration of the Gardenlet type for use -// with apply. -type GardenletApplyConfiguration struct { - Deployment *GardenletDeploymentApplyConfiguration `json:"deployment,omitempty"` - Config *runtime.RawExtension `json:"config,omitempty"` - Bootstrap *seedmanagementv1alpha1.Bootstrap `json:"bootstrap,omitempty"` - MergeWithParent *bool `json:"mergeWithParent,omitempty"` -} - -// GardenletApplyConfiguration constructs an declarative configuration of the Gardenlet type for use with -// apply. -func Gardenlet() *GardenletApplyConfiguration { - return &GardenletApplyConfiguration{} -} - -// WithDeployment sets the Deployment field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Deployment field is set to the value of the last call. -func (b *GardenletApplyConfiguration) WithDeployment(value *GardenletDeploymentApplyConfiguration) *GardenletApplyConfiguration { - b.Deployment = value - return b -} - -// WithConfig sets the Config field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Config field is set to the value of the last call. -func (b *GardenletApplyConfiguration) WithConfig(value runtime.RawExtension) *GardenletApplyConfiguration { - b.Config = &value - return b -} - -// WithBootstrap sets the Bootstrap field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Bootstrap field is set to the value of the last call. -func (b *GardenletApplyConfiguration) WithBootstrap(value seedmanagementv1alpha1.Bootstrap) *GardenletApplyConfiguration { - b.Bootstrap = &value - return b -} - -// WithMergeWithParent sets the MergeWithParent field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the MergeWithParent field is set to the value of the last call. -func (b *GardenletApplyConfiguration) WithMergeWithParent(value bool) *GardenletApplyConfiguration { - b.MergeWithParent = &value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/gardenletdeployment.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/gardenletdeployment.go deleted file mode 100644 index a7fe56c0e22..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/gardenletdeployment.go +++ /dev/null @@ -1,137 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "maps" - - v1 "k8s.io/api/core/v1" -) - -// GardenletDeploymentApplyConfiguration represents an declarative configuration of the GardenletDeployment type for use -// with apply. -type GardenletDeploymentApplyConfiguration struct { - ReplicaCount *int32 `json:"replicaCount,omitempty"` - RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` - ServiceAccountName *string `json:"serviceAccountName,omitempty"` - Image *ImageApplyConfiguration `json:"image,omitempty"` - Resources *v1.ResourceRequirements `json:"resources,omitempty"` - PodLabels map[string]string `json:"podLabels,omitempty"` - PodAnnotations map[string]string `json:"podAnnotations,omitempty"` - AdditionalVolumes []v1.Volume `json:"additionalVolumes,omitempty"` - AdditionalVolumeMounts []v1.VolumeMount `json:"additionalVolumeMounts,omitempty"` - Env []v1.EnvVar `json:"env,omitempty"` - VPA *bool `json:"vpa,omitempty"` -} - -// GardenletDeploymentApplyConfiguration constructs an declarative configuration of the GardenletDeployment type for use with -// apply. -func GardenletDeployment() *GardenletDeploymentApplyConfiguration { - return &GardenletDeploymentApplyConfiguration{} -} - -// WithReplicaCount sets the ReplicaCount field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ReplicaCount field is set to the value of the last call. -func (b *GardenletDeploymentApplyConfiguration) WithReplicaCount(value int32) *GardenletDeploymentApplyConfiguration { - b.ReplicaCount = &value - return b -} - -// WithRevisionHistoryLimit sets the RevisionHistoryLimit field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the RevisionHistoryLimit field is set to the value of the last call. -func (b *GardenletDeploymentApplyConfiguration) WithRevisionHistoryLimit(value int32) *GardenletDeploymentApplyConfiguration { - b.RevisionHistoryLimit = &value - return b -} - -// WithServiceAccountName sets the ServiceAccountName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ServiceAccountName field is set to the value of the last call. -func (b *GardenletDeploymentApplyConfiguration) WithServiceAccountName(value string) *GardenletDeploymentApplyConfiguration { - b.ServiceAccountName = &value - return b -} - -// WithImage sets the Image field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Image field is set to the value of the last call. -func (b *GardenletDeploymentApplyConfiguration) WithImage(value *ImageApplyConfiguration) *GardenletDeploymentApplyConfiguration { - b.Image = value - return b -} - -// WithResources sets the Resources field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Resources field is set to the value of the last call. -func (b *GardenletDeploymentApplyConfiguration) WithResources(value v1.ResourceRequirements) *GardenletDeploymentApplyConfiguration { - b.Resources = &value - return b -} - -// WithPodLabels puts the entries into the PodLabels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the PodLabels field, -// overwriting an existing map entries in PodLabels field with the same key. -func (b *GardenletDeploymentApplyConfiguration) WithPodLabels(entries map[string]string) *GardenletDeploymentApplyConfiguration { - if b.PodLabels == nil && len(entries) > 0 { - b.PodLabels = make(map[string]string, len(entries)) - } - maps.Copy(b.PodLabels, entries) - return b -} - -// WithPodAnnotations puts the entries into the PodAnnotations field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the PodAnnotations field, -// overwriting an existing map entries in PodAnnotations field with the same key. -func (b *GardenletDeploymentApplyConfiguration) WithPodAnnotations(entries map[string]string) *GardenletDeploymentApplyConfiguration { - if b.PodAnnotations == nil && len(entries) > 0 { - b.PodAnnotations = make(map[string]string, len(entries)) - } - maps.Copy(b.PodAnnotations, entries) - return b -} - -// WithAdditionalVolumes adds the given value to the AdditionalVolumes field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the AdditionalVolumes field. -func (b *GardenletDeploymentApplyConfiguration) WithAdditionalVolumes(values ...v1.Volume) *GardenletDeploymentApplyConfiguration { - for i := range values { - b.AdditionalVolumes = append(b.AdditionalVolumes, values[i]) - } - return b -} - -// WithAdditionalVolumeMounts adds the given value to the AdditionalVolumeMounts field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the AdditionalVolumeMounts field. -func (b *GardenletDeploymentApplyConfiguration) WithAdditionalVolumeMounts(values ...v1.VolumeMount) *GardenletDeploymentApplyConfiguration { - for i := range values { - b.AdditionalVolumeMounts = append(b.AdditionalVolumeMounts, values[i]) - } - return b -} - -// WithEnv adds the given value to the Env field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Env field. -func (b *GardenletDeploymentApplyConfiguration) WithEnv(values ...v1.EnvVar) *GardenletDeploymentApplyConfiguration { - for i := range values { - b.Env = append(b.Env, values[i]) - } - return b -} - -// WithVPA sets the VPA field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the VPA field is set to the value of the last call. -func (b *GardenletDeploymentApplyConfiguration) WithVPA(value bool) *GardenletDeploymentApplyConfiguration { - b.VPA = &value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/image.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/image.go deleted file mode 100644 index 8117ca1106b..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/image.go +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1 "k8s.io/api/core/v1" -) - -// ImageApplyConfiguration represents an declarative configuration of the Image type for use -// with apply. -type ImageApplyConfiguration struct { - Repository *string `json:"repository,omitempty"` - Tag *string `json:"tag,omitempty"` - PullPolicy *v1.PullPolicy `json:"pullPolicy,omitempty"` -} - -// ImageApplyConfiguration constructs an declarative configuration of the Image type for use with -// apply. -func Image() *ImageApplyConfiguration { - return &ImageApplyConfiguration{} -} - -// WithRepository sets the Repository field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Repository field is set to the value of the last call. -func (b *ImageApplyConfiguration) WithRepository(value string) *ImageApplyConfiguration { - b.Repository = &value - return b -} - -// WithTag sets the Tag field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Tag field is set to the value of the last call. -func (b *ImageApplyConfiguration) WithTag(value string) *ImageApplyConfiguration { - b.Tag = &value - return b -} - -// WithPullPolicy sets the PullPolicy field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the PullPolicy field is set to the value of the last call. -func (b *ImageApplyConfiguration) WithPullPolicy(value v1.PullPolicy) *ImageApplyConfiguration { - b.PullPolicy = &value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseed.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseed.go deleted file mode 100644 index e5716ff3e8f..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseed.go +++ /dev/null @@ -1,206 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "maps" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" -) - -// ManagedSeedApplyConfiguration represents an declarative configuration of the ManagedSeed type for use -// with apply. -type ManagedSeedApplyConfiguration struct { - v1.TypeMetaApplyConfiguration `json:",inline"` - *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` - Spec *ManagedSeedSpecApplyConfiguration `json:"spec,omitempty"` - Status *ManagedSeedStatusApplyConfiguration `json:"status,omitempty"` -} - -// ManagedSeed constructs an declarative configuration of the ManagedSeed type for use with -// apply. -func ManagedSeed(name, namespace string) *ManagedSeedApplyConfiguration { - b := &ManagedSeedApplyConfiguration{} - b.WithName(name) - b.WithNamespace(namespace) - b.WithKind("ManagedSeed") - b.WithAPIVersion("seedmanagement.gardener.cloud/v1alpha1") - return b -} - -// WithKind sets the Kind field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Kind field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithKind(value string) *ManagedSeedApplyConfiguration { - b.Kind = &value - return b -} - -// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the APIVersion field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithAPIVersion(value string) *ManagedSeedApplyConfiguration { - b.APIVersion = &value - return b -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithName(value string) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Name = &value - return b -} - -// WithGenerateName sets the GenerateName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the GenerateName field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithGenerateName(value string) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.GenerateName = &value - return b -} - -// WithNamespace sets the Namespace field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Namespace field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithNamespace(value string) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Namespace = &value - return b -} - -// WithUID sets the UID field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UID field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithUID(value types.UID) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.UID = &value - return b -} - -// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ResourceVersion field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithResourceVersion(value string) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.ResourceVersion = &value - return b -} - -// WithGeneration sets the Generation field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Generation field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithGeneration(value int64) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Generation = &value - return b -} - -// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CreationTimestamp field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithCreationTimestamp(value metav1.Time) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.CreationTimestamp = &value - return b -} - -// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionTimestamp field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionTimestamp = &value - return b -} - -// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionGracePeriodSeconds = &value - return b -} - -// WithLabels puts the entries into the Labels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Labels field, -// overwriting an existing map entries in Labels field with the same key. -func (b *ManagedSeedApplyConfiguration) WithLabels(entries map[string]string) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Labels == nil && len(entries) > 0 { - b.Labels = make(map[string]string, len(entries)) - } - - maps.Copy(b.Labels, entries) - return b -} - -// WithAnnotations puts the entries into the Annotations field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Annotations field, -// overwriting an existing map entries in Annotations field with the same key. -func (b *ManagedSeedApplyConfiguration) WithAnnotations(entries map[string]string) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Annotations == nil && len(entries) > 0 { - b.Annotations = make(map[string]string, len(entries)) - } - maps.Copy(b.Annotations, entries) - return b -} - -// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the OwnerReferences field. -func (b *ManagedSeedApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - if values[i] == nil { - panic("nil value passed to WithOwnerReferences") - } - b.OwnerReferences = append(b.OwnerReferences, *values[i]) - } - return b -} - -// WithFinalizers adds the given value to the Finalizers field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Finalizers field. -func (b *ManagedSeedApplyConfiguration) WithFinalizers(values ...string) *ManagedSeedApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - b.Finalizers = append(b.Finalizers, values[i]) - } - return b -} - -func (b *ManagedSeedApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { - if b.ObjectMetaApplyConfiguration == nil { - b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} - } -} - -// WithSpec sets the Spec field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Spec field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithSpec(value *ManagedSeedSpecApplyConfiguration) *ManagedSeedApplyConfiguration { - b.Spec = value - return b -} - -// WithStatus sets the Status field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Status field is set to the value of the last call. -func (b *ManagedSeedApplyConfiguration) WithStatus(value *ManagedSeedStatusApplyConfiguration) *ManagedSeedApplyConfiguration { - b.Status = value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedset.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedset.go deleted file mode 100644 index 2525ac6e89d..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedset.go +++ /dev/null @@ -1,205 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "maps" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" -) - -// ManagedSeedSetApplyConfiguration represents an declarative configuration of the ManagedSeedSet type for use -// with apply. -type ManagedSeedSetApplyConfiguration struct { - v1.TypeMetaApplyConfiguration `json:",inline"` - *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` - Spec *ManagedSeedSetSpecApplyConfiguration `json:"spec,omitempty"` - Status *ManagedSeedSetStatusApplyConfiguration `json:"status,omitempty"` -} - -// ManagedSeedSet constructs an declarative configuration of the ManagedSeedSet type for use with -// apply. -func ManagedSeedSet(name, namespace string) *ManagedSeedSetApplyConfiguration { - b := &ManagedSeedSetApplyConfiguration{} - b.WithName(name) - b.WithNamespace(namespace) - b.WithKind("ManagedSeedSet") - b.WithAPIVersion("seedmanagement.gardener.cloud/v1alpha1") - return b -} - -// WithKind sets the Kind field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Kind field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithKind(value string) *ManagedSeedSetApplyConfiguration { - b.Kind = &value - return b -} - -// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the APIVersion field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithAPIVersion(value string) *ManagedSeedSetApplyConfiguration { - b.APIVersion = &value - return b -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithName(value string) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Name = &value - return b -} - -// WithGenerateName sets the GenerateName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the GenerateName field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithGenerateName(value string) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.GenerateName = &value - return b -} - -// WithNamespace sets the Namespace field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Namespace field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithNamespace(value string) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Namespace = &value - return b -} - -// WithUID sets the UID field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UID field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithUID(value types.UID) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.UID = &value - return b -} - -// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ResourceVersion field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithResourceVersion(value string) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.ResourceVersion = &value - return b -} - -// WithGeneration sets the Generation field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Generation field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithGeneration(value int64) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Generation = &value - return b -} - -// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CreationTimestamp field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithCreationTimestamp(value metav1.Time) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.CreationTimestamp = &value - return b -} - -// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionTimestamp field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionTimestamp = &value - return b -} - -// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionGracePeriodSeconds = &value - return b -} - -// WithLabels puts the entries into the Labels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Labels field, -// overwriting an existing map entries in Labels field with the same key. -func (b *ManagedSeedSetApplyConfiguration) WithLabels(entries map[string]string) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Labels == nil && len(entries) > 0 { - b.Labels = make(map[string]string, len(entries)) - } - maps.Copy(b.Labels, entries) - return b -} - -// WithAnnotations puts the entries into the Annotations field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Annotations field, -// overwriting an existing map entries in Annotations field with the same key. -func (b *ManagedSeedSetApplyConfiguration) WithAnnotations(entries map[string]string) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Annotations == nil && len(entries) > 0 { - b.Annotations = make(map[string]string, len(entries)) - } - maps.Copy(b.Annotations, entries) - return b -} - -// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the OwnerReferences field. -func (b *ManagedSeedSetApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - if values[i] == nil { - panic("nil value passed to WithOwnerReferences") - } - b.OwnerReferences = append(b.OwnerReferences, *values[i]) - } - return b -} - -// WithFinalizers adds the given value to the Finalizers field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Finalizers field. -func (b *ManagedSeedSetApplyConfiguration) WithFinalizers(values ...string) *ManagedSeedSetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - b.Finalizers = append(b.Finalizers, values[i]) - } - return b -} - -func (b *ManagedSeedSetApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { - if b.ObjectMetaApplyConfiguration == nil { - b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} - } -} - -// WithSpec sets the Spec field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Spec field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithSpec(value *ManagedSeedSetSpecApplyConfiguration) *ManagedSeedSetApplyConfiguration { - b.Spec = value - return b -} - -// WithStatus sets the Status field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Status field is set to the value of the last call. -func (b *ManagedSeedSetApplyConfiguration) WithStatus(value *ManagedSeedSetStatusApplyConfiguration) *ManagedSeedSetApplyConfiguration { - b.Status = value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedsetspec.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedsetspec.go deleted file mode 100644 index 57b21d0efcb..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedsetspec.go +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// ManagedSeedSetSpecApplyConfiguration represents an declarative configuration of the ManagedSeedSetSpec type for use -// with apply. -type ManagedSeedSetSpecApplyConfiguration struct { - Replicas *int32 `json:"replicas,omitempty"` - Selector *v1.LabelSelector `json:"selector,omitempty"` - Template *ManagedSeedTemplateApplyConfiguration `json:"template,omitempty"` - ShootTemplate *v1beta1.ShootTemplate `json:"shootTemplate,omitempty"` - UpdateStrategy *UpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` - RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` -} - -// ManagedSeedSetSpecApplyConfiguration constructs an declarative configuration of the ManagedSeedSetSpec type for use with -// apply. -func ManagedSeedSetSpec() *ManagedSeedSetSpecApplyConfiguration { - return &ManagedSeedSetSpecApplyConfiguration{} -} - -// WithReplicas sets the Replicas field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Replicas field is set to the value of the last call. -func (b *ManagedSeedSetSpecApplyConfiguration) WithReplicas(value int32) *ManagedSeedSetSpecApplyConfiguration { - b.Replicas = &value - return b -} - -// WithSelector sets the Selector field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Selector field is set to the value of the last call. -func (b *ManagedSeedSetSpecApplyConfiguration) WithSelector(value v1.LabelSelector) *ManagedSeedSetSpecApplyConfiguration { - b.Selector = &value - return b -} - -// WithTemplate sets the Template field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Template field is set to the value of the last call. -func (b *ManagedSeedSetSpecApplyConfiguration) WithTemplate(value *ManagedSeedTemplateApplyConfiguration) *ManagedSeedSetSpecApplyConfiguration { - b.Template = value - return b -} - -// WithShootTemplate sets the ShootTemplate field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ShootTemplate field is set to the value of the last call. -func (b *ManagedSeedSetSpecApplyConfiguration) WithShootTemplate(value v1beta1.ShootTemplate) *ManagedSeedSetSpecApplyConfiguration { - b.ShootTemplate = &value - return b -} - -// WithUpdateStrategy sets the UpdateStrategy field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UpdateStrategy field is set to the value of the last call. -func (b *ManagedSeedSetSpecApplyConfiguration) WithUpdateStrategy(value *UpdateStrategyApplyConfiguration) *ManagedSeedSetSpecApplyConfiguration { - b.UpdateStrategy = value - return b -} - -// WithRevisionHistoryLimit sets the RevisionHistoryLimit field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the RevisionHistoryLimit field is set to the value of the last call. -func (b *ManagedSeedSetSpecApplyConfiguration) WithRevisionHistoryLimit(value int32) *ManagedSeedSetSpecApplyConfiguration { - b.RevisionHistoryLimit = &value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedsetstatus.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedsetstatus.go deleted file mode 100644 index d57e1731125..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedsetstatus.go +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" -) - -// ManagedSeedSetStatusApplyConfiguration represents an declarative configuration of the ManagedSeedSetStatus type for use -// with apply. -type ManagedSeedSetStatusApplyConfiguration struct { - ObservedGeneration *int64 `json:"observedGeneration,omitempty"` - Replicas *int32 `json:"replicas,omitempty"` - ReadyReplicas *int32 `json:"readyReplicas,omitempty"` - NextReplicaNumber *int32 `json:"nextReplicaNumber,omitempty"` - CurrentReplicas *int32 `json:"currentReplicas,omitempty"` - UpdatedReplicas *int32 `json:"updatedReplicas,omitempty"` - CurrentRevision *string `json:"currentRevision,omitempty"` - UpdateRevision *string `json:"updateRevision,omitempty"` - CollisionCount *int32 `json:"collisionCount,omitempty"` - Conditions []v1beta1.Condition `json:"conditions,omitempty"` - PendingReplica *PendingReplicaApplyConfiguration `json:"pendingReplica,omitempty"` -} - -// ManagedSeedSetStatusApplyConfiguration constructs an declarative configuration of the ManagedSeedSetStatus type for use with -// apply. -func ManagedSeedSetStatus() *ManagedSeedSetStatusApplyConfiguration { - return &ManagedSeedSetStatusApplyConfiguration{} -} - -// WithObservedGeneration sets the ObservedGeneration field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ObservedGeneration field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithObservedGeneration(value int64) *ManagedSeedSetStatusApplyConfiguration { - b.ObservedGeneration = &value - return b -} - -// WithReplicas sets the Replicas field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Replicas field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithReplicas(value int32) *ManagedSeedSetStatusApplyConfiguration { - b.Replicas = &value - return b -} - -// WithReadyReplicas sets the ReadyReplicas field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ReadyReplicas field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithReadyReplicas(value int32) *ManagedSeedSetStatusApplyConfiguration { - b.ReadyReplicas = &value - return b -} - -// WithNextReplicaNumber sets the NextReplicaNumber field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the NextReplicaNumber field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithNextReplicaNumber(value int32) *ManagedSeedSetStatusApplyConfiguration { - b.NextReplicaNumber = &value - return b -} - -// WithCurrentReplicas sets the CurrentReplicas field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CurrentReplicas field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithCurrentReplicas(value int32) *ManagedSeedSetStatusApplyConfiguration { - b.CurrentReplicas = &value - return b -} - -// WithUpdatedReplicas sets the UpdatedReplicas field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UpdatedReplicas field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithUpdatedReplicas(value int32) *ManagedSeedSetStatusApplyConfiguration { - b.UpdatedReplicas = &value - return b -} - -// WithCurrentRevision sets the CurrentRevision field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CurrentRevision field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithCurrentRevision(value string) *ManagedSeedSetStatusApplyConfiguration { - b.CurrentRevision = &value - return b -} - -// WithUpdateRevision sets the UpdateRevision field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UpdateRevision field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithUpdateRevision(value string) *ManagedSeedSetStatusApplyConfiguration { - b.UpdateRevision = &value - return b -} - -// WithCollisionCount sets the CollisionCount field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CollisionCount field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithCollisionCount(value int32) *ManagedSeedSetStatusApplyConfiguration { - b.CollisionCount = &value - return b -} - -// WithConditions adds the given value to the Conditions field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Conditions field. -func (b *ManagedSeedSetStatusApplyConfiguration) WithConditions(values ...v1beta1.Condition) *ManagedSeedSetStatusApplyConfiguration { - for i := range values { - b.Conditions = append(b.Conditions, values[i]) - } - return b -} - -// WithPendingReplica sets the PendingReplica field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the PendingReplica field is set to the value of the last call. -func (b *ManagedSeedSetStatusApplyConfiguration) WithPendingReplica(value *PendingReplicaApplyConfiguration) *ManagedSeedSetStatusApplyConfiguration { - b.PendingReplica = value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedspec.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedspec.go deleted file mode 100644 index d5afc989112..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedspec.go +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -// ManagedSeedSpecApplyConfiguration represents an declarative configuration of the ManagedSeedSpec type for use -// with apply. -type ManagedSeedSpecApplyConfiguration struct { - Shoot *ShootApplyConfiguration `json:"shoot,omitempty"` - Gardenlet *GardenletApplyConfiguration `json:"gardenlet,omitempty"` -} - -// ManagedSeedSpecApplyConfiguration constructs an declarative configuration of the ManagedSeedSpec type for use with -// apply. -func ManagedSeedSpec() *ManagedSeedSpecApplyConfiguration { - return &ManagedSeedSpecApplyConfiguration{} -} - -// WithShoot sets the Shoot field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Shoot field is set to the value of the last call. -func (b *ManagedSeedSpecApplyConfiguration) WithShoot(value *ShootApplyConfiguration) *ManagedSeedSpecApplyConfiguration { - b.Shoot = value - return b -} - -// WithGardenlet sets the Gardenlet field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Gardenlet field is set to the value of the last call. -func (b *ManagedSeedSpecApplyConfiguration) WithGardenlet(value *GardenletApplyConfiguration) *ManagedSeedSpecApplyConfiguration { - b.Gardenlet = value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedstatus.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedstatus.go deleted file mode 100644 index 097272ffd7e..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedstatus.go +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" -) - -// ManagedSeedStatusApplyConfiguration represents an declarative configuration of the ManagedSeedStatus type for use -// with apply. -type ManagedSeedStatusApplyConfiguration struct { - Conditions []v1beta1.Condition `json:"conditions,omitempty"` - ObservedGeneration *int64 `json:"observedGeneration,omitempty"` -} - -// ManagedSeedStatusApplyConfiguration constructs an declarative configuration of the ManagedSeedStatus type for use with -// apply. -func ManagedSeedStatus() *ManagedSeedStatusApplyConfiguration { - return &ManagedSeedStatusApplyConfiguration{} -} - -// WithConditions adds the given value to the Conditions field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Conditions field. -func (b *ManagedSeedStatusApplyConfiguration) WithConditions(values ...v1beta1.Condition) *ManagedSeedStatusApplyConfiguration { - for i := range values { - b.Conditions = append(b.Conditions, values[i]) - } - return b -} - -// WithObservedGeneration sets the ObservedGeneration field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ObservedGeneration field is set to the value of the last call. -func (b *ManagedSeedStatusApplyConfiguration) WithObservedGeneration(value int64) *ManagedSeedStatusApplyConfiguration { - b.ObservedGeneration = &value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedtemplate.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedtemplate.go deleted file mode 100644 index 4ab011a1190..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/managedseedtemplate.go +++ /dev/null @@ -1,174 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "maps" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" -) - -// ManagedSeedTemplateApplyConfiguration represents an declarative configuration of the ManagedSeedTemplate type for use -// with apply. -type ManagedSeedTemplateApplyConfiguration struct { - *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` - Spec *ManagedSeedSpecApplyConfiguration `json:"spec,omitempty"` -} - -// ManagedSeedTemplateApplyConfiguration constructs an declarative configuration of the ManagedSeedTemplate type for use with -// apply. -func ManagedSeedTemplate() *ManagedSeedTemplateApplyConfiguration { - return &ManagedSeedTemplateApplyConfiguration{} -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithName(value string) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Name = &value - return b -} - -// WithGenerateName sets the GenerateName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the GenerateName field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithGenerateName(value string) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.GenerateName = &value - return b -} - -// WithNamespace sets the Namespace field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Namespace field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithNamespace(value string) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Namespace = &value - return b -} - -// WithUID sets the UID field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UID field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithUID(value types.UID) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.UID = &value - return b -} - -// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ResourceVersion field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithResourceVersion(value string) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.ResourceVersion = &value - return b -} - -// WithGeneration sets the Generation field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Generation field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithGeneration(value int64) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Generation = &value - return b -} - -// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CreationTimestamp field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithCreationTimestamp(value metav1.Time) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.CreationTimestamp = &value - return b -} - -// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionTimestamp field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionTimestamp = &value - return b -} - -// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionGracePeriodSeconds = &value - return b -} - -// WithLabels puts the entries into the Labels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Labels field, -// overwriting an existing map entries in Labels field with the same key. -func (b *ManagedSeedTemplateApplyConfiguration) WithLabels(entries map[string]string) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Labels == nil && len(entries) > 0 { - b.Labels = make(map[string]string, len(entries)) - } - maps.Copy(b.Labels, entries) - return b -} - -// WithAnnotations puts the entries into the Annotations field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Annotations field, -// overwriting an existing map entries in Annotations field with the same key. -func (b *ManagedSeedTemplateApplyConfiguration) WithAnnotations(entries map[string]string) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Annotations == nil && len(entries) > 0 { - b.Annotations = make(map[string]string, len(entries)) - } - maps.Copy(b.Annotations, entries) - return b -} - -// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the OwnerReferences field. -func (b *ManagedSeedTemplateApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - if values[i] == nil { - panic("nil value passed to WithOwnerReferences") - } - b.OwnerReferences = append(b.OwnerReferences, *values[i]) - } - return b -} - -// WithFinalizers adds the given value to the Finalizers field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Finalizers field. -func (b *ManagedSeedTemplateApplyConfiguration) WithFinalizers(values ...string) *ManagedSeedTemplateApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - b.Finalizers = append(b.Finalizers, values[i]) - } - return b -} - -func (b *ManagedSeedTemplateApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { - if b.ObjectMetaApplyConfiguration == nil { - b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} - } -} - -// WithSpec sets the Spec field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Spec field is set to the value of the last call. -func (b *ManagedSeedTemplateApplyConfiguration) WithSpec(value *ManagedSeedSpecApplyConfiguration) *ManagedSeedTemplateApplyConfiguration { - b.Spec = value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/pendingreplica.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/pendingreplica.go deleted file mode 100644 index ba9510b3f1a..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/pendingreplica.go +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// PendingReplicaApplyConfiguration represents an declarative configuration of the PendingReplica type for use -// with apply. -type PendingReplicaApplyConfiguration struct { - Name *string `json:"name,omitempty"` - Reason *v1alpha1.PendingReplicaReason `json:"reason,omitempty"` - Since *v1.Time `json:"since,omitempty"` - Retries *int32 `json:"retries,omitempty"` -} - -// PendingReplicaApplyConfiguration constructs an declarative configuration of the PendingReplica type for use with -// apply. -func PendingReplica() *PendingReplicaApplyConfiguration { - return &PendingReplicaApplyConfiguration{} -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *PendingReplicaApplyConfiguration) WithName(value string) *PendingReplicaApplyConfiguration { - b.Name = &value - return b -} - -// WithReason sets the Reason field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Reason field is set to the value of the last call. -func (b *PendingReplicaApplyConfiguration) WithReason(value v1alpha1.PendingReplicaReason) *PendingReplicaApplyConfiguration { - b.Reason = &value - return b -} - -// WithSince sets the Since field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Since field is set to the value of the last call. -func (b *PendingReplicaApplyConfiguration) WithSince(value v1.Time) *PendingReplicaApplyConfiguration { - b.Since = &value - return b -} - -// WithRetries sets the Retries field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Retries field is set to the value of the last call. -func (b *PendingReplicaApplyConfiguration) WithRetries(value int32) *PendingReplicaApplyConfiguration { - b.Retries = &value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/rollingupdatestrategy.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/rollingupdatestrategy.go deleted file mode 100644 index 7be8b4b7d6e..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/rollingupdatestrategy.go +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -// RollingUpdateStrategyApplyConfiguration represents an declarative configuration of the RollingUpdateStrategy type for use -// with apply. -type RollingUpdateStrategyApplyConfiguration struct { - Partition *int32 `json:"partition,omitempty"` -} - -// RollingUpdateStrategyApplyConfiguration constructs an declarative configuration of the RollingUpdateStrategy type for use with -// apply. -func RollingUpdateStrategy() *RollingUpdateStrategyApplyConfiguration { - return &RollingUpdateStrategyApplyConfiguration{} -} - -// WithPartition sets the Partition field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Partition field is set to the value of the last call. -func (b *RollingUpdateStrategyApplyConfiguration) WithPartition(value int32) *RollingUpdateStrategyApplyConfiguration { - b.Partition = &value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/shoot.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/shoot.go deleted file mode 100644 index 7526f8352d7..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/shoot.go +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -// ShootApplyConfiguration represents an declarative configuration of the Shoot type for use -// with apply. -type ShootApplyConfiguration struct { - Name *string `json:"name,omitempty"` -} - -// ShootApplyConfiguration constructs an declarative configuration of the Shoot type for use with -// apply. -func Shoot() *ShootApplyConfiguration { - return &ShootApplyConfiguration{} -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *ShootApplyConfiguration) WithName(value string) *ShootApplyConfiguration { - b.Name = &value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/updatestrategy.go b/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/updatestrategy.go deleted file mode 100644 index 3876e366d33..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1/updatestrategy.go +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1" -) - -// UpdateStrategyApplyConfiguration represents an declarative configuration of the UpdateStrategy type for use -// with apply. -type UpdateStrategyApplyConfiguration struct { - Type *v1alpha1.UpdateStrategyType `json:"type,omitempty"` - RollingUpdate *RollingUpdateStrategyApplyConfiguration `json:"rollingUpdate,omitempty"` -} - -// UpdateStrategyApplyConfiguration constructs an declarative configuration of the UpdateStrategy type for use with -// apply. -func UpdateStrategy() *UpdateStrategyApplyConfiguration { - return &UpdateStrategyApplyConfiguration{} -} - -// WithType sets the Type field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Type field is set to the value of the last call. -func (b *UpdateStrategyApplyConfiguration) WithType(value v1alpha1.UpdateStrategyType) *UpdateStrategyApplyConfiguration { - b.Type = &value - return b -} - -// WithRollingUpdate sets the RollingUpdate field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the RollingUpdate field is set to the value of the last call. -func (b *UpdateStrategyApplyConfiguration) WithRollingUpdate(value *RollingUpdateStrategyApplyConfiguration) *UpdateStrategyApplyConfiguration { - b.RollingUpdate = value - return b -} diff --git a/pkg/client/seedmanagement/applyconfiguration/utils.go b/pkg/client/seedmanagement/applyconfiguration/utils.go deleted file mode 100644 index 2cc6a227229..00000000000 --- a/pkg/client/seedmanagement/applyconfiguration/utils.go +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package applyconfiguration - -import ( - v1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1" - seedmanagementv1alpha1 "github.com/gardener/gardener/pkg/client/seedmanagement/applyconfiguration/seedmanagement/v1alpha1" - schema "k8s.io/apimachinery/pkg/runtime/schema" -) - -// ForKind returns an apply configuration type for the given GroupVersionKind, or nil if no -// apply configuration type exists for the given GroupVersionKind. -func ForKind(kind schema.GroupVersionKind) any { - switch kind { - // Group=seedmanagement.gardener.cloud, Version=v1alpha1 - case v1alpha1.SchemeGroupVersion.WithKind("GardenletConfig"): - return &seedmanagementv1alpha1.GardenletApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("GardenletDeployment"): - return &seedmanagementv1alpha1.GardenletDeploymentApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("Image"): - return &seedmanagementv1alpha1.ImageApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("ManagedSeed"): - return &seedmanagementv1alpha1.ManagedSeedApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("ManagedSeedSet"): - return &seedmanagementv1alpha1.ManagedSeedSetApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("ManagedSeedSetSpec"): - return &seedmanagementv1alpha1.ManagedSeedSetSpecApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("ManagedSeedSetStatus"): - return &seedmanagementv1alpha1.ManagedSeedSetStatusApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("ManagedSeedSpec"): - return &seedmanagementv1alpha1.ManagedSeedSpecApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("ManagedSeedStatus"): - return &seedmanagementv1alpha1.ManagedSeedStatusApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("ManagedSeedTemplate"): - return &seedmanagementv1alpha1.ManagedSeedTemplateApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("PendingReplica"): - return &seedmanagementv1alpha1.PendingReplicaApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("RollingUpdateStrategy"): - return &seedmanagementv1alpha1.RollingUpdateStrategyApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("Shoot"): - return &seedmanagementv1alpha1.ShootApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("UpdateStrategy"): - return &seedmanagementv1alpha1.UpdateStrategyApplyConfiguration{} - - } - return nil -} diff --git a/pkg/client/settings/applyconfiguration/internal/internal.go b/pkg/client/settings/applyconfiguration/internal/internal.go deleted file mode 100644 index 6b06d7c5835..00000000000 --- a/pkg/client/settings/applyconfiguration/internal/internal.go +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package internal - -import ( - "fmt" - "sync" - - typed "sigs.k8s.io/structured-merge-diff/v4/typed" -) - -func Parser() *typed.Parser { - parserOnce.Do(func() { - var err error - parser, err = typed.NewParser(schemaYAML) - if err != nil { - panic(fmt.Sprintf("Failed to parse schema: %v", err)) - } - }) - return parser -} - -var parserOnce sync.Once -var parser *typed.Parser -var schemaYAML = typed.YAMLObject(`types: -- name: __untyped_atomic_ - scalar: untyped - list: - elementType: - namedType: __untyped_atomic_ - elementRelationship: atomic - map: - elementType: - namedType: __untyped_atomic_ - elementRelationship: atomic -- name: __untyped_deduced_ - scalar: untyped - list: - elementType: - namedType: __untyped_atomic_ - elementRelationship: atomic - map: - elementType: - namedType: __untyped_deduced_ - elementRelationship: separable -`) diff --git a/pkg/client/settings/applyconfiguration/settings/v1alpha1/clusteropenidconnectpreset.go b/pkg/client/settings/applyconfiguration/settings/v1alpha1/clusteropenidconnectpreset.go deleted file mode 100644 index 3375c19abf2..00000000000 --- a/pkg/client/settings/applyconfiguration/settings/v1alpha1/clusteropenidconnectpreset.go +++ /dev/null @@ -1,195 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "maps" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" -) - -// ClusterOpenIDConnectPresetApplyConfiguration represents an declarative configuration of the ClusterOpenIDConnectPreset type for use -// with apply. -type ClusterOpenIDConnectPresetApplyConfiguration struct { - v1.TypeMetaApplyConfiguration `json:",inline"` - *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` - Spec *ClusterOpenIDConnectPresetSpecApplyConfiguration `json:"spec,omitempty"` -} - -// ClusterOpenIDConnectPreset constructs an declarative configuration of the ClusterOpenIDConnectPreset type for use with -// apply. -func ClusterOpenIDConnectPreset(name string) *ClusterOpenIDConnectPresetApplyConfiguration { - b := &ClusterOpenIDConnectPresetApplyConfiguration{} - b.WithName(name) - b.WithKind("ClusterOpenIDConnectPreset") - b.WithAPIVersion("settings.gardener.cloud/v1alpha1") - return b -} - -// WithKind sets the Kind field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Kind field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithKind(value string) *ClusterOpenIDConnectPresetApplyConfiguration { - b.Kind = &value - return b -} - -// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the APIVersion field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithAPIVersion(value string) *ClusterOpenIDConnectPresetApplyConfiguration { - b.APIVersion = &value - return b -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithName(value string) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Name = &value - return b -} - -// WithGenerateName sets the GenerateName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the GenerateName field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithGenerateName(value string) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.GenerateName = &value - return b -} - -// WithNamespace sets the Namespace field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Namespace field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithNamespace(value string) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Namespace = &value - return b -} - -// WithUID sets the UID field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UID field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithUID(value types.UID) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.UID = &value - return b -} - -// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ResourceVersion field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithResourceVersion(value string) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.ResourceVersion = &value - return b -} - -// WithGeneration sets the Generation field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Generation field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithGeneration(value int64) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Generation = &value - return b -} - -// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CreationTimestamp field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithCreationTimestamp(value metav1.Time) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.CreationTimestamp = &value - return b -} - -// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionTimestamp field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionTimestamp = &value - return b -} - -// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionGracePeriodSeconds = &value - return b -} - -// WithLabels puts the entries into the Labels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Labels field, -// overwriting an existing map entries in Labels field with the same key. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithLabels(entries map[string]string) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Labels == nil && len(entries) > 0 { - b.Labels = make(map[string]string, len(entries)) - } - maps.Copy(b.Labels, entries) - return b -} - -// WithAnnotations puts the entries into the Annotations field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Annotations field, -// overwriting an existing map entries in Annotations field with the same key. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithAnnotations(entries map[string]string) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Annotations == nil && len(entries) > 0 { - b.Annotations = make(map[string]string, len(entries)) - } - maps.Copy(b.Annotations, entries) - return b -} - -// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the OwnerReferences field. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - if values[i] == nil { - panic("nil value passed to WithOwnerReferences") - } - b.OwnerReferences = append(b.OwnerReferences, *values[i]) - } - return b -} - -// WithFinalizers adds the given value to the Finalizers field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Finalizers field. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithFinalizers(values ...string) *ClusterOpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - b.Finalizers = append(b.Finalizers, values[i]) - } - return b -} - -func (b *ClusterOpenIDConnectPresetApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { - if b.ObjectMetaApplyConfiguration == nil { - b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} - } -} - -// WithSpec sets the Spec field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Spec field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetApplyConfiguration) WithSpec(value *ClusterOpenIDConnectPresetSpecApplyConfiguration) *ClusterOpenIDConnectPresetApplyConfiguration { - b.Spec = value - return b -} diff --git a/pkg/client/settings/applyconfiguration/settings/v1alpha1/clusteropenidconnectpresetspec.go b/pkg/client/settings/applyconfiguration/settings/v1alpha1/clusteropenidconnectpresetspec.go deleted file mode 100644 index da870d4de5d..00000000000 --- a/pkg/client/settings/applyconfiguration/settings/v1alpha1/clusteropenidconnectpresetspec.go +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// ClusterOpenIDConnectPresetSpecApplyConfiguration represents an declarative configuration of the ClusterOpenIDConnectPresetSpec type for use -// with apply. -type ClusterOpenIDConnectPresetSpecApplyConfiguration struct { - OpenIDConnectPresetSpecApplyConfiguration `json:",inline"` - ProjectSelector *v1.LabelSelector `json:"projectSelector,omitempty"` -} - -// ClusterOpenIDConnectPresetSpecApplyConfiguration constructs an declarative configuration of the ClusterOpenIDConnectPresetSpec type for use with -// apply. -func ClusterOpenIDConnectPresetSpec() *ClusterOpenIDConnectPresetSpecApplyConfiguration { - return &ClusterOpenIDConnectPresetSpecApplyConfiguration{} -} - -// WithServer sets the Server field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Server field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetSpecApplyConfiguration) WithServer(value *KubeAPIServerOpenIDConnectApplyConfiguration) *ClusterOpenIDConnectPresetSpecApplyConfiguration { - b.Server = value - return b -} - -// WithClient sets the Client field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Client field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetSpecApplyConfiguration) WithClient(value *OpenIDConnectClientAuthenticationApplyConfiguration) *ClusterOpenIDConnectPresetSpecApplyConfiguration { - b.Client = value - return b -} - -// WithShootSelector sets the ShootSelector field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ShootSelector field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetSpecApplyConfiguration) WithShootSelector(value v1.LabelSelector) *ClusterOpenIDConnectPresetSpecApplyConfiguration { - b.ShootSelector = &value - return b -} - -// WithWeight sets the Weight field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Weight field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetSpecApplyConfiguration) WithWeight(value int32) *ClusterOpenIDConnectPresetSpecApplyConfiguration { - b.Weight = &value - return b -} - -// WithProjectSelector sets the ProjectSelector field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ProjectSelector field is set to the value of the last call. -func (b *ClusterOpenIDConnectPresetSpecApplyConfiguration) WithProjectSelector(value v1.LabelSelector) *ClusterOpenIDConnectPresetSpecApplyConfiguration { - b.ProjectSelector = &value - return b -} diff --git a/pkg/client/settings/applyconfiguration/settings/v1alpha1/kubeapiserveropenidconnect.go b/pkg/client/settings/applyconfiguration/settings/v1alpha1/kubeapiserveropenidconnect.go deleted file mode 100644 index 4da7c566e8c..00000000000 --- a/pkg/client/settings/applyconfiguration/settings/v1alpha1/kubeapiserveropenidconnect.go +++ /dev/null @@ -1,107 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import "maps" - -// KubeAPIServerOpenIDConnectApplyConfiguration represents an declarative configuration of the KubeAPIServerOpenIDConnect type for use -// with apply. -type KubeAPIServerOpenIDConnectApplyConfiguration struct { - CABundle *string `json:"caBundle,omitempty"` - ClientID *string `json:"clientID,omitempty"` - GroupsClaim *string `json:"groupsClaim,omitempty"` - GroupsPrefix *string `json:"groupsPrefix,omitempty"` - IssuerURL *string `json:"issuerURL,omitempty"` - RequiredClaims map[string]string `json:"requiredClaims,omitempty"` - SigningAlgs []string `json:"signingAlgs,omitempty"` - UsernameClaim *string `json:"usernameClaim,omitempty"` - UsernamePrefix *string `json:"usernamePrefix,omitempty"` -} - -// KubeAPIServerOpenIDConnectApplyConfiguration constructs an declarative configuration of the KubeAPIServerOpenIDConnect type for use with -// apply. -func KubeAPIServerOpenIDConnect() *KubeAPIServerOpenIDConnectApplyConfiguration { - return &KubeAPIServerOpenIDConnectApplyConfiguration{} -} - -// WithCABundle sets the CABundle field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CABundle field is set to the value of the last call. -func (b *KubeAPIServerOpenIDConnectApplyConfiguration) WithCABundle(value string) *KubeAPIServerOpenIDConnectApplyConfiguration { - b.CABundle = &value - return b -} - -// WithClientID sets the ClientID field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ClientID field is set to the value of the last call. -func (b *KubeAPIServerOpenIDConnectApplyConfiguration) WithClientID(value string) *KubeAPIServerOpenIDConnectApplyConfiguration { - b.ClientID = &value - return b -} - -// WithGroupsClaim sets the GroupsClaim field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the GroupsClaim field is set to the value of the last call. -func (b *KubeAPIServerOpenIDConnectApplyConfiguration) WithGroupsClaim(value string) *KubeAPIServerOpenIDConnectApplyConfiguration { - b.GroupsClaim = &value - return b -} - -// WithGroupsPrefix sets the GroupsPrefix field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the GroupsPrefix field is set to the value of the last call. -func (b *KubeAPIServerOpenIDConnectApplyConfiguration) WithGroupsPrefix(value string) *KubeAPIServerOpenIDConnectApplyConfiguration { - b.GroupsPrefix = &value - return b -} - -// WithIssuerURL sets the IssuerURL field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the IssuerURL field is set to the value of the last call. -func (b *KubeAPIServerOpenIDConnectApplyConfiguration) WithIssuerURL(value string) *KubeAPIServerOpenIDConnectApplyConfiguration { - b.IssuerURL = &value - return b -} - -// WithRequiredClaims puts the entries into the RequiredClaims field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the RequiredClaims field, -// overwriting an existing map entries in RequiredClaims field with the same key. -func (b *KubeAPIServerOpenIDConnectApplyConfiguration) WithRequiredClaims(entries map[string]string) *KubeAPIServerOpenIDConnectApplyConfiguration { - if b.RequiredClaims == nil && len(entries) > 0 { - b.RequiredClaims = make(map[string]string, len(entries)) - } - maps.Copy(b.RequiredClaims, entries) - return b -} - -// WithSigningAlgs adds the given value to the SigningAlgs field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the SigningAlgs field. -func (b *KubeAPIServerOpenIDConnectApplyConfiguration) WithSigningAlgs(values ...string) *KubeAPIServerOpenIDConnectApplyConfiguration { - for i := range values { - b.SigningAlgs = append(b.SigningAlgs, values[i]) - } - return b -} - -// WithUsernameClaim sets the UsernameClaim field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UsernameClaim field is set to the value of the last call. -func (b *KubeAPIServerOpenIDConnectApplyConfiguration) WithUsernameClaim(value string) *KubeAPIServerOpenIDConnectApplyConfiguration { - b.UsernameClaim = &value - return b -} - -// WithUsernamePrefix sets the UsernamePrefix field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UsernamePrefix field is set to the value of the last call. -func (b *KubeAPIServerOpenIDConnectApplyConfiguration) WithUsernamePrefix(value string) *KubeAPIServerOpenIDConnectApplyConfiguration { - b.UsernamePrefix = &value - return b -} diff --git a/pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectclientauthentication.go b/pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectclientauthentication.go deleted file mode 100644 index 292c361e649..00000000000 --- a/pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectclientauthentication.go +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import "maps" - -// OpenIDConnectClientAuthenticationApplyConfiguration represents an declarative configuration of the OpenIDConnectClientAuthentication type for use -// with apply. -type OpenIDConnectClientAuthenticationApplyConfiguration struct { - Secret *string `json:"secret,omitempty"` - ExtraConfig map[string]string `json:"extraConfig,omitempty"` -} - -// OpenIDConnectClientAuthenticationApplyConfiguration constructs an declarative configuration of the OpenIDConnectClientAuthentication type for use with -// apply. -func OpenIDConnectClientAuthentication() *OpenIDConnectClientAuthenticationApplyConfiguration { - return &OpenIDConnectClientAuthenticationApplyConfiguration{} -} - -// WithSecret sets the Secret field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Secret field is set to the value of the last call. -func (b *OpenIDConnectClientAuthenticationApplyConfiguration) WithSecret(value string) *OpenIDConnectClientAuthenticationApplyConfiguration { - b.Secret = &value - return b -} - -// WithExtraConfig puts the entries into the ExtraConfig field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the ExtraConfig field, -// overwriting an existing map entries in ExtraConfig field with the same key. -func (b *OpenIDConnectClientAuthenticationApplyConfiguration) WithExtraConfig(entries map[string]string) *OpenIDConnectClientAuthenticationApplyConfiguration { - if b.ExtraConfig == nil && len(entries) > 0 { - b.ExtraConfig = make(map[string]string, len(entries)) - } - maps.Copy(b.ExtraConfig, entries) - return b -} diff --git a/pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectpreset.go b/pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectpreset.go deleted file mode 100644 index 697066c318a..00000000000 --- a/pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectpreset.go +++ /dev/null @@ -1,196 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "maps" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" -) - -// OpenIDConnectPresetApplyConfiguration represents an declarative configuration of the OpenIDConnectPreset type for use -// with apply. -type OpenIDConnectPresetApplyConfiguration struct { - v1.TypeMetaApplyConfiguration `json:",inline"` - *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` - Spec *OpenIDConnectPresetSpecApplyConfiguration `json:"spec,omitempty"` -} - -// OpenIDConnectPreset constructs an declarative configuration of the OpenIDConnectPreset type for use with -// apply. -func OpenIDConnectPreset(name, namespace string) *OpenIDConnectPresetApplyConfiguration { - b := &OpenIDConnectPresetApplyConfiguration{} - b.WithName(name) - b.WithNamespace(namespace) - b.WithKind("OpenIDConnectPreset") - b.WithAPIVersion("settings.gardener.cloud/v1alpha1") - return b -} - -// WithKind sets the Kind field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Kind field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithKind(value string) *OpenIDConnectPresetApplyConfiguration { - b.Kind = &value - return b -} - -// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the APIVersion field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithAPIVersion(value string) *OpenIDConnectPresetApplyConfiguration { - b.APIVersion = &value - return b -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithName(value string) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Name = &value - return b -} - -// WithGenerateName sets the GenerateName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the GenerateName field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithGenerateName(value string) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.GenerateName = &value - return b -} - -// WithNamespace sets the Namespace field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Namespace field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithNamespace(value string) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Namespace = &value - return b -} - -// WithUID sets the UID field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UID field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithUID(value types.UID) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.UID = &value - return b -} - -// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ResourceVersion field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithResourceVersion(value string) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.ResourceVersion = &value - return b -} - -// WithGeneration sets the Generation field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Generation field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithGeneration(value int64) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Generation = &value - return b -} - -// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CreationTimestamp field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithCreationTimestamp(value metav1.Time) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.CreationTimestamp = &value - return b -} - -// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionTimestamp field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionTimestamp = &value - return b -} - -// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionGracePeriodSeconds = &value - return b -} - -// WithLabels puts the entries into the Labels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Labels field, -// overwriting an existing map entries in Labels field with the same key. -func (b *OpenIDConnectPresetApplyConfiguration) WithLabels(entries map[string]string) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Labels == nil && len(entries) > 0 { - b.Labels = make(map[string]string, len(entries)) - } - maps.Copy(b.Labels, entries) - return b -} - -// WithAnnotations puts the entries into the Annotations field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Annotations field, -// overwriting an existing map entries in Annotations field with the same key. -func (b *OpenIDConnectPresetApplyConfiguration) WithAnnotations(entries map[string]string) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Annotations == nil && len(entries) > 0 { - b.Annotations = make(map[string]string, len(entries)) - } - maps.Copy(b.Annotations, entries) - return b -} - -// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the OwnerReferences field. -func (b *OpenIDConnectPresetApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - if values[i] == nil { - panic("nil value passed to WithOwnerReferences") - } - b.OwnerReferences = append(b.OwnerReferences, *values[i]) - } - return b -} - -// WithFinalizers adds the given value to the Finalizers field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Finalizers field. -func (b *OpenIDConnectPresetApplyConfiguration) WithFinalizers(values ...string) *OpenIDConnectPresetApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - b.Finalizers = append(b.Finalizers, values[i]) - } - return b -} - -func (b *OpenIDConnectPresetApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { - if b.ObjectMetaApplyConfiguration == nil { - b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} - } -} - -// WithSpec sets the Spec field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Spec field is set to the value of the last call. -func (b *OpenIDConnectPresetApplyConfiguration) WithSpec(value *OpenIDConnectPresetSpecApplyConfiguration) *OpenIDConnectPresetApplyConfiguration { - b.Spec = value - return b -} diff --git a/pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectpresetspec.go b/pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectpresetspec.go deleted file mode 100644 index 799790d7534..00000000000 --- a/pkg/client/settings/applyconfiguration/settings/v1alpha1/openidconnectpresetspec.go +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// OpenIDConnectPresetSpecApplyConfiguration represents an declarative configuration of the OpenIDConnectPresetSpec type for use -// with apply. -type OpenIDConnectPresetSpecApplyConfiguration struct { - Server *KubeAPIServerOpenIDConnectApplyConfiguration `json:"server,omitempty"` - Client *OpenIDConnectClientAuthenticationApplyConfiguration `json:"client,omitempty"` - ShootSelector *v1.LabelSelector `json:"shootSelector,omitempty"` - Weight *int32 `json:"weight,omitempty"` -} - -// OpenIDConnectPresetSpecApplyConfiguration constructs an declarative configuration of the OpenIDConnectPresetSpec type for use with -// apply. -func OpenIDConnectPresetSpec() *OpenIDConnectPresetSpecApplyConfiguration { - return &OpenIDConnectPresetSpecApplyConfiguration{} -} - -// WithServer sets the Server field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Server field is set to the value of the last call. -func (b *OpenIDConnectPresetSpecApplyConfiguration) WithServer(value *KubeAPIServerOpenIDConnectApplyConfiguration) *OpenIDConnectPresetSpecApplyConfiguration { - b.Server = value - return b -} - -// WithClient sets the Client field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Client field is set to the value of the last call. -func (b *OpenIDConnectPresetSpecApplyConfiguration) WithClient(value *OpenIDConnectClientAuthenticationApplyConfiguration) *OpenIDConnectPresetSpecApplyConfiguration { - b.Client = value - return b -} - -// WithShootSelector sets the ShootSelector field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ShootSelector field is set to the value of the last call. -func (b *OpenIDConnectPresetSpecApplyConfiguration) WithShootSelector(value v1.LabelSelector) *OpenIDConnectPresetSpecApplyConfiguration { - b.ShootSelector = &value - return b -} - -// WithWeight sets the Weight field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Weight field is set to the value of the last call. -func (b *OpenIDConnectPresetSpecApplyConfiguration) WithWeight(value int32) *OpenIDConnectPresetSpecApplyConfiguration { - b.Weight = &value - return b -} diff --git a/pkg/client/settings/applyconfiguration/utils.go b/pkg/client/settings/applyconfiguration/utils.go deleted file mode 100644 index 6314caad932..00000000000 --- a/pkg/client/settings/applyconfiguration/utils.go +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package applyconfiguration - -import ( - v1alpha1 "github.com/gardener/gardener/pkg/apis/settings/v1alpha1" - settingsv1alpha1 "github.com/gardener/gardener/pkg/client/settings/applyconfiguration/settings/v1alpha1" - schema "k8s.io/apimachinery/pkg/runtime/schema" -) - -// ForKind returns an apply configuration type for the given GroupVersionKind, or nil if no -// apply configuration type exists for the given GroupVersionKind. -func ForKind(kind schema.GroupVersionKind) any { - switch kind { - // Group=settings.gardener.cloud, Version=v1alpha1 - case v1alpha1.SchemeGroupVersion.WithKind("ClusterOpenIDConnectPreset"): - return &settingsv1alpha1.ClusterOpenIDConnectPresetApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("ClusterOpenIDConnectPresetSpec"): - return &settingsv1alpha1.ClusterOpenIDConnectPresetSpecApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("KubeAPIServerOpenIDConnect"): - return &settingsv1alpha1.KubeAPIServerOpenIDConnectApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("OpenIDConnectClientAuthentication"): - return &settingsv1alpha1.OpenIDConnectClientAuthenticationApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("OpenIDConnectPreset"): - return &settingsv1alpha1.OpenIDConnectPresetApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("OpenIDConnectPresetSpec"): - return &settingsv1alpha1.OpenIDConnectPresetSpecApplyConfiguration{} - - } - return nil -} diff --git a/pkg/component/autoscaling/vpa/templates/crd-autoscaling.k8s.io_verticalpodautoscalercheckpoints.yaml b/pkg/component/autoscaling/vpa/templates/crd-autoscaling.k8s.io_verticalpodautoscalercheckpoints.yaml index 4266f366c00..2203cba9f2d 100644 --- a/pkg/component/autoscaling/vpa/templates/crd-autoscaling.k8s.io_verticalpodautoscalercheckpoints.yaml +++ b/pkg/component/autoscaling/vpa/templates/crd-autoscaling.k8s.io_verticalpodautoscalercheckpoints.yaml @@ -5,7 +5,7 @@ metadata: annotations: resources.gardener.cloud/keep-object: "true" api-approved.kubernetes.io: https://github.com/kubernetes/kubernetes/pull/63797 - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: verticalpodautoscalercheckpoints.autoscaling.k8s.io spec: group: autoscaling.k8s.io diff --git a/pkg/component/autoscaling/vpa/templates/crd-autoscaling.k8s.io_verticalpodautoscalers.yaml b/pkg/component/autoscaling/vpa/templates/crd-autoscaling.k8s.io_verticalpodautoscalers.yaml index 8e94ce1ca28..191fe201933 100644 --- a/pkg/component/autoscaling/vpa/templates/crd-autoscaling.k8s.io_verticalpodautoscalers.yaml +++ b/pkg/component/autoscaling/vpa/templates/crd-autoscaling.k8s.io_verticalpodautoscalers.yaml @@ -5,7 +5,7 @@ metadata: annotations: resources.gardener.cloud/keep-object: "true" api-approved.kubernetes.io: https://github.com/kubernetes/kubernetes/pull/63797 - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: verticalpodautoscalers.autoscaling.k8s.io spec: group: autoscaling.k8s.io diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_backupbuckets.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_backupbuckets.yaml index f56520169b6..71ee9569aeb 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_backupbuckets.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_backupbuckets.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: backupbuckets.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_backupentries.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_backupentries.yaml index b03875890a8..322b116e836 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_backupentries.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_backupentries.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: backupentries.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_bastions.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_bastions.yaml index a670a7007f6..6ae72f0e9ff 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_bastions.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_bastions.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: bastions.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_clusters.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_clusters.yaml index 9b5611899d0..c670d2421b3 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_clusters.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusters.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_containerruntimes.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_containerruntimes.yaml index 38f11e81ac5..a635fc787aa 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_containerruntimes.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_containerruntimes.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: containerruntimes.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_controlplanes.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_controlplanes.yaml index 81f25012c68..29fcbba6f78 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_controlplanes.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_controlplanes.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: controlplanes.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_dnsrecords.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_dnsrecords.yaml index f5bc16037ee..0e75b35b7ea 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_dnsrecords.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_dnsrecords.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: dnsrecords.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_extensions.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_extensions.yaml index 1bcdad48489..e265f7b7481 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_extensions.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_extensions.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: extensions.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_infrastructures.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_infrastructures.yaml index 0cfebbcfb0e..a88b01db554 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_infrastructures.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_infrastructures.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: infrastructures.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_networks.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_networks.yaml index 33b6c9ea4b2..f5cc01ac850 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_networks.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_networks.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: networks.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_operatingsystemconfigs.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_operatingsystemconfigs.yaml index 3c35391fe7c..e870cf9af4d 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_operatingsystemconfigs.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_operatingsystemconfigs.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: operatingsystemconfigs.extensions.gardener.cloud spec: group: extensions.gardener.cloud diff --git a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_workers.yaml b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_workers.yaml index 8981af70b25..2082d1997fb 100644 --- a/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_workers.yaml +++ b/pkg/component/extensions/crds/assets/crd-extensions.gardener.cloud_workers.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: workers.extensions.gardener.cloud spec: group: extensions.gardener.cloud @@ -650,9 +650,8 @@ spec: a node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which the + taint was added. format: date-time type: string value: diff --git a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/rbac_test.go b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/rbac_test.go index 575e5205a43..f09301db120 100644 --- a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/rbac_test.go +++ b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/rbac_test.go @@ -26,7 +26,6 @@ var _ = Describe("RBAC", func() { clusterRoleBindingNodeBootstrapperYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - creationTimestamp: null name: system:node-bootstrapper roleRef: apiGroup: rbac.authorization.k8s.io @@ -41,7 +40,6 @@ subjects: clusterRoleBindingNodeClientYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - creationTimestamp: null name: system:certificates.k8s.io:certificatesigningrequests:nodeclient roleRef: apiGroup: rbac.authorization.k8s.io @@ -55,7 +53,6 @@ subjects: clusterRoleBindingSelfNodeClientYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - creationTimestamp: null name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient roleRef: apiGroup: rbac.authorization.k8s.io diff --git a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/secrets_test.go b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/secrets_test.go index ada4895fe65..1560cf8e77e 100644 --- a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/secrets_test.go +++ b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/secrets_test.go @@ -109,7 +109,7 @@ var _ = Describe("Secrets", func() { Name: secretName, Namespace: "kube-system", Annotations: map[string]string{ - "checksum/data-script": "a3efca387130b4a0291595c1608d80cad758ca95f71901f1fc91b0d74847c0ff", + "checksum/data-script": "e92163ba634e290c2e13de35a778007114e201e168c0a27149d05ab5592e4d36", }, Labels: map[string]string{ "gardener.cloud/role": "operating-system-config", @@ -118,8 +118,7 @@ var _ = Describe("Secrets", func() { }, Data: map[string][]byte{"osc.yaml": []byte(`apiVersion: extensions.gardener.cloud/v1alpha1 kind: OperatingSystemConfig -metadata: - creationTimestamp: null +metadata: {} spec: criConfig: containerd: diff --git a/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/config_test.go b/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/config_test.go index b218ed7f856..7e087c3cceb 100644 --- a/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/config_test.go +++ b/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/config_test.go @@ -250,7 +250,6 @@ contexts: name: shoot current-context: shoot kind: Config -preferences: {} users: - name: shoot user: diff --git a/pkg/component/gardener/access/access_test.go b/pkg/component/gardener/access/access_test.go index a411d7c4a80..e0ff81fae83 100644 --- a/pkg/component/gardener/access/access_test.go +++ b/pkg/component/gardener/access/access_test.go @@ -197,7 +197,6 @@ contexts: name: ` + namespace + ` current-context: ` + namespace + ` kind: Config -preferences: {} users: - name: ` + namespace + ` user: {} @@ -231,7 +230,6 @@ contexts: name: ` + namespace + ` current-context: ` + namespace + ` kind: Config -preferences: {} users: - name: ` + namespace + ` user: {} diff --git a/pkg/component/gardener/apiserver/apiserver_test.go b/pkg/component/gardener/apiserver/apiserver_test.go index 0de2538363a..744b8632ca6 100644 --- a/pkg/component/gardener/apiserver/apiserver_test.go +++ b/pkg/component/gardener/apiserver/apiserver_test.go @@ -269,7 +269,7 @@ var _ = Describe("GardenerAPIServer", func() { "role": "apiserver", }, Annotations: map[string]string{ - "reference.resources.gardener.cloud/configmap-0e4e3fd5": "gardener-apiserver-audit-policy-config-f5b578b4", + "reference.resources.gardener.cloud/configmap-1b2d9b42": "gardener-apiserver-audit-policy-config-1e270362", "reference.resources.gardener.cloud/configmap-6e5f123b": "gardener-apiserver-admission-config-07c5248a", "reference.resources.gardener.cloud/secret-9dca243c": "shoot-access-gardener-apiserver", "reference.resources.gardener.cloud/secret-47fc132b": "gardener-apiserver-admission-kubeconfigs-e3b0c442", @@ -309,7 +309,7 @@ var _ = Describe("GardenerAPIServer", func() { "networking.resources.gardener.cloud/to-virtual-garden-kube-apiserver-tcp-443": "allowed", }, Annotations: map[string]string{ - "reference.resources.gardener.cloud/configmap-0e4e3fd5": "gardener-apiserver-audit-policy-config-f5b578b4", + "reference.resources.gardener.cloud/configmap-1b2d9b42": "gardener-apiserver-audit-policy-config-1e270362", "reference.resources.gardener.cloud/configmap-6e5f123b": "gardener-apiserver-admission-config-07c5248a", "reference.resources.gardener.cloud/secret-9dca243c": "shoot-access-gardener-apiserver", "reference.resources.gardener.cloud/secret-47fc132b": "gardener-apiserver-admission-kubeconfigs-e3b0c442", @@ -481,7 +481,7 @@ var _ = Describe("GardenerAPIServer", func() { VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "gardener-apiserver-audit-policy-config-f5b578b4", + Name: "gardener-apiserver-audit-policy-config-1e270362", }, }, }, @@ -983,8 +983,7 @@ resources: ObjectMeta: metav1.ObjectMeta{Name: "gardener-apiserver-audit-policy-config", Namespace: namespace}, Data: map[string]string{"audit-policy.yaml": `apiVersion: audit.k8s.io/v1 kind: Policy -metadata: - creationTimestamp: null +metadata: {} rules: - level: None `}, diff --git a/pkg/component/gardener/dashboard/terminal/assets/crd-dashboard.gardener.cloud_terminals.yaml b/pkg/component/gardener/dashboard/terminal/assets/crd-dashboard.gardener.cloud_terminals.yaml index eb6a403376a..0942283556c 100644 --- a/pkg/component/gardener/dashboard/terminal/assets/crd-dashboard.gardener.cloud_terminals.yaml +++ b/pkg/component/gardener/dashboard/terminal/assets/crd-dashboard.gardener.cloud_terminals.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/component: controller-manager app.kubernetes.io/name: terminal @@ -172,7 +172,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. diff --git a/pkg/component/gardener/resourcemanager/assets/crd-resources.gardener.cloud_managedresources.yaml b/pkg/component/gardener/resourcemanager/assets/crd-resources.gardener.cloud_managedresources.yaml index 2bf40ce0eea..621ec802011 100644 --- a/pkg/component/gardener/resourcemanager/assets/crd-resources.gardener.cloud_managedresources.yaml +++ b/pkg/component/gardener/resourcemanager/assets/crd-resources.gardener.cloud_managedresources.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: managedresources.resources.gardener.cloud spec: group: resources.gardener.cloud diff --git a/pkg/component/gardener/resourcemanager/resource_manager_test.go b/pkg/component/gardener/resourcemanager/resource_manager_test.go index 2bb30e98f45..6449c427a86 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager_test.go +++ b/pkg/component/gardener/resourcemanager/resource_manager_test.go @@ -1302,7 +1302,6 @@ var _ = Describe("ResourceManager", func() { out := `apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: - creationTimestamp: null labels: app: gardener-resource-manager name: gardener-resource-manager-shoot @@ -1579,7 +1578,6 @@ kind: ClusterRoleBinding metadata: annotations: resources.gardener.cloud/keep-object: "true" - creationTimestamp: null name: gardener.cloud:target:resource-manager roleRef: apiGroup: rbac.authorization.k8s.io diff --git a/pkg/component/kubernetes/apiserver/apiserver_test.go b/pkg/component/kubernetes/apiserver/apiserver_test.go index c6932e84929..37a59aff10b 100644 --- a/pkg/component/kubernetes/apiserver/apiserver_test.go +++ b/pkg/component/kubernetes/apiserver/apiserver_test.go @@ -109,7 +109,7 @@ var _ = Describe("KubeAPIServer", func() { configMapNameAdmissionConfigs = "kube-apiserver-admission-config-07c5248a" secretNameAdmissionKubeconfigs = "kube-apiserver-admission-kubeconfigs-e3b0c442" secretNameETCDEncryptionConfig = "kube-apiserver-etcd-encryption-configuration-b2b49c90" - configMapNameAuditPolicy = "audit-policy-config-f5b578b4" + configMapNameAuditPolicy = "audit-policy-config-1e270362" configMapNameEgressPolicy = "kube-apiserver-egress-selector-config-53d92abc" configMapEnvoyConfig = "kube-apiserver-envoy-config-a60282c1" configMapEgressSelectorConfig = "kube-apiserver-egress-selector-config-02bc784a" @@ -1682,8 +1682,7 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook ObjectMeta: metav1.ObjectMeta{Name: "audit-policy-config", Namespace: namespace}, Data: map[string]string{"audit-policy.yaml": `apiVersion: audit.k8s.io/v1 kind: Policy -metadata: - creationTimestamp: null +metadata: {} rules: - level: None `}, @@ -2764,7 +2763,7 @@ kind: AuthorizationConfiguration "reference.resources.gardener.cloud/secret-c4700ce9": secretNameETCDEncryptionConfig, "reference.resources.gardener.cloud/configmap-0ddefe9e": configMapNameAdmissionConfigs, "reference.resources.gardener.cloud/secret-5613e39f": secretNameAdmissionKubeconfigs, - "reference.resources.gardener.cloud/configmap-d4419cd4": configMapNameAuditPolicy, + "reference.resources.gardener.cloud/configmap-cc790102": configMapNameAuditPolicy, } }) @@ -2986,7 +2985,7 @@ kind: AuthorizationConfiguration "reference.resources.gardener.cloud/secret-c4700ce9": secretNameETCDEncryptionConfig, "reference.resources.gardener.cloud/configmap-0ddefe9e": configMapNameAdmissionConfigs, "reference.resources.gardener.cloud/secret-5613e39f": secretNameAdmissionKubeconfigs, - "reference.resources.gardener.cloud/configmap-d4419cd4": configMapNameAuditPolicy, + "reference.resources.gardener.cloud/configmap-cc790102": configMapNameAuditPolicy, } }) diff --git a/pkg/component/networking/coredns/coredns_test.go b/pkg/component/networking/coredns/coredns_test.go index 7527e5a1314..967e59a665d 100644 --- a/pkg/component/networking/coredns/coredns_test.go +++ b/pkg/component/networking/coredns/coredns_test.go @@ -65,14 +65,12 @@ var _ = Describe("CoreDNS", func() { automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null name: coredns namespace: kube-system ` clusterRoleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: system:coredns rules: - apiGroups: @@ -104,7 +102,6 @@ kind: ClusterRoleBinding metadata: annotations: resources.gardener.cloud/delete-on-invalid-update: "true" - creationTimestamp: null name: system:coredns roleRef: apiGroup: rbac.authorization.k8s.io @@ -158,7 +155,6 @@ data: import custom/*.server kind: ConfigMap metadata: - creationTimestamp: null name: coredns namespace: kube-system ` @@ -172,7 +168,6 @@ kind: ConfigMap metadata: annotations: resources.gardener.cloud/ignore: "true" - creationTimestamp: null name: coredns-custom namespace: kube-system ` @@ -180,7 +175,6 @@ metadata: out := `apiVersion: v1 kind: Service metadata: - creationTimestamp: null labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" @@ -217,7 +211,6 @@ kind: NetworkPolicy metadata: annotations: gardener.cloud/description: Allows CoreDNS to lookup DNS records, talk to the API Server. Also allows CoreDNS to be reachable via its service and its metrics endpoint. - creationTimestamp: null name: gardener.cloud--allow-dns namespace: kube-system spec: @@ -262,7 +255,6 @@ spec: out := `apiVersion: apps/v1 kind: Deployment metadata: - creationTimestamp: null labels: gardener.cloud/role: system-component ` @@ -304,8 +296,7 @@ spec: ` } - out += ` creationTimestamp: null - labels: + out += ` labels: gardener.cloud/role: system-component k8s-app: kube-dns origin: gardener @@ -406,7 +397,6 @@ status: {} pdbYAML = `apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - creationTimestamp: null labels: k8s-app: kube-dns name: coredns @@ -427,7 +417,6 @@ status: hpaYAML = `apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - creationTimestamp: null labels: high-availability-config.resources.gardener.cloud/type: server name: coredns @@ -455,14 +444,12 @@ status: automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null name: coredns-autoscaler namespace: kube-system ` cpacrYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: system:coredns-autoscaler rules: - apiGroups: @@ -498,7 +485,6 @@ rules: cpacrbYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - creationTimestamp: null name: system:coredns-autoscaler roleRef: apiGroup: rbac.authorization.k8s.io @@ -512,7 +498,6 @@ subjects: cpaDeploymentYAML = `apiVersion: apps/v1 kind: Deployment metadata: - creationTimestamp: null labels: gardener.cloud/role: system-component k8s-app: coredns-autoscaler @@ -527,7 +512,6 @@ spec: strategy: {} template: metadata: - creationTimestamp: null labels: gardener.cloud/role: system-component k8s-app: coredns-autoscaler @@ -574,7 +558,6 @@ status: {} cpaDeploymentVpaYAML = `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - creationTimestamp: null name: coredns-autoscaler namespace: kube-system spec: diff --git a/pkg/component/networking/istio/test_charts/ingress_namespace.yaml b/pkg/component/networking/istio/test_charts/ingress_namespace.yaml index 7e5ce02eded..853090161ca 100644 --- a/pkg/component/networking/istio/test_charts/ingress_namespace.yaml +++ b/pkg/component/networking/istio/test_charts/ingress_namespace.yaml @@ -3,7 +3,6 @@ kind: Namespace metadata: annotations: high-availability-config.resources.gardener.cloud/zones: a,b,c - creationTimestamp: null labels: gardener.cloud/role: istio-ingress high-availability-config.resources.gardener.cloud/consider: "true" diff --git a/pkg/component/networking/istio/test_charts/ingress_servicemonitor.yaml b/pkg/component/networking/istio/test_charts/ingress_servicemonitor.yaml index 90ca571028a..c0d4121f95f 100644 --- a/pkg/component/networking/istio/test_charts/ingress_servicemonitor.yaml +++ b/pkg/component/networking/istio/test_charts/ingress_servicemonitor.yaml @@ -1,7 +1,6 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - creationTimestamp: null labels: prometheus: aggregate name: aggregate-istio-ingressgateway diff --git a/pkg/component/networking/istio/test_charts/ingress_telemetry.yaml b/pkg/component/networking/istio/test_charts/ingress_telemetry.yaml index 2850e2d4de6..ba9ea10173e 100644 --- a/pkg/component/networking/istio/test_charts/ingress_telemetry.yaml +++ b/pkg/component/networking/istio/test_charts/ingress_telemetry.yaml @@ -1,7 +1,6 @@ apiVersion: telemetry.istio.io/v1 kind: Telemetry metadata: - creationTimestamp: null labels: prometheus: aggregate name: aggregate-enable-upstream-address diff --git a/pkg/component/networking/istio/test_charts/istiod_servicemonitor.yaml b/pkg/component/networking/istio/test_charts/istiod_servicemonitor.yaml index beaad08ed45..546eb495d85 100644 --- a/pkg/component/networking/istio/test_charts/istiod_servicemonitor.yaml +++ b/pkg/component/networking/istio/test_charts/istiod_servicemonitor.yaml @@ -1,7 +1,6 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - creationTimestamp: null labels: prometheus: aggregate name: aggregate-istiod diff --git a/pkg/component/networking/nginxingress/nginxingress_test.go b/pkg/component/networking/nginxingress/nginxingress_test.go index fa8e98cec75..038ed3337ad 100644 --- a/pkg/component/networking/nginxingress/nginxingress_test.go +++ b/pkg/component/networking/nginxingress/nginxingress_test.go @@ -114,7 +114,6 @@ data: immutable: true kind: ConfigMap metadata: - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -128,7 +127,6 @@ metadata: clusterRoleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null labels: app: nginx-ingress name: gardener.cloud:seed:nginx-ingress @@ -207,7 +205,6 @@ rules: clusterRoleBindingYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - creationTimestamp: null labels: app: nginx-ingress name: gardener.cloud:seed:nginx-ingress @@ -223,7 +220,6 @@ subjects: roleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - creationTimestamp: null labels: app: nginx-ingress name: gardener.cloud:seed:nginx-ingress:role @@ -273,7 +269,6 @@ rules: roleBindingYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - creationTimestamp: null labels: app: nginx-ingress name: gardener.cloud:seed:nginx-ingress:role-binding @@ -294,7 +289,6 @@ metadata: networking.istio.io/exportTo: '*' networking.resources.gardener.cloud/namespace-selectors: '[{"matchLabels":{"gardener.cloud/role":"istio-ingress"}}]' some: value - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -320,7 +314,6 @@ status: serviceBackendYAML = `apiVersion: v1 kind: Service metadata: - creationTimestamp: null labels: app: nginx-ingress component: nginx-ingress-k8s-backend @@ -341,7 +334,6 @@ status: automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null labels: app: nginx-ingress name: nginx-ingress @@ -350,7 +342,6 @@ metadata: ingressClassYAML = `apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -361,7 +352,6 @@ spec: podDisruptionBudgetYAML = `apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -383,7 +373,6 @@ status: vpaYAML = `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - creationTimestamp: null name: nginx-ingress-controller namespace: ` + namespace + ` spec: @@ -403,7 +392,6 @@ status: {} deploymentBackendYAML = `apiVersion: apps/v1 kind: Deployment metadata: - creationTimestamp: null labels: app: nginx-ingress component: nginx-ingress-k8s-backend @@ -420,7 +408,6 @@ spec: strategy: {} template: metadata: - creationTimestamp: null labels: app: nginx-ingress component: nginx-ingress-k8s-backend @@ -462,7 +449,6 @@ kind: Deployment metadata: annotations: ` + references.AnnotationKey(references.KindConfigMap, configMapName) + `: ` + configMapName + ` - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -481,7 +467,6 @@ spec: metadata: annotations: ` + references.AnnotationKey(references.KindConfigMap, configMapName) + `: ` + configMapName + ` - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -572,7 +557,6 @@ status: {} destinationRuleYAML = `apiVersion: networking.istio.io/v1beta1 kind: DestinationRule metadata: - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -602,7 +586,6 @@ status: {} gatewayYAML = `apiVersion: networking.istio.io/v1beta1 kind: Gateway metadata: - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -627,7 +610,6 @@ status: {} return `apiVersion: networking.istio.io/v1beta1 kind: VirtualService metadata: - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -659,7 +641,6 @@ kind: Lease metadata: annotations: resources.gardener.cloud/ignore: "true" - creationTimestamp: null name: ingress-controller-seed-leader namespace: ` + namespace + ` spec: {} @@ -754,7 +735,6 @@ data: foo: bar kind: ConfigMap metadata: - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -766,7 +746,6 @@ metadata: ingressClassYAML = `apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -782,7 +761,6 @@ spec: automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null labels: app: nginx-ingress release: addons @@ -795,7 +773,6 @@ kind: NetworkPolicy metadata: annotations: gardener.cloud/description: Allows all egress and ingress traffic for the nginx-ingress controller. - creationTimestamp: null labels: origin: gardener name: gardener.cloud--allow-to-from-nginx @@ -818,7 +795,6 @@ spec: serviceBackendYAML = `apiVersion: v1 kind: Service metadata: - creationTimestamp: null labels: app: nginx-ingress component: nginx-ingress-k8s-backend @@ -841,7 +817,6 @@ status: clusterRoleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null labels: app: nginx-ingress release: addons @@ -924,7 +899,6 @@ kind: ClusterRoleBinding metadata: annotations: resources.gardener.cloud/delete-on-invalid-update: "true" - creationTimestamp: null labels: app: nginx-ingress release: addons @@ -944,7 +918,6 @@ kind: Service metadata: annotations: service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: '*' - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -975,7 +948,6 @@ status: deploymentBackendYAML = `apiVersion: apps/v1 kind: Deployment metadata: - creationTimestamp: null labels: app: nginx-ingress component: nginx-ingress-k8s-backend @@ -995,7 +967,6 @@ spec: strategy: {} template: metadata: - creationTimestamp: null labels: app: nginx-ingress component: nginx-ingress-k8s-backend @@ -1044,7 +1015,6 @@ status: {} out := `apiVersion: apps/v1 kind: Deployment metadata: - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -1066,7 +1036,6 @@ spec: metadata: annotations: checksum/config: ` + utils.ComputeChecksum(configMapData) + ` - creationTimestamp: null labels: app: nginx-ingress component: controller @@ -1162,7 +1131,6 @@ status: {} roleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - creationTimestamp: null labels: app: nginx-ingress release: addons @@ -1216,7 +1184,6 @@ kind: RoleBinding metadata: annotations: resources.gardener.cloud/delete-on-invalid-update: "true" - creationTimestamp: null labels: app: nginx-ingress release: addons @@ -1235,7 +1202,6 @@ subjects: vpaYAML = `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - creationTimestamp: null name: addons-nginx-ingress-controller namespace: kube-system spec: @@ -1258,7 +1224,6 @@ kind: Lease metadata: annotations: resources.gardener.cloud/ignore: "true" - creationTimestamp: null name: ingress-controller-leader namespace: kube-system spec: {} diff --git a/pkg/component/networking/nodelocaldns/nodelocaldns_test.go b/pkg/component/networking/nodelocaldns/nodelocaldns_test.go index 7c4af444a8e..2a0ff6eeed3 100644 --- a/pkg/component/networking/nodelocaldns/nodelocaldns_test.go +++ b/pkg/component/networking/nodelocaldns/nodelocaldns_test.go @@ -35,7 +35,6 @@ import ( . "github.com/gardener/gardener/pkg/component/networking/nodelocaldns" "github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references" "github.com/gardener/gardener/pkg/utils" - kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/managedresources" "github.com/gardener/gardener/pkg/utils/retry" retryfake "github.com/gardener/gardener/pkg/utils/retry/fake" @@ -270,17 +269,12 @@ var _ = Describe("NodeLocalDNS", func() { }, }, } - shootYAML, err := kubernetesutils.Serialize(shoot, kubernetes.GardenScheme) - Expect(err).NotTo(HaveOccurred()) - Expect(shootYAML).ToNot(BeEmpty()) - cluster = &extensionsv1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, }, Spec: extensionsv1alpha1.ClusterSpec{ Shoot: runtime.RawExtension{ - Raw: []byte(shootYAML), Object: shoot, }, Seed: runtime.RawExtension{ @@ -299,7 +293,6 @@ var _ = Describe("NodeLocalDNS", func() { automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null name: node-local-dns namespace: kube-system ` @@ -362,7 +355,6 @@ data: immutable: true kind: ConfigMap metadata: - creationTimestamp: null labels: k8s-app: node-local-dns resources.gardener.cloud/garbage-collectable-reference: "true" @@ -376,7 +368,6 @@ metadata: serviceYAML = `apiVersion: v1 kind: Service metadata: - creationTimestamp: null labels: k8s-app: kube-dns-upstream name: kube-dns-upstream @@ -650,7 +641,6 @@ status: vpaYAML = `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - creationTimestamp: null name: node-local-dns-worker-aaaa namespace: kube-system spec: diff --git a/pkg/component/nodemanagement/dependencywatchdog/access_test.go b/pkg/component/nodemanagement/dependencywatchdog/access_test.go index 29bcb819efc..121391a7726 100644 --- a/pkg/component/nodemanagement/dependencywatchdog/access_test.go +++ b/pkg/component/nodemanagement/dependencywatchdog/access_test.go @@ -45,7 +45,6 @@ var _ = Describe("Access", func() { roleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - creationTimestamp: null name: gardener.cloud:target:dependency-watchdog namespace: kube-node-lease rules: @@ -61,7 +60,6 @@ rules: roleBindingYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - creationTimestamp: null name: gardener.cloud:target:dependency-watchdog namespace: kube-node-lease roleRef: @@ -77,7 +75,6 @@ subjects: clusterRoleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: gardener.cloud:target:dependency-watchdog rules: - apiGroups: @@ -92,7 +89,6 @@ rules: clusterRoleBindingYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - creationTimestamp: null name: gardener.cloud:target:dependency-watchdog roleRef: apiGroup: rbac.authorization.k8s.io @@ -143,7 +139,6 @@ contexts: name: ` + namespace + ` current-context: ` + namespace + ` kind: Config -preferences: {} users: - name: ` + namespace + ` user: {} @@ -155,13 +150,13 @@ users: Name: "shoot-core-dependency-watchdog", Namespace: namespace, Labels: map[string]string{"origin": "gardener"}, - Annotations: map[string]string{"reference.resources.gardener.cloud/secret-dd60c006": "managedresource-shoot-core-dependency-watchdog-412f1efe"}, + Annotations: map[string]string{"reference.resources.gardener.cloud/secret-e3a35806": "managedresource-shoot-core-dependency-watchdog-3cb55785"}, ResourceVersion: "1", }, Spec: resourcesv1alpha1.ManagedResourceSpec{ SecretRefs: []corev1.LocalObjectReference{ { - Name: "managedresource-shoot-core-dependency-watchdog-412f1efe", + Name: "managedresource-shoot-core-dependency-watchdog-3cb55785", }, }, InjectLabels: map[string]string{"shoot.gardener.cloud/no-cleanup": "true"}, @@ -170,7 +165,7 @@ users: } expectedManagedResourceSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: "managedresource-shoot-core-dependency-watchdog-412f1efe", + Name: "managedresource-shoot-core-dependency-watchdog-3cb55785", Namespace: namespace, Labels: map[string]string{ "resources.gardener.cloud/garbage-collectable-reference": "true", @@ -197,7 +192,7 @@ users: Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledManagedResource), reconciledManagedResource)).To(Succeed()) Expect(reconciledManagedResource).To(DeepEqual(expectedManagedResource)) - reconciledManagedResourceSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "managedresource-shoot-core-dependency-watchdog-412f1efe", Namespace: namespace}} + reconciledManagedResourceSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "managedresource-shoot-core-dependency-watchdog-3cb55785", Namespace: namespace}} Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledManagedResourceSecret), reconciledManagedResourceSecret)).To(Succeed()) Expect(reconciledManagedResourceSecret.Type).To(Equal(corev1.SecretTypeOpaque)) Expect(reconciledManagedResourceSecret.Immutable).To(Equal(ptr.To(true))) diff --git a/pkg/component/nodemanagement/dependencywatchdog/bootstrap_test.go b/pkg/component/nodemanagement/dependencywatchdog/bootstrap_test.go index 15e5a9d0efb..92f1e4ffca1 100644 --- a/pkg/component/nodemanagement/dependencywatchdog/bootstrap_test.go +++ b/pkg/component/nodemanagement/dependencywatchdog/bootstrap_test.go @@ -59,7 +59,6 @@ var _ = Describe("DependencyWatchdog", func() { automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null name: ` + dwdName + ` namespace: ` + namespace + ` ` @@ -68,7 +67,6 @@ metadata: out := `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: gardener.cloud:` + dwdName + ` rules:` if role == RoleWeeder { @@ -140,7 +138,6 @@ rules:` clusterRoleBindingYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - creationTimestamp: null name: gardener.cloud:` + dwdName + ` roleRef: apiGroup: rbac.authorization.k8s.io @@ -156,7 +153,6 @@ subjects: out := `apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - creationTimestamp: null name: gardener.cloud:` + dwdName + ` namespace: ` + namespace + ` rules:` @@ -225,7 +221,6 @@ rules:` roleBindingYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - creationTimestamp: null name: gardener.cloud:` + dwdName + ` namespace: ` + namespace + ` roleRef: @@ -259,7 +254,6 @@ data: out += `immutable: true kind: ConfigMap metadata: - creationTimestamp: null labels: app: ` + dwdName + ` resources.gardener.cloud/garbage-collectable-reference: "true" @@ -276,7 +270,6 @@ kind: Deployment metadata: annotations: ` + references.AnnotationKey(references.KindConfigMap, configMapName) + `: ` + configMapName + ` - creationTimestamp: null labels: app: ` + dwdName + ` high-availability-config.resources.gardener.cloud/type: controller @@ -293,7 +286,6 @@ spec: metadata: annotations: ` + references.AnnotationKey(references.KindConfigMap, configMapName) + `: ` + configMapName + ` - creationTimestamp: null labels: app: ` + dwdName @@ -375,7 +367,6 @@ status: {} out := `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - creationTimestamp: null name: ` + dwdName + ` namespace: ` + namespace + ` spec: @@ -409,7 +400,6 @@ status: {} podDisruptionYAML = `apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - creationTimestamp: null labels: app: ` + dwdName + ` name: ` + dwdName + ` diff --git a/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager_test.go b/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager_test.go index acbfe7041be..54eb7789f81 100644 --- a/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager_test.go +++ b/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager_test.go @@ -406,7 +406,6 @@ var _ = Describe("MachineControllerManager", func() { clusterRoleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: gardener.cloud:target:machine-controller-manager rules: - apiGroups: @@ -485,7 +484,6 @@ rules: clusterRoleBindingYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - creationTimestamp: null name: gardener.cloud:target:machine-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io @@ -500,7 +498,6 @@ subjects: roleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - creationTimestamp: null name: gardener.cloud:target:machine-controller-manager namespace: kube-system rules: @@ -518,7 +515,6 @@ rules: roleBindingYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - creationTimestamp: null name: gardener.cloud:target:machine-controller-manager namespace: kube-system roleRef: diff --git a/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml b/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml index 925b4e4a82b..7329a338f25 100644 --- a/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml +++ b/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: machineclasses.machine.sapcloud.io spec: group: machine.sapcloud.io diff --git a/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml b/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml index 00bcea01a56..450e8bf6284 100644 --- a/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml +++ b/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: machinedeployments.machine.sapcloud.io spec: group: machine.sapcloud.io @@ -399,9 +399,8 @@ spec: to a node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at + which the taint was added. format: date-time type: string value: diff --git a/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machines.yaml b/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machines.yaml index f0c7c59bacc..7983b7f29f9 100644 --- a/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machines.yaml +++ b/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machines.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: machines.machine.sapcloud.io spec: group: machine.sapcloud.io @@ -196,9 +196,8 @@ spec: a node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which + the taint was added. format: date-time type: string value: diff --git a/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinesets.yaml b/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinesets.yaml index 5512b600e36..d9933286359 100644 --- a/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinesets.yaml +++ b/pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinesets.yaml @@ -5,7 +5,7 @@ metadata: labels: gardener.cloud/deletion-protected: "true" annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: machinesets.machine.sapcloud.io spec: group: machine.sapcloud.io @@ -281,9 +281,8 @@ spec: to a node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at + which the taint was added. format: date-time type: string value: diff --git a/pkg/component/nodemanagement/nodeproblemdetector/node_problem_detector_test.go b/pkg/component/nodemanagement/nodeproblemdetector/node_problem_detector_test.go index a1824680326..11cb780a7bd 100644 --- a/pkg/component/nodemanagement/nodeproblemdetector/node_problem_detector_test.go +++ b/pkg/component/nodemanagement/nodeproblemdetector/node_problem_detector_test.go @@ -78,7 +78,6 @@ var _ = Describe("NodeProblemDetector", func() { automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null labels: app.kubernetes.io/instance: shoot-core app.kubernetes.io/name: node-problem-detector @@ -88,7 +87,6 @@ metadata: clusterRoleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null labels: app.kubernetes.io/instance: shoot-core app.kubernetes.io/name: node-problem-detector @@ -120,7 +118,6 @@ kind: ClusterRoleBinding metadata: annotations: resources.gardener.cloud/delete-on-invalid-update: "true" - creationTimestamp: null labels: app.kubernetes.io/instance: shoot-core app.kubernetes.io/name: node-problem-detector @@ -138,7 +135,6 @@ subjects: serviceYAML = `apiVersion: v1 kind: Service metadata: - creationTimestamp: null labels: app: node-problem-detector app.kubernetes.io/instance: shoot-core @@ -163,7 +159,6 @@ status: out := `apiVersion: apps/v1 kind: DaemonSet metadata: - creationTimestamp: null labels: app.kubernetes.io/instance: shoot-core app.kubernetes.io/name: node-problem-detector @@ -180,7 +175,6 @@ spec: app.kubernetes.io/name: node-problem-detector template: metadata: - creationTimestamp: null labels: app: node-problem-detector app.kubernetes.io/instance: shoot-core @@ -265,7 +259,6 @@ status: vpaYAML = `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - creationTimestamp: null name: node-problem-detector namespace: kube-system spec: diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfilters.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfilters.yaml index a5789bad004..624039566a0 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfilters.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfilters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusterfilters.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 8506b8bcccc..2aaeb309652 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusterfluentbitconfigs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterinputs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterinputs.yaml index 7931598ceb0..f3813ba4343 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterinputs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterinputs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusterinputs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clustermultilineparsers.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clustermultilineparsers.yaml index e6fe12dea7d..e56ff886a2d 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clustermultilineparsers.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clustermultilineparsers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clustermultilineparsers.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusteroutputs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusteroutputs.yaml index 79e5b28708d..f53fd251fb4 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusteroutputs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusteroutputs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusteroutputs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterparsers.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterparsers.yaml index e36b427f0b9..0e2983036e5 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterparsers.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterparsers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: clusterparsers.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_collectors.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_collectors.yaml index 0f7bc38be2f..c99f8c00ad4 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_collectors.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_collectors.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: collectors.fluentbit.fluent.io spec: group: fluentbit.fluent.io @@ -610,8 +610,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -1252,15 +1252,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -1420,13 +1418,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -1514,7 +1510,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2545,15 +2541,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -2735,12 +2729,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -2819,7 +2811,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -3239,6 +3231,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -3373,7 +3470,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_filters.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_filters.yaml index fcb3d3d56c4..55135835718 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_filters.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_filters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: filters.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbitconfigs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbitconfigs.yaml index 55ec28f4b8d..aafe67c640f 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbitconfigs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbitconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: fluentbitconfigs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbits.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbits.yaml index 506efe9bfa1..99e95fff5a2 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbits.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbits.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: fluentbits.fluentbit.fluent.io spec: group: fluentbit.fluent.io @@ -610,8 +610,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -1183,7 +1183,9 @@ spec: a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1241,6 +1243,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1391,8 +1430,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1450,6 +1490,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1510,8 +1587,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -1538,8 +1615,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -2210,7 +2288,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2264,10 +2342,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -2279,6 +2357,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -3707,15 +3838,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -3896,12 +4025,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details Glusterfs + topology. type: string path: description: |- @@ -3980,7 +4107,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -4392,6 +4519,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be addressed + to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -4526,7 +4758,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- @@ -5019,7 +5250,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -6055,15 +6286,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6245,12 +6474,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -6329,7 +6556,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -6749,6 +6976,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -6883,7 +7215,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_multilineparsers.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_multilineparsers.yaml index ec684d1414c..be64ff720a4 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_multilineparsers.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_multilineparsers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: multilineparsers.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_outputs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_outputs.yaml index 830a88f558e..7da1e0b4203 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_outputs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_outputs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: outputs.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_parsers.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_parsers.yaml index 9d6c5136586..6bfe5e95284 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_parsers.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_parsers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: parsers.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/pkg/component/observability/monitoring/blackboxexporter/blackboxexporter_test.go b/pkg/component/observability/monitoring/blackboxexporter/blackboxexporter_test.go index 04ea3e23c31..2a72a3bfc03 100644 --- a/pkg/component/observability/monitoring/blackboxexporter/blackboxexporter_test.go +++ b/pkg/component/observability/monitoring/blackboxexporter/blackboxexporter_test.go @@ -117,7 +117,6 @@ var _ = Describe("BlackboxExporter", func() { automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null labels: app: blackbox-exporter gardener.cloud/role: monitoring @@ -134,7 +133,6 @@ data: immutable: true kind: ConfigMap metadata: - creationTimestamp: null labels: app: prometheus resources.gardener.cloud/garbage-collectable-reference: "true" @@ -146,7 +144,6 @@ metadata: pdbYAML = `apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - creationTimestamp: null labels: app: blackbox-exporter gardener.cloud/role: monitoring @@ -171,7 +168,6 @@ kind: Deployment metadata: annotations: ` + references.AnnotationKey(references.KindConfigMap, configMapName) + `: ` + configMapName + ` - creationTimestamp: null labels: app: blackbox-exporter gardener.cloud/role: monitoring @@ -190,7 +186,6 @@ spec: metadata: annotations: ` + references.AnnotationKey(references.KindConfigMap, configMapName) + `: ` + configMapName + ` - creationTimestamp: null labels: app: blackbox-exporter bar: foo @@ -309,7 +304,6 @@ metadata:` } out += ` - creationTimestamp: null labels: app: blackbox-exporter name: blackbox-exporter @@ -332,7 +326,6 @@ status: vpaYAML = `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - creationTimestamp: null name: blackbox-exporter namespace: ` + resourcesNamespace + ` spec: @@ -438,7 +431,6 @@ status: {} scrapeConfigYAML = `apiVersion: monitoring.coreos.com/v1alpha1 kind: ScrapeConfig metadata: - creationTimestamp: null name: foo namespace: ` + namespace + ` spec: {} @@ -447,7 +439,6 @@ spec: {} prometheusRuleYAML = `apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: - creationTimestamp: null name: bar namespace: ` + namespace + ` spec: {} diff --git a/pkg/component/observability/monitoring/metricsserver/metricsserver_test.go b/pkg/component/observability/monitoring/metricsserver/metricsserver_test.go index 1cc5642f24c..e8e0d0f7178 100644 --- a/pkg/component/observability/monitoring/metricsserver/metricsserver_test.go +++ b/pkg/component/observability/monitoring/metricsserver/metricsserver_test.go @@ -44,7 +44,6 @@ var _ = Describe("MetricsServer", func() { serviceYAML = `apiVersion: v1 kind: Service metadata: - creationTimestamp: null labels: kubernetes.io/name: metrics-server name: metrics-server @@ -62,7 +61,6 @@ status: apiServiceYAML = `apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: - creationTimestamp: null name: v1beta1.metrics.k8s.io spec: group: metrics.k8s.io @@ -77,7 +75,6 @@ status: {} vpaYAML = `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - creationTimestamp: null name: metrics-server namespace: kube-system spec: @@ -98,7 +95,6 @@ status: {} clusterRoleYAML = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: system:metrics-server rules: - apiGroups: @@ -119,7 +115,6 @@ kind: ClusterRoleBinding metadata: annotations: resources.gardener.cloud/delete-on-invalid-update: "true" - creationTimestamp: null name: system:metrics-server roleRef: apiGroup: rbac.authorization.k8s.io @@ -135,7 +130,6 @@ kind: ClusterRoleBinding metadata: annotations: resources.gardener.cloud/delete-on-invalid-update: "true" - creationTimestamp: null name: metrics-server:system:auth-delegator roleRef: apiGroup: rbac.authorization.k8s.io @@ -151,7 +145,6 @@ kind: RoleBinding metadata: annotations: resources.gardener.cloud/delete-on-invalid-update: "true" - creationTimestamp: null name: metrics-server-auth-reader namespace: kube-system roleRef: @@ -167,7 +160,6 @@ subjects: automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null name: metrics-server namespace: kube-system ` @@ -183,7 +175,6 @@ kind: Deployment metadata: annotations: ` + references.AnnotationKey(references.KindSecret, secretName) + `: ` + secretName + ` - creationTimestamp: null labels: gardener.cloud/role: system-component high-availability-config.resources.gardener.cloud/type: server @@ -204,7 +195,6 @@ spec: metadata: annotations: ` + references.AnnotationKey(references.KindSecret, secretName) + `: ` + secretName + ` - creationTimestamp: null labels: gardener.cloud/role: system-component k8s-app: metrics-server @@ -288,7 +278,6 @@ status: {} pdbYAML = `apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - creationTimestamp: null labels: k8s-app: metrics-server name: metrics-server diff --git a/pkg/component/observability/monitoring/nodeexporter/nodeexporter_test.go b/pkg/component/observability/monitoring/nodeexporter/nodeexporter_test.go index e6dec3899b1..681ab09f530 100644 --- a/pkg/component/observability/monitoring/nodeexporter/nodeexporter_test.go +++ b/pkg/component/observability/monitoring/nodeexporter/nodeexporter_test.go @@ -270,7 +270,6 @@ var _ = Describe("NodeExporter", func() { automountServiceAccountToken: false kind: ServiceAccount metadata: - creationTimestamp: null labels: component: node-exporter name: node-exporter @@ -279,7 +278,6 @@ metadata: serviceYAML = `apiVersion: v1 kind: Service metadata: - creationTimestamp: null labels: component: node-exporter name: node-exporter @@ -300,7 +298,6 @@ status: daemonSetYAML = `apiVersion: apps/v1 kind: DaemonSet metadata: - creationTimestamp: null labels: component: node-exporter gardener.cloud/role: monitoring @@ -314,7 +311,6 @@ spec: component: node-exporter template: metadata: - creationTimestamp: null labels: component: node-exporter gardener.cloud/role: monitoring @@ -411,7 +407,6 @@ status: vpaYAML = `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - creationTimestamp: null name: node-exporter namespace: kube-system spec: diff --git a/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_perses.yaml b/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_perses.yaml index 78ded4e5638..3a9e93b42ce 100644 --- a/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_perses.yaml +++ b/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_perses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: perses.perses.dev spec: group: perses.dev @@ -610,8 +610,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm diff --git a/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_persesdashboards.yaml b/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_persesdashboards.yaml index e88d62e47f1..301ea4eb0e6 100644 --- a/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_persesdashboards.yaml +++ b/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_persesdashboards.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: persesdashboards.perses.dev spec: group: perses.dev diff --git a/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_persesdatasources.yaml b/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_persesdatasources.yaml index 07c4e8a0fef..4e4838af27b 100644 --- a/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_persesdatasources.yaml +++ b/pkg/component/observability/monitoring/persesoperator/templates/crd-perses.dev_persesdatasources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: persesdatasources.perses.dev spec: group: perses.dev diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagerconfigs.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagerconfigs.yaml index 0298bac64ca..88429306f60 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagerconfigs.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagerconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: alertmanagerconfigs.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagers.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagers.yaml index c1319eb26a8..96fad4d0d5d 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagers.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: alertmanagers.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -674,8 +674,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -2887,8 +2887,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2946,6 +2947,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3006,8 +3044,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -3034,8 +3072,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -3706,7 +3745,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -3760,10 +3799,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -3775,6 +3814,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -4476,8 +4568,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4535,6 +4628,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4595,8 +4725,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -4623,8 +4753,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -5295,7 +5426,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -5349,10 +5480,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -5364,6 +5495,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -6009,7 +6193,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -6597,15 +6781,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6853,15 +7035,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -7023,13 +7203,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -8034,15 +8212,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -8224,12 +8400,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -8308,7 +8482,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -8728,6 +8902,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -8862,7 +9141,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_podmonitors.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_podmonitors.yaml index 5aac7b91cce..c888967e7c4 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_podmonitors.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_podmonitors.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: podmonitors.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_probes.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_probes.yaml index 0cc48a5bab3..4ac47e50aa1 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_probes.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_probes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: probes.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusagents.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusagents.yaml index 5d17a634635..74e741011b3 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusagents.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusagents.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: prometheusagents.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -705,8 +705,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -1510,8 +1510,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1569,6 +1570,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1629,8 +1667,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -1657,8 +1695,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -2329,7 +2368,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2383,10 +2422,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -2398,6 +2437,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -3337,8 +3429,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -3396,6 +3489,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3456,8 +3586,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -3484,8 +3614,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -4156,7 +4287,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -4210,10 +4341,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -4225,6 +4356,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -6354,7 +6538,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -7736,15 +7920,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -7992,15 +8174,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -8162,13 +8342,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -9432,15 +9610,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -9622,12 +9798,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -9706,7 +9880,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -10126,6 +10300,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -10260,7 +10539,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheuses.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheuses.yaml index 00768cd7ecc..c5420605b20 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheuses.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheuses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: prometheuses.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -783,8 +783,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -2248,8 +2248,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2307,6 +2308,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2367,8 +2405,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -2395,8 +2433,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -3067,7 +3106,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -3121,10 +3160,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -3136,6 +3175,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -4117,8 +4209,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4176,6 +4269,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4236,8 +4366,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -4264,8 +4394,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -4936,7 +5067,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -4990,10 +5121,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -5005,6 +5136,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -7920,7 +8104,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -9485,15 +9669,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -9741,15 +9923,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -9911,13 +10091,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -10304,7 +10482,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -11698,15 +11876,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -11888,12 +12064,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -11972,7 +12146,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -12392,6 +12566,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -12526,7 +12805,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusrules.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusrules.yaml index 6e41ee48a8d..f4800ee9ed9 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusrules.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusrules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: prometheusrules.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_scrapeconfigs.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_scrapeconfigs.yaml index e9157384362..b6a61bb8bad 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_scrapeconfigs.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_scrapeconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: scrapeconfigs.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_servicemonitors.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_servicemonitors.yaml index 0d868a6074e..5913993c581 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_servicemonitors.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_servicemonitors.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: servicemonitors.monitoring.coreos.com spec: group: monitoring.coreos.com diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_thanosrulers.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_thanosrulers.yaml index be079491bcd..2285730da9d 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_thanosrulers.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_thanosrulers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: thanosrulers.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -671,8 +671,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -1165,8 +1165,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1224,6 +1225,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1284,8 +1322,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -1312,8 +1350,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -1984,7 +2023,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2038,10 +2077,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -2053,6 +2092,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -2974,8 +3066,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -3033,6 +3126,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3093,8 +3223,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -3121,8 +3251,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -3793,7 +3924,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -3847,10 +3978,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -3862,6 +3993,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -5680,7 +5864,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -6374,15 +6558,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6630,15 +6812,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6800,13 +6980,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -7850,15 +8028,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -8040,12 +8216,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -8124,7 +8298,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -8544,6 +8718,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -8678,7 +8957,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_instrumentations.yaml b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_instrumentations.yaml index 53925b8a724..4ce815cc83d 100644 --- a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_instrumentations.yaml +++ b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_instrumentations.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: instrumentations.opentelemetry.io spec: group: opentelemetry.io @@ -69,8 +69,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -128,6 +129,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -197,8 +235,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -256,6 +295,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -321,7 +397,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -568,15 +644,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -626,8 +700,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -685,6 +760,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -750,7 +862,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -993,15 +1105,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -1036,7 +1146,9 @@ spec: a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1094,6 +1206,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1207,8 +1356,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1266,6 +1416,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1331,7 +1518,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -1574,15 +1761,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -1626,8 +1811,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1685,6 +1871,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1770,7 +1993,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2013,15 +2236,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -2059,8 +2280,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2118,6 +2340,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2187,8 +2446,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2246,6 +2506,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2311,7 +2608,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2554,15 +2851,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -2600,8 +2895,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2659,6 +2955,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2724,7 +3057,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2967,15 +3300,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -3031,8 +3362,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -3090,6 +3422,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3155,7 +3524,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -3398,15 +3767,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- diff --git a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opampbridges.yaml b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opampbridges.yaml index b05dde2b7c2..0e570643ee7 100644 --- a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opampbridges.yaml +++ b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opampbridges.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: opampbridges.opentelemetry.io spec: group: opentelemetry.io @@ -619,8 +619,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -999,7 +999,9 @@ spec: a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1057,6 +1059,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1135,8 +1174,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each environment - variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -1570,7 +1610,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2781,15 +2821,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -2971,12 +3009,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -3055,7 +3091,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -3475,6 +3511,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -3609,7 +3750,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opentelemetrycollectors.yaml b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opentelemetrycollectors.yaml index 96453700115..1e4a435e3ab 100644 --- a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opentelemetrycollectors.yaml +++ b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opentelemetrycollectors.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: opentelemetrycollectors.opentelemetry.io spec: group: opentelemetry.io @@ -118,8 +118,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -177,6 +178,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -237,8 +275,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -265,8 +303,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -937,7 +976,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -991,10 +1030,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -1006,6 +1045,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -2073,8 +2165,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -2850,7 +2942,7 @@ spec: pod is available (Ready for at least minReadySeconds) the old DaemonSet pod on that node is marked deleted. If the old pod becomes unavailable for any reason (Ready transitions to false, is evicted, or is drained) an updated - pod is immediatedly created on that node without considering surge limits. + pod is immediately created on that node without considering surge limits. Allowing surge implies the possibility that the resources consumed by the daemonset on any given node can double if the readiness check fails, and so resource intensive daemonsets should take into account that they may @@ -2942,7 +3034,9 @@ spec: a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -3000,6 +3094,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3078,8 +3209,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each environment - variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -3242,8 +3374,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -3301,6 +3434,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3361,8 +3531,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -3389,8 +3559,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -4061,7 +4232,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -4115,10 +4286,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -4130,6 +4301,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -5470,7 +5694,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -6364,8 +6588,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -6749,8 +6973,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6808,6 +7033,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -7418,7 +7680,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -8333,15 +8595,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -8504,13 +8764,11 @@ spec: description: |- currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). type: string modifyVolumeStatus: description: |- ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. - This is a beta field and requires enabling VolumeAttributesClass feature (off by default). properties: status: description: "status is the status of the ControllerModifyVolume @@ -9281,15 +9539,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -9471,12 +9727,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -9555,7 +9809,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -9975,6 +10229,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -10109,7 +10468,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_targetallocators.yaml b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_targetallocators.yaml index 67af03ca807..ef04fe2003e 100644 --- a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_targetallocators.yaml +++ b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_targetallocators.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: targetallocators.opentelemetry.io spec: group: opentelemetry.io @@ -103,8 +103,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -162,6 +163,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -222,8 +260,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -250,8 +288,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -922,7 +961,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -976,10 +1015,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -991,6 +1030,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -2058,8 +2150,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -2436,7 +2528,9 @@ spec: a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2494,6 +2588,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2572,8 +2703,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each environment - variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -2669,8 +2801,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2728,6 +2861,43 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2788,8 +2958,8 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. @@ -2816,8 +2986,9 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: Optional text to prepend to the name of each - environment variable. Must be a C_IDENTIFIER. + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: description: The Secret to select from @@ -3488,7 +3659,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -3542,10 +3713,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, + This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -3557,6 +3728,59 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -4995,7 +5219,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -6236,15 +6460,13 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6426,12 +6648,10 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -6510,7 +6730,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support iSCSI @@ -6930,6 +7150,111 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be + addressed to this signer. + type: string + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -7064,7 +7389,6 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- diff --git a/pkg/component/observability/plutono/plutono_test.go b/pkg/component/observability/plutono/plutono_test.go index 11086be19c4..91ce11abd75 100644 --- a/pkg/component/observability/plutono/plutono_test.go +++ b/pkg/component/observability/plutono/plutono_test.go @@ -166,7 +166,6 @@ data: immutable: true kind: ConfigMap metadata: - creationTimestamp: null labels: component: plutono resources.gardener.cloud/garbage-collectable-reference: "true" @@ -254,7 +253,6 @@ data: ` + configMapData + ` kind: ConfigMap metadata: - creationTimestamp: null labels: component: plutono datasource.monitoring.gardener.cloud/` + clusterLabelKey(values) + `: "true" @@ -491,7 +489,6 @@ metadata: out := `apiVersion: v1 kind: Service metadata: - creationTimestamp: null labels: component: plutono ` @@ -542,7 +539,6 @@ metadata: location /api/admin/ { return 403; } - creationTimestamp: null ` if values.ClusterType == comp.ClusterTypeShoot { out += ` labels: diff --git a/pkg/component/seed/system/system_test.go b/pkg/component/seed/system/system_test.go index 4eb02dc8af1..b74bc70ccc3 100644 --- a/pkg/component/seed/system/system_test.go +++ b/pkg/component/seed/system/system_test.go @@ -50,7 +50,6 @@ kind: Deployment metadata: annotations: resources.gardener.cloud/skip-health-check: "true" - creationTimestamp: null labels: app: kubernetes role: reserve-excess-capacity @@ -66,7 +65,6 @@ spec: strategy: {} template: metadata: - creationTimestamp: null labels: app: kubernetes role: reserve-excess-capacity @@ -93,7 +91,6 @@ kind: Deployment metadata: annotations: resources.gardener.cloud/skip-health-check: "true" - creationTimestamp: null labels: app: kubernetes role: reserve-excess-capacity @@ -109,7 +106,6 @@ spec: strategy: {} template: metadata: - creationTimestamp: null labels: app: kubernetes role: reserve-excess-capacity @@ -375,7 +371,6 @@ func expectedPriorityClasses() []string { description: `+pc.description+` kind: PriorityClass metadata: - creationTimestamp: null name: `+pc.name+` value: `+strconv.FormatInt(int64(pc.value), 10)+` `) diff --git a/pkg/component/shared/kubeapiserver_test.go b/pkg/component/shared/kubeapiserver_test.go index 8d12882fddb..f48301b902e 100644 --- a/pkg/component/shared/kubeapiserver_test.go +++ b/pkg/component/shared/kubeapiserver_test.go @@ -929,7 +929,9 @@ authorizers: Kubeconfig: []byte("webhook-kubeconfig"), WebhookConfiguration: apiserverv1beta1.WebhookConfiguration{ AuthorizedTTL: metav1.Duration{Duration: 5 * time.Minute}, + CacheAuthorizedRequests: ptr.To(true), UnauthorizedTTL: metav1.Duration{Duration: 30 * time.Second}, + CacheUnauthorizedRequests: ptr.To(true), Timeout: metav1.Duration{Duration: 3 * time.Second}, SubjectAccessReviewVersion: "v1", MatchConditionSubjectAccessReviewVersion: "v1", diff --git a/pkg/controller/networkpolicy/add.go b/pkg/controller/networkpolicy/add.go index 08a94b64467..58a3b8d5b37 100644 --- a/pkg/controller/networkpolicy/add.go +++ b/pkg/controller/networkpolicy/add.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/gardener/gardener/pkg/controller/networkpolicy/hostnameresolver" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -69,6 +70,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, runtimeCluster cluster.Cl Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource(source.Kind[client.Object]( runtimeCluster.GetCache(), diff --git a/pkg/controller/networkpolicy/reconciler.go b/pkg/controller/networkpolicy/reconciler.go index e3fc0b61edf..37bf0e9145a 100644 --- a/pkg/controller/networkpolicy/reconciler.go +++ b/pkg/controller/networkpolicy/reconciler.go @@ -86,9 +86,6 @@ func (r RuntimeNetworkConfig) getBlockedNetworkPeers(ipFamily gardencorev1beta1. func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - namespace := &corev1.Namespace{} if err := r.RuntimeClient.Get(ctx, request.NamespacedName, namespace); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controller/reference/add.go b/pkg/controller/reference/add.go index a1b9739dd65..19e3fbb8ab8 100644 --- a/pkg/controller/reference/add.go +++ b/pkg/controller/reference/add.go @@ -14,6 +14,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" ) // controllerNameSuffix is the suffix added to the controller name. @@ -31,6 +32,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, name string) error { For(r.NewObjectFunc(), builder.WithPredicates(r.Predicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controller/reference/reconciler.go b/pkg/controller/reference/reconciler.go index ac3a8f46ad5..2f9c4f76761 100644 --- a/pkg/controller/reference/reconciler.go +++ b/pkg/controller/reference/reconciler.go @@ -46,9 +46,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - obj := r.NewObjectFunc() if err := r.Client.Get(ctx, request.NamespacedName, obj); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controller/tokenrequestor/add.go b/pkg/controller/tokenrequestor/add.go index 37613f54101..ab3b41fb893 100644 --- a/pkg/controller/tokenrequestor/add.go +++ b/pkg/controller/tokenrequestor/add.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of the controller. @@ -46,6 +47,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, sourceCluster, targetClus For(&corev1.Secret{}, builder.WithPredicates(r.SecretPredicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: r.ConcurrentSyncs, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controller/tokenrequestor/reconciler.go b/pkg/controller/tokenrequestor/reconciler.go index 51d7d5ecb1d..e584645f69d 100644 --- a/pkg/controller/tokenrequestor/reconciler.go +++ b/pkg/controller/tokenrequestor/reconciler.go @@ -28,7 +28,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" ) const ( @@ -53,11 +52,8 @@ type Reconciler struct { } // Reconcile requests and populates tokens. -func (r *Reconciler) Reconcile(reconcileCtx context.Context, req reconcile.Request) (reconcile.Result, error) { - log := logf.FromContext(reconcileCtx) - - ctx, cancel := controllerutils.GetMainReconciliationContext(reconcileCtx, controllerutils.DefaultReconciliationTimeout) - defer cancel() +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + log := logf.FromContext(ctx) secret := &corev1.Secret{} if err := r.SourceClient.Get(ctx, req.NamespacedName, secret); err != nil { diff --git a/pkg/controller/tokenrequestor/reconciler_test.go b/pkg/controller/tokenrequestor/reconciler_test.go index c36629050ea..f9b87e2bd8a 100644 --- a/pkg/controller/tokenrequestor/reconciler_test.go +++ b/pkg/controller/tokenrequestor/reconciler_test.go @@ -545,7 +545,6 @@ contexts: name: shoot--foo--bar current-context: shoot--foo--bar kind: Config -preferences: {} users: - name: shoot--foo--bar-token user: diff --git a/pkg/controller/vpaevictionrequirements/add.go b/pkg/controller/vpaevictionrequirements/add.go index 09bf9e9851e..5f91ab407c5 100644 --- a/pkg/controller/vpaevictionrequirements/add.go +++ b/pkg/controller/vpaevictionrequirements/add.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -48,6 +49,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, seedCluster cluster.Clust Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource( source.Kind[client.Object](seedCluster.GetCache(), diff --git a/pkg/controller/vpaevictionrequirements/reconciler.go b/pkg/controller/vpaevictionrequirements/reconciler.go index e516d37ee2a..83504586201 100644 --- a/pkg/controller/vpaevictionrequirements/reconciler.go +++ b/pkg/controller/vpaevictionrequirements/reconciler.go @@ -21,7 +21,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/timewindow" ) @@ -40,8 +39,6 @@ type Reconciler struct { // Reconcile implements the reconciliation logic for adding/removing EvictionRequirements to VPA objects. func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() vpa := &vpaautoscalingv1.VerticalPodAutoscaler{} if err := r.SeedClient.Get(ctx, request.NamespacedName, vpa); err != nil { diff --git a/pkg/controllermanager/apis/config/v1alpha1/validation/validation_test.go b/pkg/controllermanager/apis/config/v1alpha1/validation/validation_test.go index 8d286b10954..ec19757b11c 100644 --- a/pkg/controllermanager/apis/config/v1alpha1/validation/validation_test.go +++ b/pkg/controllermanager/apis/config/v1alpha1/validation/validation_test.go @@ -88,7 +88,7 @@ var _ = Describe("#ValidateControllerManagerConfiguration", func() { Expect(ValidateControllerManagerConfiguration(conf)).To(ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), + "Type": Equal(field.ErrorTypeRequired), "Field": Equal("leaderElection.resourceNamespace"), })), )) diff --git a/pkg/controllermanager/controller/bastion/add.go b/pkg/controllermanager/controller/bastion/add.go index d235e0816c7..07752758980 100644 --- a/pkg/controllermanager/controller/bastion/add.go +++ b/pkg/controllermanager/controller/bastion/add.go @@ -23,6 +23,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" "github.com/gardener/gardener/pkg/apis/operations" operationsv1alpha1 "github.com/gardener/gardener/pkg/apis/operations/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/controllerutils/mapper" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -45,6 +46,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&operationsv1alpha1.Bastion{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Watches( &gardencorev1beta1.Shoot{}, diff --git a/pkg/controllermanager/controller/bastion/reconciler.go b/pkg/controllermanager/controller/bastion/reconciler.go index 44bae20c830..ac2dab4977b 100644 --- a/pkg/controllermanager/controller/bastion/reconciler.go +++ b/pkg/controllermanager/controller/bastion/reconciler.go @@ -18,7 +18,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" operationsv1alpha1 "github.com/gardener/gardener/pkg/apis/operations/v1alpha1" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" ) // Reconciler reconciles Bastions. @@ -32,9 +31,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - bastion := &operationsv1alpha1.Bastion{} if err := r.Client.Get(ctx, request.NamespacedName, bastion); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/certificatesigningrequest/add.go b/pkg/controllermanager/controller/certificatesigningrequest/add.go index 4a9ce5215b3..9ce3009a147 100644 --- a/pkg/controllermanager/controller/certificatesigningrequest/add.go +++ b/pkg/controllermanager/controller/certificatesigningrequest/add.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -29,6 +30,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&certificatesv1.CertificateSigningRequest{}, builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create, predicateutils.Update))). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/certificatesigningrequest/reconciler.go b/pkg/controllermanager/controller/certificatesigningrequest/reconciler.go index b6c63998e7e..686dc0ddead 100644 --- a/pkg/controllermanager/controller/certificatesigningrequest/reconciler.go +++ b/pkg/controllermanager/controller/certificatesigningrequest/reconciler.go @@ -43,7 +43,6 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/client/kubernetes" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" gardenletutils "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" @@ -65,9 +64,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( finalState string ) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - csr := &certificatesv1.CertificateSigningRequest{} if err := r.Client.Get(ctx, request.NamespacedName, csr); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/cloudprofile/add.go b/pkg/controllermanager/controller/cloudprofile/add.go index ba67fcd462f..05e4fd51b78 100644 --- a/pkg/controllermanager/controller/cloudprofile/add.go +++ b/pkg/controllermanager/controller/cloudprofile/add.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -31,6 +32,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.CloudProfile{}). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/cloudprofile/reconciler.go b/pkg/controllermanager/controller/cloudprofile/reconciler.go index 51ca7507265..7b3fe07d4ef 100644 --- a/pkg/controllermanager/controller/cloudprofile/reconciler.go +++ b/pkg/controllermanager/controller/cloudprofile/reconciler.go @@ -35,9 +35,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - cloudProfile := &gardencorev1beta1.CloudProfile{} if err := r.Client.Get(ctx, request.NamespacedName, cloudProfile); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/controllerdeployment/add.go b/pkg/controllermanager/controller/controllerdeployment/add.go index 409fb808214..340698681a3 100644 --- a/pkg/controllermanager/controller/controllerdeployment/add.go +++ b/pkg/controllermanager/controller/controllerdeployment/add.go @@ -12,6 +12,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" gardencorev1 "github.com/gardener/gardener/pkg/apis/core/v1" + "github.com/gardener/gardener/pkg/controllerutils" ) const ( @@ -34,6 +35,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), RateLimiter: r.RateLimiter, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/controllerdeployment/reconciler.go b/pkg/controllermanager/controller/controllerdeployment/reconciler.go index 6c44a0ff724..73cee294cab 100644 --- a/pkg/controllermanager/controller/controllerdeployment/reconciler.go +++ b/pkg/controllermanager/controller/controllerdeployment/reconciler.go @@ -35,9 +35,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - controllerDeployment := &gardencorev1.ControllerDeployment{} if err := r.Client.Get(ctx, client.ObjectKey{Name: req.Name}, controllerDeployment); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/controllerregistration/controllerregistrationfinalizer/add.go b/pkg/controllermanager/controller/controllerregistration/controllerregistrationfinalizer/add.go index 134c1416cf1..a9528181c70 100644 --- a/pkg/controllermanager/controller/controllerregistration/controllerregistrationfinalizer/add.go +++ b/pkg/controllermanager/controller/controllerregistration/controllerregistrationfinalizer/add.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -28,6 +29,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.ControllerRegistration{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). WithOptions(controller.Options{ MaxConcurrentReconciles: 5, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/controllerregistration/controllerregistrationfinalizer/reconciler.go b/pkg/controllermanager/controller/controllerregistration/controllerregistrationfinalizer/reconciler.go index ecba930f692..fcf9956c60d 100644 --- a/pkg/controllermanager/controller/controllerregistration/controllerregistrationfinalizer/reconciler.go +++ b/pkg/controllermanager/controller/controllerregistration/controllerregistrationfinalizer/reconciler.go @@ -34,9 +34,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - controllerRegistration := &gardencorev1beta1.ControllerRegistration{} if err := r.Client.Get(ctx, request.NamespacedName, controllerRegistration); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/controllerregistration/extensionclusterrole/add.go b/pkg/controllermanager/controller/controllerregistration/extensionclusterrole/add.go index 4ed0dfd3c72..d49bdd6110b 100644 --- a/pkg/controllermanager/controller/controllerregistration/extensionclusterrole/add.go +++ b/pkg/controllermanager/controller/controllerregistration/extensionclusterrole/add.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -64,7 +65,10 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { ControllerManagedBy(mgr). Named(ControllerName). For(clusterRole, builder.WithPredicates(labelSelectorPredicate)). - WithOptions(controller.Options{MaxConcurrentReconciles: 5}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 5, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, + }). Watches( serviceAccount, handler.EnqueueRequestsFromMapFunc(r.MapToMatchingClusterRoles(mgr.GetLogger().WithValues("controller", ControllerName))), diff --git a/pkg/controllermanager/controller/controllerregistration/extensionclusterrole/reconciler.go b/pkg/controllermanager/controller/controllerregistration/extensionclusterrole/reconciler.go index 954f5f05a36..3def65afc2b 100644 --- a/pkg/controllermanager/controller/controllerregistration/extensionclusterrole/reconciler.go +++ b/pkg/controllermanager/controller/controllerregistration/extensionclusterrole/reconciler.go @@ -35,9 +35,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - clusterRole := &metav1.PartialObjectMetadata{} clusterRole.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("ClusterRole")) if err := r.Client.Get(ctx, request.NamespacedName, clusterRole); err != nil { diff --git a/pkg/controllermanager/controller/controllerregistration/seed/add.go b/pkg/controllermanager/controller/controllerregistration/seed/add.go index 000a8240128..12d103b4c45 100644 --- a/pkg/controllermanager/controller/controllerregistration/seed/add.go +++ b/pkg/controllermanager/controller/controllerregistration/seed/add.go @@ -24,6 +24,7 @@ import ( gardencorev1 "github.com/gardener/gardener/pkg/apis/core/v1" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/controllerutils/mapper" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -46,6 +47,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Seed{}, builder.WithPredicates(r.SeedPredicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Watches( &gardencorev1beta1.ControllerRegistration{}, diff --git a/pkg/controllermanager/controller/controllerregistration/seed/reconciler.go b/pkg/controllermanager/controller/controllerregistration/seed/reconciler.go index fb434b51d6d..6961731a62c 100644 --- a/pkg/controllermanager/controller/controllerregistration/seed/reconciler.go +++ b/pkg/controllermanager/controller/controllerregistration/seed/reconciler.go @@ -57,9 +57,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - seed := &gardencorev1beta1.Seed{} if err := r.Client.Get(ctx, request.NamespacedName, seed); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/controllerregistration/seedfinalizer/add.go b/pkg/controllermanager/controller/controllerregistration/seedfinalizer/add.go index 8bbd2c331a2..e9e6df7fc64 100644 --- a/pkg/controllermanager/controller/controllerregistration/seedfinalizer/add.go +++ b/pkg/controllermanager/controller/controllerregistration/seedfinalizer/add.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -35,6 +36,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Seed{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). WithOptions(controller.Options{ MaxConcurrentReconciles: 5, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Watches( &gardencorev1beta1.ControllerInstallation{}, diff --git a/pkg/controllermanager/controller/controllerregistration/seedfinalizer/reconciler.go b/pkg/controllermanager/controller/controllerregistration/seedfinalizer/reconciler.go index a17366ed710..fec718ac872 100644 --- a/pkg/controllermanager/controller/controllerregistration/seedfinalizer/reconciler.go +++ b/pkg/controllermanager/controller/controllerregistration/seedfinalizer/reconciler.go @@ -34,9 +34,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - seed := &gardencorev1beta1.Seed{} if err := r.Client.Get(ctx, request.NamespacedName, seed); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/credentialsbinding/add.go b/pkg/controllermanager/controller/credentialsbinding/add.go index 903d6f6e1b5..d6d9e7fca58 100644 --- a/pkg/controllermanager/controller/credentialsbinding/add.go +++ b/pkg/controllermanager/controller/credentialsbinding/add.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" securityv1alpha1 "github.com/gardener/gardener/pkg/apis/security/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -31,6 +32,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&securityv1alpha1.CredentialsBinding{}). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/credentialsbinding/reconciler.go b/pkg/controllermanager/controller/credentialsbinding/reconciler.go index c8928a8ea3c..7b63d622ed5 100644 --- a/pkg/controllermanager/controller/credentialsbinding/reconciler.go +++ b/pkg/controllermanager/controller/credentialsbinding/reconciler.go @@ -16,6 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -23,6 +24,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" securityv1alpha1 "github.com/gardener/gardener/pkg/apis/security/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils" @@ -42,9 +44,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - credentialsBinding := &securityv1alpha1.CredentialsBinding{} if err := r.Client.Get(ctx, request.NamespacedName, credentialsBinding); err != nil { if apierrors.IsNotFound(err) { @@ -78,7 +77,11 @@ func (r *Reconciler) reconcile(ctx context.Context, credentialsBinding *security if err != nil { return err } - kind := credential.GetObjectKind().GroupVersionKind().Kind + gvk, err := apiutil.GVKForObject(credential, kubernetes.GardenScheme) + if err != nil { + return err + } + kind := gvk.Kind if !controllerutil.ContainsFinalizer(credential, finalizerName) { log.Info("Adding finalizer", "finalizer", finalizerName, kind, client.ObjectKeyFromObject(credential)) //nolint:logcheck @@ -100,8 +103,8 @@ func (r *Reconciler) reconcile(ctx context.Context, credentialsBinding *security _, hasProviderKeyLabel := labels[providerTypeLabelKey] _, hasCredentialsBindingRefLabel := labels[v1beta1constants.LabelCredentialsBindingReference] - secretNeedsLabel := !hasProviderKeyLabel && credential.GetObjectKind().GroupVersionKind().Kind == "Secret" - workloadIdentityNeedsLabelRemoval := hasProviderKeyLabel && credential.GetObjectKind().GroupVersionKind().Kind == "WorkloadIdentity" + secretNeedsLabel := !hasProviderKeyLabel && kind == "Secret" + workloadIdentityNeedsLabelRemoval := hasProviderKeyLabel && kind == "WorkloadIdentity" if secretNeedsLabel || workloadIdentityNeedsLabelRemoval || !hasCredentialsBindingRefLabel { patch := client.MergeFrom(credential.DeepCopyObject().(client.Object)) if secretNeedsLabel { diff --git a/pkg/controllermanager/controller/event/add.go b/pkg/controllermanager/controller/event/add.go index 7e2da50da50..04963742a3c 100644 --- a/pkg/controllermanager/controller/event/add.go +++ b/pkg/controllermanager/controller/event/add.go @@ -12,6 +12,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -33,6 +34,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&corev1.Event{}, builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create))). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/event/reconciler.go b/pkg/controllermanager/controller/event/reconciler.go index 8115f669615..0f4f007001e 100644 --- a/pkg/controllermanager/controller/event/reconciler.go +++ b/pkg/controllermanager/controller/event/reconciler.go @@ -18,7 +18,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" ) // Reconciler reconciles Event. @@ -32,9 +31,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - event := &corev1.Event{} if err := r.Client.Get(ctx, req.NamespacedName, event); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/exposureclass/add.go b/pkg/controllermanager/controller/exposureclass/add.go index 640ef3465d9..dd1a062e549 100644 --- a/pkg/controllermanager/controller/exposureclass/add.go +++ b/pkg/controllermanager/controller/exposureclass/add.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -32,6 +33,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), RateLimiter: r.RateLimiter, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/exposureclass/reconciler.go b/pkg/controllermanager/controller/exposureclass/reconciler.go index cddfc8cf746..5e543b741f6 100644 --- a/pkg/controllermanager/controller/exposureclass/reconciler.go +++ b/pkg/controllermanager/controller/exposureclass/reconciler.go @@ -38,9 +38,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - exposureClass := &gardencorev1beta1.ExposureClass{} if err := r.Client.Get(ctx, request.NamespacedName, exposureClass); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/managedseedset/add.go b/pkg/controllermanager/controller/managedseedset/add.go index dab297ef634..cc729639615 100644 --- a/pkg/controllermanager/controller/managedseedset/add.go +++ b/pkg/controllermanager/controller/managedseedset/add.go @@ -49,6 +49,7 @@ func (r *Reconciler) AddToManager(ctx context.Context, mgr manager.Manager) erro For(&seedmanagementv1alpha1.ManagedSeedSet{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). Watches( &gardencorev1beta1.Shoot{}, diff --git a/pkg/controllermanager/controller/managedseedset/reconciler.go b/pkg/controllermanager/controller/managedseedset/reconciler.go index 58231b75373..bad2d2b3787 100644 --- a/pkg/controllermanager/controller/managedseedset/reconciler.go +++ b/pkg/controllermanager/controller/managedseedset/reconciler.go @@ -32,9 +32,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() - managedSeedSet := &seedmanagementv1alpha1.ManagedSeedSet{} if err := r.Client.Get(ctx, request.NamespacedName, managedSeedSet); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/namespacedcloudprofile/add.go b/pkg/controllermanager/controller/namespacedcloudprofile/add.go index 4517b96ee66..e64c39bb472 100644 --- a/pkg/controllermanager/controller/namespacedcloudprofile/add.go +++ b/pkg/controllermanager/controller/namespacedcloudprofile/add.go @@ -40,6 +40,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.NamespacedCloudProfile{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Watches( &gardencorev1beta1.CloudProfile{}, diff --git a/pkg/controllermanager/controller/namespacedcloudprofile/reconciler.go b/pkg/controllermanager/controller/namespacedcloudprofile/reconciler.go index d692774aea1..b7b5fde79bf 100644 --- a/pkg/controllermanager/controller/namespacedcloudprofile/reconciler.go +++ b/pkg/controllermanager/controller/namespacedcloudprofile/reconciler.go @@ -40,9 +40,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - namespacedCloudProfile := &gardencorev1beta1.NamespacedCloudProfile{} if err := r.Client.Get(ctx, request.NamespacedName, namespacedCloudProfile); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/project/activity/add.go b/pkg/controllermanager/controller/project/activity/add.go index 1218534d2c5..f22b4b9b525 100644 --- a/pkg/controllermanager/controller/project/activity/add.go +++ b/pkg/controllermanager/controller/project/activity/add.go @@ -26,6 +26,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -46,6 +47,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Watches( &gardencorev1beta1.Shoot{}, diff --git a/pkg/controllermanager/controller/project/activity/reconciler.go b/pkg/controllermanager/controller/project/activity/reconciler.go index 8f133962707..88ebc3d56de 100644 --- a/pkg/controllermanager/controller/project/activity/reconciler.go +++ b/pkg/controllermanager/controller/project/activity/reconciler.go @@ -17,7 +17,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" ) // Reconciler reconciles Projects and updates the lastActivityTimestamp in the status. @@ -31,9 +30,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - project := &gardencorev1beta1.Project{} if err := r.Client.Get(ctx, request.NamespacedName, project); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/project/project/add.go b/pkg/controllermanager/controller/project/project/add.go index 9045a16def8..79c35463b4d 100644 --- a/pkg/controllermanager/controller/project/project/add.go +++ b/pkg/controllermanager/controller/project/project/add.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -40,6 +41,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), RateLimiter: r.RateLimiter, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/project/project/reconciler.go b/pkg/controllermanager/controller/project/project/reconciler.go index f3736584f79..19acc6eb524 100644 --- a/pkg/controllermanager/controller/project/project/reconciler.go +++ b/pkg/controllermanager/controller/project/project/reconciler.go @@ -51,9 +51,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - project := &gardencorev1beta1.Project{} if err := r.Client.Get(ctx, request.NamespacedName, project); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/project/stale/add.go b/pkg/controllermanager/controller/project/stale/add.go index 752a368971c..6ba28f296a2 100644 --- a/pkg/controllermanager/controller/project/stale/add.go +++ b/pkg/controllermanager/controller/project/stale/add.go @@ -36,6 +36,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Project{}, builder.WithPredicates(r.ProjectPredicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.StaleSyncPeriod.Duration, }). Complete(r) } diff --git a/pkg/controllermanager/controller/project/stale/reconciler.go b/pkg/controllermanager/controller/project/stale/reconciler.go index 7f421bfd69f..89654c7a059 100644 --- a/pkg/controllermanager/controller/project/stale/reconciler.go +++ b/pkg/controllermanager/controller/project/stale/reconciler.go @@ -25,7 +25,6 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" securityv1alpha1 "github.com/gardener/gardener/pkg/apis/security/v1alpha1" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) @@ -41,9 +40,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.StaleSyncPeriod.Duration) - defer cancel() - project := &gardencorev1beta1.Project{} if err := r.Client.Get(ctx, request.NamespacedName, project); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/quota/add.go b/pkg/controllermanager/controller/quota/add.go index bdc20f38680..a172ce9e8ed 100644 --- a/pkg/controllermanager/controller/quota/add.go +++ b/pkg/controllermanager/controller/quota/add.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -31,6 +32,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Quota{}). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/quota/reconciler.go b/pkg/controllermanager/controller/quota/reconciler.go index 82dbcb9d044..ce20a5bd0c2 100644 --- a/pkg/controllermanager/controller/quota/reconciler.go +++ b/pkg/controllermanager/controller/quota/reconciler.go @@ -34,9 +34,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - quota := &gardencorev1beta1.Quota{} if err := r.Client.Get(ctx, request.NamespacedName, quota); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/secretbinding/add.go b/pkg/controllermanager/controller/secretbinding/add.go index f892f0d50e4..5e3453a889d 100644 --- a/pkg/controllermanager/controller/secretbinding/add.go +++ b/pkg/controllermanager/controller/secretbinding/add.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -31,6 +32,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.SecretBinding{}). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/secretbinding/reconciler.go b/pkg/controllermanager/controller/secretbinding/reconciler.go index afd9c2bf0ec..05362addee7 100644 --- a/pkg/controllermanager/controller/secretbinding/reconciler.go +++ b/pkg/controllermanager/controller/secretbinding/reconciler.go @@ -39,9 +39,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - secretBinding := &gardencorev1beta1.SecretBinding{} if err := r.Client.Get(ctx, request.NamespacedName, secretBinding); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/seed/backupbucketscheck/add.go b/pkg/controllermanager/controller/seed/backupbucketscheck/add.go index ed601aaa4f7..2e4e83cb14c 100644 --- a/pkg/controllermanager/controller/seed/backupbucketscheck/add.go +++ b/pkg/controllermanager/controller/seed/backupbucketscheck/add.go @@ -39,6 +39,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). Watches( &gardencorev1beta1.BackupBucket{}, diff --git a/pkg/controllermanager/controller/seed/backupbucketscheck/reconciler.go b/pkg/controllermanager/controller/seed/backupbucketscheck/reconciler.go index 556ad33bede..b9008a2d624 100644 --- a/pkg/controllermanager/controller/seed/backupbucketscheck/reconciler.go +++ b/pkg/controllermanager/controller/seed/backupbucketscheck/reconciler.go @@ -19,7 +19,6 @@ import ( v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/controllermanager/controller/seed/utils" - "github.com/gardener/gardener/pkg/controllerutils" ) // Reconciler reconciles Seeds and maintains the BackupBucketsReady condition according to the observed status of the @@ -35,9 +34,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() - seed := &gardencorev1beta1.Seed{} if err := r.Client.Get(ctx, req.NamespacedName, seed); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/seed/extensionscheck/add.go b/pkg/controllermanager/controller/seed/extensionscheck/add.go index 6e5453c2b4c..3c654291674 100644 --- a/pkg/controllermanager/controller/seed/extensionscheck/add.go +++ b/pkg/controllermanager/controller/seed/extensionscheck/add.go @@ -38,6 +38,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). Watches( &gardencorev1beta1.ControllerInstallation{}, diff --git a/pkg/controllermanager/controller/seed/extensionscheck/reconciler.go b/pkg/controllermanager/controller/seed/extensionscheck/reconciler.go index 0a3afa37b45..fa04eb753de 100644 --- a/pkg/controllermanager/controller/seed/extensionscheck/reconciler.go +++ b/pkg/controllermanager/controller/seed/extensionscheck/reconciler.go @@ -20,7 +20,6 @@ import ( "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/controllermanager/controller/seed/utils" - "github.com/gardener/gardener/pkg/controllerutils" ) var conditionsToCheck = []gardencorev1beta1.ConditionType{ @@ -43,9 +42,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() - seed := &gardencorev1beta1.Seed{} if err := r.Client.Get(ctx, request.NamespacedName, seed); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/seed/lifecycle/add.go b/pkg/controllermanager/controller/seed/lifecycle/add.go index 6180f4730a8..6e093a9067d 100644 --- a/pkg/controllermanager/controller/seed/lifecycle/add.go +++ b/pkg/controllermanager/controller/seed/lifecycle/add.go @@ -36,6 +36,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Seed{}, builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create))). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). Complete(r) } diff --git a/pkg/controllermanager/controller/seed/lifecycle/reconciler.go b/pkg/controllermanager/controller/seed/lifecycle/reconciler.go index c5d5408aa87..6e0778f5ed9 100644 --- a/pkg/controllermanager/controller/seed/lifecycle/reconciler.go +++ b/pkg/controllermanager/controller/seed/lifecycle/reconciler.go @@ -21,7 +21,6 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/flow" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" @@ -43,9 +42,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() - seed := &gardencorev1beta1.Seed{} if err := r.Client.Get(ctx, req.NamespacedName, seed); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/seed/secrets/add.go b/pkg/controllermanager/controller/seed/secrets/add.go index 4b065836ee7..276bc88f0d2 100644 --- a/pkg/controllermanager/controller/seed/secrets/add.go +++ b/pkg/controllermanager/controller/seed/secrets/add.go @@ -24,6 +24,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/controllerutils/mapper" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" "github.com/gardener/gardener/pkg/utils" @@ -48,6 +49,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Seed{}, builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create))). WithOptions(controller.Options{ MaxConcurrentReconciles: 5, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Watches( &corev1.Secret{}, diff --git a/pkg/controllermanager/controller/seed/secrets/reconciler.go b/pkg/controllermanager/controller/seed/secrets/reconciler.go index 73453e9eb6b..2039af0b708 100644 --- a/pkg/controllermanager/controller/seed/secrets/reconciler.go +++ b/pkg/controllermanager/controller/seed/secrets/reconciler.go @@ -35,9 +35,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - seed := &gardencorev1beta1.Seed{} if err := r.Client.Get(ctx, req.NamespacedName, seed); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/shoot/conditions/add.go b/pkg/controllermanager/controller/shoot/conditions/add.go index 3e6ac58af31..a2b76fd0fc5 100644 --- a/pkg/controllermanager/controller/shoot/conditions/add.go +++ b/pkg/controllermanager/controller/shoot/conditions/add.go @@ -24,6 +24,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" seedmanagementv1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -42,6 +43,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Shoot{}, builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create))). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Watches( &gardencorev1beta1.Seed{}, diff --git a/pkg/controllermanager/controller/shoot/conditions/reconciler.go b/pkg/controllermanager/controller/shoot/conditions/reconciler.go index 415ea8a3631..c466c7c1e4a 100644 --- a/pkg/controllermanager/controller/shoot/conditions/reconciler.go +++ b/pkg/controllermanager/controller/shoot/conditions/reconciler.go @@ -16,7 +16,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) @@ -31,9 +30,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.Client.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/shoot/hibernation/add.go b/pkg/controllermanager/controller/shoot/hibernation/add.go index fe86a6a0906..541353d57a7 100644 --- a/pkg/controllermanager/controller/shoot/hibernation/add.go +++ b/pkg/controllermanager/controller/shoot/hibernation/add.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -39,6 +40,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Shoot{}, builder.WithPredicates(r.ShootPredicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/shoot/hibernation/reconciler.go b/pkg/controllermanager/controller/shoot/hibernation/reconciler.go index 7ab527e3f1f..faedb7d92fc 100644 --- a/pkg/controllermanager/controller/shoot/hibernation/reconciler.go +++ b/pkg/controllermanager/controller/shoot/hibernation/reconciler.go @@ -23,7 +23,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -80,9 +79,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.Client.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/shoot/maintenance/add.go b/pkg/controllermanager/controller/shoot/maintenance/add.go index 20dd2842a91..4c012b75140 100644 --- a/pkg/controllermanager/controller/shoot/maintenance/add.go +++ b/pkg/controllermanager/controller/shoot/maintenance/add.go @@ -15,6 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -38,6 +39,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Shoot{}, builder.WithPredicates(r.ShootPredicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/shoot/maintenance/reconciler.go b/pkg/controllermanager/controller/shoot/maintenance/reconciler.go index ee9440fef18..5bc6f29379e 100644 --- a/pkg/controllermanager/controller/shoot/maintenance/reconciler.go +++ b/pkg/controllermanager/controller/shoot/maintenance/reconciler.go @@ -49,9 +49,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.Client.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/shoot/migration/add.go b/pkg/controllermanager/controller/shoot/migration/add.go index 6cc6ad4bddc..95d396be4d1 100644 --- a/pkg/controllermanager/controller/shoot/migration/add.go +++ b/pkg/controllermanager/controller/shoot/migration/add.go @@ -14,6 +14,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -31,6 +32,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Shoot{}, builder.WithPredicates(r.ShootPredicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/shoot/migration/reconciler.go b/pkg/controllermanager/controller/shoot/migration/reconciler.go index 259fb48e0e6..a0d0fb2c2c8 100644 --- a/pkg/controllermanager/controller/shoot/migration/reconciler.go +++ b/pkg/controllermanager/controller/shoot/migration/reconciler.go @@ -18,7 +18,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/kubernetes/health" ) @@ -34,9 +33,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.Client.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/shoot/quota/add.go b/pkg/controllermanager/controller/shoot/quota/add.go index a94b5da83f1..90497b8ef77 100644 --- a/pkg/controllermanager/controller/shoot/quota/add.go +++ b/pkg/controllermanager/controller/shoot/quota/add.go @@ -33,6 +33,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Shoot{}, builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create))). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). Complete(r) } diff --git a/pkg/controllermanager/controller/shoot/quota/reconciler.go b/pkg/controllermanager/controller/shoot/quota/reconciler.go index cc587c096d6..05c9e2f5941 100644 --- a/pkg/controllermanager/controller/shoot/quota/reconciler.go +++ b/pkg/controllermanager/controller/shoot/quota/reconciler.go @@ -20,7 +20,6 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" securityv1alpha1 "github.com/gardener/gardener/pkg/apis/security/v1alpha1" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -35,9 +34,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.Client.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/shoot/retry/add.go b/pkg/controllermanager/controller/shoot/retry/add.go index b2b1a73b519..f07834e74e2 100644 --- a/pkg/controllermanager/controller/shoot/retry/add.go +++ b/pkg/controllermanager/controller/shoot/retry/add.go @@ -14,6 +14,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -31,6 +32,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Shoot{}, builder.WithPredicates(r.ShootPredicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/shoot/retry/reconciler.go b/pkg/controllermanager/controller/shoot/retry/reconciler.go index f9623df5575..c6b18ae161e 100644 --- a/pkg/controllermanager/controller/shoot/retry/reconciler.go +++ b/pkg/controllermanager/controller/shoot/retry/reconciler.go @@ -18,7 +18,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils" ) @@ -32,9 +31,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.Client.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllermanager/controller/shoot/statuslabel/add.go b/pkg/controllermanager/controller/shoot/statuslabel/add.go index 643b41a35c4..58ae178b4ee 100644 --- a/pkg/controllermanager/controller/shoot/statuslabel/add.go +++ b/pkg/controllermanager/controller/shoot/statuslabel/add.go @@ -14,6 +14,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -32,6 +33,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Shoot{}, builder.WithPredicates(r.ShootPredicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/controllermanager/controller/shoot/statuslabel/reconciler.go b/pkg/controllermanager/controller/shoot/statuslabel/reconciler.go index 12cb08a606f..4700134d586 100644 --- a/pkg/controllermanager/controller/shoot/statuslabel/reconciler.go +++ b/pkg/controllermanager/controller/shoot/statuslabel/reconciler.go @@ -17,7 +17,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -31,9 +30,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.Client.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/controllerutils/miscellaneous.go b/pkg/controllerutils/miscellaneous.go index 5f8e50f0037..2b8b8bf5cc7 100644 --- a/pkg/controllerutils/miscellaneous.go +++ b/pkg/controllerutils/miscellaneous.go @@ -77,14 +77,6 @@ func setTaskAnnotations(annotations map[string]string, tasks []string) { annotations[v1beta1constants.ShootTasks] = strings.Join(tasks, separator) } -// GetMainReconciliationContext returns a context with timeout for the controller's main client. The resulting context has a timeout equal to the timeout passed in the argument but -// not more than DefaultReconciliationTimeout. -func GetMainReconciliationContext(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { - t := min(timeout, DefaultReconciliationTimeout) - - return context.WithTimeout(ctx, t) -} - // GetChildReconciliationContext returns context with timeout for the controller's secondary client. The resulting context has a timeout equal to half of the timeout // for the controller's main client. func GetChildReconciliationContext(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { diff --git a/pkg/extensions/cluster_test.go b/pkg/extensions/cluster_test.go index 28a86b65268..8a2626cea9b 100644 --- a/pkg/extensions/cluster_test.go +++ b/pkg/extensions/cluster_test.go @@ -12,6 +12,8 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -39,7 +41,10 @@ var _ = Describe("Cluster", func() { BeforeEach(func() { fakeGardenClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).Build() - fakeSeedClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder()) + fakeSeedClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithObjectTracker(objectTracker).Build() }) Describe("#GenericTokenKubeconfigSecretNameFromCluster", func() { diff --git a/pkg/gardenadm/staticpod/translator_test.go b/pkg/gardenadm/staticpod/translator_test.go index 2b8868079cc..11f77f0d168 100644 --- a/pkg/gardenadm/staticpod/translator_test.go +++ b/pkg/gardenadm/staticpod/translator_test.go @@ -72,7 +72,6 @@ metadata: annotations: bar: baz gardener.cloud/config.mirror: ` + hash + ` - creationTimestamp: null labels: baz: foo static-pod: "true" @@ -116,7 +115,6 @@ metadata: bar: baz foo: bar gardener.cloud/config.mirror: ` + hash + ` - creationTimestamp: null labels: foo: bar static-pod: "true" @@ -237,7 +235,6 @@ metadata: annotations: bar: baz gardener.cloud/config.mirror: ` + hash + ` - creationTimestamp: null labels: baz: foo static-pod: "true" @@ -364,7 +361,6 @@ metadata: annotations: bar: baz gardener.cloud/config.mirror: ` + hash + ` - creationTimestamp: null labels: baz: foo static-pod: "true" @@ -415,7 +411,6 @@ metadata: bar: baz foo: bar gardener.cloud/config.mirror: ` + hash + ` - creationTimestamp: null labels: foo: bar static-pod: "true" @@ -543,7 +538,6 @@ metadata: annotations: bar: baz gardener.cloud/config.mirror: ` + hash + ` - creationTimestamp: null labels: baz: foo static-pod: "true" @@ -665,7 +659,6 @@ metadata: annotations: bar: baz gardener.cloud/config.mirror: ` + hash + ` - creationTimestamp: null labels: baz: foo static-pod: "true" @@ -707,7 +700,6 @@ metadata: bar: baz foo: bar gardener.cloud/config.mirror: ` + hash + ` - creationTimestamp: null labels: foo: bar static-pod: "true" @@ -826,7 +818,6 @@ metadata: annotations: bar: baz gardener.cloud/config.mirror: ` + hash + ` - creationTimestamp: null labels: baz: foo static-pod: "true" diff --git a/pkg/gardenlet/apis/config/v1alpha1/validation/validation_test.go b/pkg/gardenlet/apis/config/v1alpha1/validation/validation_test.go index fd31afcf283..f8859c619e6 100644 --- a/pkg/gardenlet/apis/config/v1alpha1/validation/validation_test.go +++ b/pkg/gardenlet/apis/config/v1alpha1/validation/validation_test.go @@ -286,7 +286,7 @@ var _ = Describe("GardenletConfiguration", func() { Expect(ValidateGardenletConfiguration(cfg, nil)).To(ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), + "Type": Equal(field.ErrorTypeRequired), "Field": Equal("leaderElection.resourceNamespace"), })), )) diff --git a/pkg/gardenlet/controller/backupbucket/add.go b/pkg/gardenlet/controller/backupbucket/add.go index 3d51f00a2ff..09cd858eec4 100644 --- a/pkg/gardenlet/controller/backupbucket/add.go +++ b/pkg/gardenlet/controller/backupbucket/add.go @@ -23,6 +23,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -53,6 +54,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster cluster.Clu WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), RateLimiter: r.RateLimiter, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource(source.Kind[client.Object]( gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/backupbucket/reconciler.go b/pkg/gardenlet/controller/backupbucket/reconciler.go index 69ecf85a866..2c1c0ca2d97 100644 --- a/pkg/gardenlet/controller/backupbucket/reconciler.go +++ b/pkg/gardenlet/controller/backupbucket/reconciler.go @@ -20,6 +20,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -29,6 +30,7 @@ import ( v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" securityv1alpha1 "github.com/gardener/gardener/pkg/apis/security/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/extensions" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" @@ -61,8 +63,7 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - gardenCtx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() + gardenCtx := ctx seedCtx, cancel := controllerutils.GetChildReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) defer cancel() @@ -379,12 +380,16 @@ func (r *Reconciler) reconcileBackupBucketExtensionSecret(ctx context.Context, e }) return err case *securityv1alpha1.WorkloadIdentity: + gvk, err := apiutil.GVKForObject(backupBucket, kubernetes.GardenScheme) + if err != nil { + return err + } s, err := workloadidentity.NewSecret( extensionSecret.Name, extensionSecret.Namespace, workloadidentity.For(credentials.GetName(), credentials.GetNamespace(), credentials.Spec.TargetSystem.Type), workloadidentity.WithProviderConfig(credentials.Spec.TargetSystem.ProviderConfig), - workloadidentity.WithContextObject(securityv1alpha1.ContextObject{APIVersion: backupBucket.APIVersion, Kind: backupBucket.Kind, Name: backupBucket.GetName(), UID: backupBucket.GetUID()}), + workloadidentity.WithContextObject(securityv1alpha1.ContextObject{APIVersion: gvk.GroupVersion().String(), Kind: gvk.Kind, Name: backupBucket.GetName(), UID: backupBucket.GetUID()}), workloadidentity.WithAnnotations(map[string]string{v1beta1constants.GardenerTimestamp: now}), ) if err != nil { @@ -534,10 +539,14 @@ func generateGeneratedBackupBucketSecretName(backupBucketName string) string { } func workloadIdentitySecretChanged(backupBucket *gardencorev1beta1.BackupBucket, secretToCompareTo *corev1.Secret, workloadIdentity *securityv1alpha1.WorkloadIdentity) (bool, error) { + gvk, err := apiutil.GVKForObject(backupBucket, kubernetes.GardenScheme) + if err != nil { + return false, err + } opts := []workloadidentity.SecretOption{ workloadidentity.For(workloadIdentity.GetName(), workloadIdentity.GetNamespace(), workloadIdentity.Spec.TargetSystem.Type), workloadidentity.WithProviderConfig(workloadIdentity.Spec.TargetSystem.ProviderConfig), - workloadidentity.WithContextObject(securityv1alpha1.ContextObject{APIVersion: backupBucket.APIVersion, Kind: backupBucket.Kind, Name: backupBucket.GetName(), UID: backupBucket.GetUID()}), + workloadidentity.WithContextObject(securityv1alpha1.ContextObject{APIVersion: gvk.GroupVersion().String(), Kind: gvk.Kind, Name: backupBucket.GetName(), UID: backupBucket.GetUID()}), } if val, ok := secretToCompareTo.Annotations[v1beta1constants.GardenerTimestamp]; ok { opts = append(opts, workloadidentity.WithAnnotations(map[string]string{v1beta1constants.GardenerTimestamp: val})) diff --git a/pkg/gardenlet/controller/backupentry/add.go b/pkg/gardenlet/controller/backupentry/add.go index f0f324f6600..a1d2aa01ed0 100644 --- a/pkg/gardenlet/controller/backupentry/add.go +++ b/pkg/gardenlet/controller/backupentry/add.go @@ -25,6 +25,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/controllerutils/mapper" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" "github.com/gardener/gardener/pkg/extensions" @@ -58,6 +59,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster, seedCluste WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), RateLimiter: r.RateLimiter, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource(source.Kind[client.Object]( gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/backupentry/reconciler.go b/pkg/gardenlet/controller/backupentry/reconciler.go index 938ce251a33..2f7849e4eb5 100644 --- a/pkg/gardenlet/controller/backupentry/reconciler.go +++ b/pkg/gardenlet/controller/backupentry/reconciler.go @@ -22,6 +22,7 @@ import ( "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -31,6 +32,7 @@ import ( v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" securityv1alpha1 "github.com/gardener/gardener/pkg/apis/security/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" extensionsbackupentry "github.com/gardener/gardener/pkg/component/extensions/backupentry" "github.com/gardener/gardener/pkg/controllerutils" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" @@ -70,8 +72,7 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - gardenCtx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() + gardenCtx := ctx seedCtx, cancel := controllerutils.GetChildReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) defer cancel() @@ -672,12 +673,16 @@ func (r *Reconciler) reconcileBackupEntryExtensionSecret(ctx context.Context, ex }) return err case *securityv1alpha1.WorkloadIdentity: + gvk, err := apiutil.GVKForObject(backupEntry, kubernetes.GardenScheme) + if err != nil { + return err + } s, err := workloadidentity.NewSecret( extensionSecret.Name, extensionSecret.Namespace, workloadidentity.For(credentials.GetName(), credentials.GetNamespace(), credentials.Spec.TargetSystem.Type), workloadidentity.WithProviderConfig(credentials.Spec.TargetSystem.ProviderConfig), - workloadidentity.WithContextObject(securityv1alpha1.ContextObject{APIVersion: backupEntry.APIVersion, Kind: backupEntry.Kind, Namespace: ptr.To(backupEntry.GetNamespace()), Name: backupEntry.GetName(), UID: backupEntry.GetUID()}), + workloadidentity.WithContextObject(securityv1alpha1.ContextObject{APIVersion: gvk.GroupVersion().String(), Kind: gvk.Kind, Namespace: ptr.To(backupEntry.GetNamespace()), Name: backupEntry.GetName(), UID: backupEntry.GetUID()}), workloadidentity.WithAnnotations(map[string]string{v1beta1constants.GardenerTimestamp: now}), ) if err != nil { @@ -743,10 +748,14 @@ func removeGardenerOperationAnnotation(ctx context.Context, c client.Client, be } func workloadIdentitySecretChanged(backupEntry *gardencorev1beta1.BackupEntry, secretToCompareTo *corev1.Secret, workloadIdentity *securityv1alpha1.WorkloadIdentity) (bool, error) { + gvk, err := apiutil.GVKForObject(backupEntry, kubernetes.GardenScheme) + if err != nil { + return false, err + } opts := []workloadidentity.SecretOption{ workloadidentity.For(workloadIdentity.GetName(), workloadIdentity.GetNamespace(), workloadIdentity.Spec.TargetSystem.Type), workloadidentity.WithProviderConfig(workloadIdentity.Spec.TargetSystem.ProviderConfig), - workloadidentity.WithContextObject(securityv1alpha1.ContextObject{APIVersion: backupEntry.APIVersion, Kind: backupEntry.Kind, Namespace: ptr.To(backupEntry.GetNamespace()), Name: backupEntry.GetName(), UID: backupEntry.GetUID()}), + workloadidentity.WithContextObject(securityv1alpha1.ContextObject{APIVersion: gvk.GroupVersion().String(), Kind: gvk.Kind, Namespace: ptr.To(backupEntry.GetNamespace()), Name: backupEntry.GetName(), UID: backupEntry.GetUID()}), } if val, ok := secretToCompareTo.Annotations[v1beta1constants.GardenerTimestamp]; ok { opts = append(opts, workloadidentity.WithAnnotations(map[string]string{v1beta1constants.GardenerTimestamp: val})) diff --git a/pkg/gardenlet/controller/bastion/add.go b/pkg/gardenlet/controller/bastion/add.go index fcd63826599..dc1247b21b7 100644 --- a/pkg/gardenlet/controller/bastion/add.go +++ b/pkg/gardenlet/controller/bastion/add.go @@ -23,6 +23,7 @@ import ( extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" operationsv1alpha1 "github.com/gardener/gardener/pkg/apis/operations/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" "github.com/gardener/gardener/pkg/extensions" ) @@ -48,6 +49,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster, seedCluste WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), RateLimiter: r.RateLimiter, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource(source.Kind[client.Object]( gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/bastion/reconciler.go b/pkg/gardenlet/controller/bastion/reconciler.go index f7d62a64bb2..f57bb932684 100644 --- a/pkg/gardenlet/controller/bastion/reconciler.go +++ b/pkg/gardenlet/controller/bastion/reconciler.go @@ -51,8 +51,7 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - gardenCtx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() + gardenCtx := ctx seedCtx, cancel := controllerutils.GetChildReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) defer cancel() diff --git a/pkg/gardenlet/controller/controllerinstallation/care/add.go b/pkg/gardenlet/controller/controllerinstallation/care/add.go index 3dfad024cac..e1153347a54 100644 --- a/pkg/gardenlet/controller/controllerinstallation/care/add.go +++ b/pkg/gardenlet/controller/controllerinstallation/care/add.go @@ -52,7 +52,8 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster, seedCluste WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), // if going into exponential backoff, wait at most the configured sync period - RateLimiter: workqueue.NewTypedWithMaxWaitRateLimiter(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), r.Config.SyncPeriod.Duration), + RateLimiter: workqueue.NewTypedWithMaxWaitRateLimiter(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), r.Config.SyncPeriod.Duration), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). WatchesRawSource(source.Kind[client.Object]( gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/controllerinstallation/care/reconciler.go b/pkg/gardenlet/controller/controllerinstallation/care/reconciler.go index 2f0a94e3ab2..f59d30cf7c1 100644 --- a/pkg/gardenlet/controller/controllerinstallation/care/reconciler.go +++ b/pkg/gardenlet/controller/controllerinstallation/care/reconciler.go @@ -37,8 +37,7 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - gardenCtx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() + gardenCtx := ctx seedCtx, cancel := controllerutils.GetChildReconciliationContext(ctx, r.Config.SyncPeriod.Duration) defer cancel() diff --git a/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/add.go b/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/add.go index fe056cbcd77..f0d87a802c5 100644 --- a/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/add.go +++ b/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/add.go @@ -23,6 +23,7 @@ import ( gardencorev1 "github.com/gardener/gardener/pkg/apis/core/v1" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/oci" ) @@ -52,6 +53,7 @@ func (r *Reconciler) AddToManager(ctx context.Context, mgr manager.Manager, gard Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.Controllers.ControllerInstallation.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource( source.Kind[client.Object](gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/reconciler.go b/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/reconciler.go index bb7205aca39..064e471d7b8 100644 --- a/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/reconciler.go +++ b/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/reconciler.go @@ -84,8 +84,7 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - gardenCtx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() + gardenCtx := ctx seedCtx, cancel := controllerutils.GetChildReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) defer cancel() diff --git a/pkg/gardenlet/controller/controllerinstallation/required/add.go b/pkg/gardenlet/controller/controllerinstallation/required/add.go index 8f410b3015a..949f0e71c4b 100644 --- a/pkg/gardenlet/controller/controllerinstallation/required/add.go +++ b/pkg/gardenlet/controller/controllerinstallation/required/add.go @@ -58,6 +58,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster, seedCluste controller.Options{ Reconciler: r, MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }, ) if err != nil { diff --git a/pkg/gardenlet/controller/controllerinstallation/required/reconciler.go b/pkg/gardenlet/controller/controllerinstallation/required/reconciler.go index 12bfc9d0c05..4400f10b1a5 100644 --- a/pkg/gardenlet/controller/controllerinstallation/required/reconciler.go +++ b/pkg/gardenlet/controller/controllerinstallation/required/reconciler.go @@ -19,7 +19,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" - "github.com/gardener/gardener/pkg/controllerutils" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" ) @@ -40,9 +39,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - controllerInstallation := &gardencorev1beta1.ControllerInstallation{} if err := r.GardenClient.Get(ctx, request.NamespacedName, controllerInstallation); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/gardenlet/controller/gardenlet/add.go b/pkg/gardenlet/controller/gardenlet/add.go index d654ce8ec13..dc787185490 100644 --- a/pkg/gardenlet/controller/gardenlet/add.go +++ b/pkg/gardenlet/controller/gardenlet/add.go @@ -70,6 +70,7 @@ func (r *Reconciler) AddToManager( // There can only be one Gardenlet object relevant for an instance of gardenlet, so it's enough to have one // worker only. MaxConcurrentReconciles: 1, + ReconciliationTimeout: r.Config.Controllers.Gardenlet.SyncPeriod.Duration, }). WatchesRawSource(source.Kind[client.Object]( gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/gardenlet/reconciler.go b/pkg/gardenlet/controller/gardenlet/reconciler.go index 04b5864c4d2..80deacb694f 100644 --- a/pkg/gardenlet/controller/gardenlet/reconciler.go +++ b/pkg/gardenlet/controller/gardenlet/reconciler.go @@ -27,7 +27,6 @@ import ( "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1/helper" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/controller/gardenletdeployer" - "github.com/gardener/gardener/pkg/controllerutils" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" gardenletutils "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" @@ -51,9 +50,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.Controllers.Gardenlet.SyncPeriod.Duration) - defer cancel() - gardenlet := &seedmanagementv1alpha1.Gardenlet{} if err := r.GardenClient.Get(ctx, request.NamespacedName, gardenlet); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/gardenlet/controller/managedseed/add.go b/pkg/gardenlet/controller/managedseed/add.go index 1f627ad3b0b..3c5cf7ec802 100644 --- a/pkg/gardenlet/controller/managedseed/add.go +++ b/pkg/gardenlet/controller/managedseed/add.go @@ -68,6 +68,7 @@ func (r *Reconciler) AddToManager( Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.Controllers.ManagedSeed.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.Controllers.ManagedSeed.SyncPeriod.Duration, }). WatchesRawSource(source.Kind[client.Object]( gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/managedseed/reconciler.go b/pkg/gardenlet/controller/managedseed/reconciler.go index 9a813907fda..7786201cea7 100644 --- a/pkg/gardenlet/controller/managedseed/reconciler.go +++ b/pkg/gardenlet/controller/managedseed/reconciler.go @@ -54,9 +54,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.Controllers.ManagedSeed.SyncPeriod.Duration) - defer cancel() - ms := &seedmanagementv1alpha1.ManagedSeed{} if err := r.GardenClient.Get(ctx, request.NamespacedName, ms); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/gardenlet/controller/seed/care/add.go b/pkg/gardenlet/controller/seed/care/add.go index 2490ae1fd36..2d3d8f51e78 100644 --- a/pkg/gardenlet/controller/seed/care/add.go +++ b/pkg/gardenlet/controller/seed/care/add.go @@ -48,7 +48,8 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster, seedCluste WithOptions(controller.Options{ MaxConcurrentReconciles: 1, // if going into exponential backoff, wait at most the configured sync period - RateLimiter: workqueue.NewTypedWithMaxWaitRateLimiter(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), r.Config.SyncPeriod.Duration), + RateLimiter: workqueue.NewTypedWithMaxWaitRateLimiter(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), r.Config.SyncPeriod.Duration), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). WatchesRawSource(source.Kind[client.Object]( gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/seed/care/reconciler.go b/pkg/gardenlet/controller/seed/care/reconciler.go index 0f800d5e3ee..4f308ab1844 100644 --- a/pkg/gardenlet/controller/seed/care/reconciler.go +++ b/pkg/gardenlet/controller/seed/care/reconciler.go @@ -42,11 +42,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(reconcileCtx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(reconcileCtx) - // Timeout for all calls (e.g. status updates), give status updates a bit of headroom if health checks - // themselves run into timeouts, so that we will still update the status with that timeout error. - reconcileCtx, cancel := controllerutils.GetMainReconciliationContext(reconcileCtx, r.Config.SyncPeriod.Duration) - defer cancel() - seed := &gardencorev1beta1.Seed{} if err := r.GardenClient.Get(reconcileCtx, req.NamespacedName, seed); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/gardenlet/controller/seed/lease/add.go b/pkg/gardenlet/controller/seed/lease/add.go index 6289db46386..f29ebd7682c 100644 --- a/pkg/gardenlet/controller/seed/lease/add.go +++ b/pkg/gardenlet/controller/seed/lease/add.go @@ -43,6 +43,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster cluster.Clu WithOptions(controller.Options{ MaxConcurrentReconciles: 1, RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](time.Millisecond, 2*time.Second), + ReconciliationTimeout: time.Duration(*r.Config.LeaseResyncSeconds) * time.Second, }). WatchesRawSource( source.Kind[client.Object](gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/seed/lease/reconciler.go b/pkg/gardenlet/controller/seed/lease/reconciler.go index c6d9a0f56ad..6ed700c6063 100644 --- a/pkg/gardenlet/controller/seed/lease/reconciler.go +++ b/pkg/gardenlet/controller/seed/lease/reconciler.go @@ -44,9 +44,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, time.Duration(*r.Config.LeaseResyncSeconds)*time.Second) - defer cancel() - seed := &gardencorev1beta1.Seed{} if err := r.GardenClient.Get(ctx, request.NamespacedName, seed); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/gardenlet/controller/shoot/care/add.go b/pkg/gardenlet/controller/shoot/care/add.go index ca761f4bafd..0df83805176 100644 --- a/pkg/gardenlet/controller/shoot/care/add.go +++ b/pkg/gardenlet/controller/shoot/care/add.go @@ -45,7 +45,8 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster cluster.Clu WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.Controllers.ShootCare.ConcurrentSyncs, 0), // if going into exponential backoff, wait at most the configured sync period - RateLimiter: workqueue.NewTypedWithMaxWaitRateLimiter(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), r.Config.Controllers.ShootCare.SyncPeriod.Duration), + RateLimiter: workqueue.NewTypedWithMaxWaitRateLimiter(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), r.Config.Controllers.ShootCare.SyncPeriod.Duration), + ReconciliationTimeout: r.Config.Controllers.ShootCare.SyncPeriod.Duration, }). WatchesRawSource( source.Kind[client.Object](gardenCluster.GetCache(), diff --git a/pkg/gardenlet/controller/shoot/care/health_test.go b/pkg/gardenlet/controller/shoot/care/health_test.go index 16107d932b0..a91e6cb236b 100644 --- a/pkg/gardenlet/controller/shoot/care/health_test.go +++ b/pkg/gardenlet/controller/shoot/care/health_test.go @@ -22,6 +22,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" testclock "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -72,7 +74,10 @@ var _ = Describe("health check", func() { ) BeforeEach(func() { - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder()) + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithObjectTracker(objectTracker).Build() fakeClock = testclock.NewFakeClock(time.Now()) condition = gardencorev1beta1.Condition{ Type: "test", diff --git a/pkg/gardenlet/controller/shoot/care/reconciler.go b/pkg/gardenlet/controller/shoot/care/reconciler.go index 21e3329a660..8e06c0befa0 100644 --- a/pkg/gardenlet/controller/shoot/care/reconciler.go +++ b/pkg/gardenlet/controller/shoot/care/reconciler.go @@ -65,11 +65,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - // Timeout for all calls (e.g. status updates), give status updates a bit of headroom if health checks - // themselves run into timeouts, so that we will still update the status with that timeout error. - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.Controllers.ShootCare.SyncPeriod.Duration) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.GardenClient.Get(ctx, req.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/gardenlet/controller/shoot/shoot/cleaner_test.go b/pkg/gardenlet/controller/shoot/shoot/cleaner_test.go index d0b24b8da5d..0a3a1e57790 100644 --- a/pkg/gardenlet/controller/shoot/shoot/cleaner_test.go +++ b/pkg/gardenlet/controller/shoot/shoot/cleaner_test.go @@ -15,6 +15,8 @@ import ( . "github.com/onsi/gomega/gstruct" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -63,7 +65,10 @@ var _ = Describe("cleaner", func() { log = logr.Discard() - seedClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder()) + seedClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithObjectTracker(objectTracker).Build() gardenClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).Build() cleaner = NewCleaner(log, seedClient, gardenClient, namespace) diff --git a/pkg/gardenlet/controller/shoot/state/add.go b/pkg/gardenlet/controller/shoot/state/add.go index c2e5f231993..fc2175e998c 100644 --- a/pkg/gardenlet/controller/shoot/state/add.go +++ b/pkg/gardenlet/controller/shoot/state/add.go @@ -18,6 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -38,7 +39,10 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster, seedCluste return builder. ControllerManagedBy(mgr). Named(ControllerName). - WithOptions(controller.Options{MaxConcurrentReconciles: *r.Config.ConcurrentSyncs}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: *r.Config.ConcurrentSyncs, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, + }). WatchesRawSource( source.Kind[client.Object](gardenCluster.GetCache(), &gardencorev1beta1.Shoot{}, diff --git a/pkg/gardenlet/controller/shoot/state/reconciler.go b/pkg/gardenlet/controller/shoot/state/reconciler.go index 27383cb69d6..e888cf66181 100644 --- a/pkg/gardenlet/controller/shoot/state/reconciler.go +++ b/pkg/gardenlet/controller/shoot/state/reconciler.go @@ -18,7 +18,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/gardener/gardener/pkg/controllerutils" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/gardener/shootstate" @@ -46,9 +45,6 @@ var ( func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.GardenClient.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/gardenlet/controller/shoot/status/add.go b/pkg/gardenlet/controller/shoot/status/add.go index 51550ec6b88..a31ac96087b 100644 --- a/pkg/gardenlet/controller/shoot/status/add.go +++ b/pkg/gardenlet/controller/shoot/status/add.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/extensions" ) @@ -40,7 +41,10 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster, seedCluste return builder. ControllerManagedBy(mgr). Named(ControllerName). - WithOptions(controller.Options{MaxConcurrentReconciles: *r.Config.ConcurrentSyncs}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: *r.Config.ConcurrentSyncs, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, + }). WatchesRawSource(source.Kind[client.Object]( seedCluster.GetCache(), &extensionsv1alpha1.Worker{}, diff --git a/pkg/gardenlet/controller/shoot/status/reconciler.go b/pkg/gardenlet/controller/shoot/status/reconciler.go index aca9e2f47d4..14b7a5ca5a7 100644 --- a/pkg/gardenlet/controller/shoot/status/reconciler.go +++ b/pkg/gardenlet/controller/shoot/status/reconciler.go @@ -21,7 +21,6 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" @@ -39,9 +38,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.GardenClient.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/gardenlet/controller/tokenrequestor/workloadidentity/add.go b/pkg/gardenlet/controller/tokenrequestor/workloadidentity/add.go index a2177a2293f..7fbca71ab79 100644 --- a/pkg/gardenlet/controller/tokenrequestor/workloadidentity/add.go +++ b/pkg/gardenlet/controller/tokenrequestor/workloadidentity/add.go @@ -20,6 +20,7 @@ import ( securityv1alpha1constants "github.com/gardener/gardener/pkg/apis/security/v1alpha1/constants" securityclientset "github.com/gardener/gardener/pkg/client/security/clientset/versioned" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of the controller. @@ -55,6 +56,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, sourceCluster, targetClus For(&corev1.Secret{}, builder.WithPredicates(r.SecretPredicate())). WithOptions(controller.Options{ MaxConcurrentReconciles: r.ConcurrentSyncs, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/gardenlet/controller/tokenrequestor/workloadidentity/reconciler.go b/pkg/gardenlet/controller/tokenrequestor/workloadidentity/reconciler.go index c1b00df970a..118d658ed7b 100644 --- a/pkg/gardenlet/controller/tokenrequestor/workloadidentity/reconciler.go +++ b/pkg/gardenlet/controller/tokenrequestor/workloadidentity/reconciler.go @@ -23,7 +23,6 @@ import ( securityv1alpha1 "github.com/gardener/gardener/pkg/apis/security/v1alpha1" securityv1alpha1constants "github.com/gardener/gardener/pkg/apis/security/v1alpha1/constants" securityclientset "github.com/gardener/gardener/pkg/client/security/clientset/versioned" - "github.com/gardener/gardener/pkg/controllerutils" ) const ( @@ -42,11 +41,8 @@ type Reconciler struct { } // Reconcile requests and populates tokens. -func (r *Reconciler) Reconcile(reconcileCtx context.Context, req reconcile.Request) (reconcile.Result, error) { - log := logf.FromContext(reconcileCtx) - - ctx, cancel := controllerutils.GetMainReconciliationContext(reconcileCtx, controllerutils.DefaultReconciliationTimeout) - defer cancel() +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + log := logf.FromContext(ctx) secret := &corev1.Secret{} if err := r.SeedClient.Get(ctx, req.NamespacedName, secret); err != nil { diff --git a/pkg/gardenlet/operation/botanist/kubeapiserver_test.go b/pkg/gardenlet/operation/botanist/kubeapiserver_test.go index 76440f5fcf7..e8c45a9fc64 100644 --- a/pkg/gardenlet/operation/botanist/kubeapiserver_test.go +++ b/pkg/gardenlet/operation/botanist/kubeapiserver_test.go @@ -489,7 +489,6 @@ contexts: name: authorization-webhook current-context: authorization-webhook kind: Config -preferences: {} users: - name: authorization-webhook user: {} diff --git a/pkg/gardenlet/operation/botanist/kubeproxy_test.go b/pkg/gardenlet/operation/botanist/kubeproxy_test.go index cd171669aad..2156499b6e9 100644 --- a/pkg/gardenlet/operation/botanist/kubeproxy_test.go +++ b/pkg/gardenlet/operation/botanist/kubeproxy_test.go @@ -153,7 +153,6 @@ contexts: name: ` + namespace + ` current-context: ` + namespace + ` kind: Config -preferences: {} users: - name: ` + namespace + ` user: diff --git a/pkg/gardenlet/operation/botanist/machinecontrollermanager_test.go b/pkg/gardenlet/operation/botanist/machinecontrollermanager_test.go index fb9f17ba939..dc3366c0c01 100644 --- a/pkg/gardenlet/operation/botanist/machinecontrollermanager_test.go +++ b/pkg/gardenlet/operation/botanist/machinecontrollermanager_test.go @@ -18,6 +18,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -54,7 +56,10 @@ var _ = Describe("MachineControllerManager", func() { ctrl = gomock.NewController(GinkgoT()) kubernetesClient = kubernetesmock.NewMockInterface(ctrl) - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder()) + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithObjectTracker(objectTracker).Build() fakeSecretsManager = fakesecretsmanager.New(fakeClient, namespace) shoot = &gardencorev1beta1.Shoot{Spec: gardencorev1beta1.ShootSpec{Kubernetes: gardencorev1beta1.Kubernetes{Version: "1.31.1"}}} diff --git a/pkg/gardenlet/operation/botanist/migration_test.go b/pkg/gardenlet/operation/botanist/migration_test.go index 4739a0b6a0d..b6ec944e9d2 100644 --- a/pkg/gardenlet/operation/botanist/migration_test.go +++ b/pkg/gardenlet/operation/botanist/migration_test.go @@ -17,6 +17,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -67,7 +69,10 @@ var _ = Describe("migration", func() { operatingSystemConfig = mockoperatingsystemconfig.NewMockInterface(ctrl) worker = mockworker.NewMockInterface(ctrl) - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder()) + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithObjectTracker(objectTracker).Build() fakeKubernetesInterface = fakekubernetes.NewClientSetBuilder().WithClient(fakeClient).Build() botanist = &Botanist{Operation: &operation.Operation{ diff --git a/pkg/gardenlet/operation/botanist/secrets.go b/pkg/gardenlet/operation/botanist/secrets.go index 7010338494f..925fa53702d 100644 --- a/pkg/gardenlet/operation/botanist/secrets.go +++ b/pkg/gardenlet/operation/botanist/secrets.go @@ -15,12 +15,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" securityv1alpha1 "github.com/gardener/gardener/pkg/apis/security/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" kubeapiserver "github.com/gardener/gardener/pkg/component/kubernetes/apiserver" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/flow" @@ -523,9 +525,13 @@ func (b *Botanist) DeployCloudProviderSecret(ctx context.Context) error { switch credentials := b.Shoot.Credentials.(type) { case *securityv1alpha1.WorkloadIdentity: shootInfo := b.Shoot.GetInfo() + gvk, err := apiutil.GVKForObject(shootInfo, kubernetes.GardenScheme) + if err != nil { + return err + } shootMeta := securityv1alpha1.ContextObject{ - APIVersion: shootInfo.GroupVersionKind().GroupVersion().String(), - Kind: shootInfo.Kind, + APIVersion: gvk.GroupVersion().String(), + Kind: gvk.Kind, Namespace: ptr.To(shootInfo.Namespace), Name: shootInfo.Name, UID: shootInfo.UID, diff --git a/pkg/nodeagent/bootstrappers/kubelet_bootstrap_kubeconfig_test.go b/pkg/nodeagent/bootstrappers/kubelet_bootstrap_kubeconfig_test.go index bfe33c01fe5..52dc0390296 100644 --- a/pkg/nodeagent/bootstrappers/kubelet_bootstrap_kubeconfig_test.go +++ b/pkg/nodeagent/bootstrappers/kubelet_bootstrap_kubeconfig_test.go @@ -100,7 +100,6 @@ contexts: name: kubelet-bootstrap current-context: kubelet-bootstrap kind: Config -preferences: {} users: - name: kubelet-bootstrap user: diff --git a/pkg/nodeagent/controller/node/add.go b/pkg/nodeagent/controller/node/add.go index 1a285080228..c7dabde8734 100644 --- a/pkg/nodeagent/controller/node/add.go +++ b/pkg/nodeagent/controller/node/add.go @@ -12,6 +12,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/nodeagent/dbus" ) @@ -36,7 +37,10 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, nodePredicate predicate.P ControllerManagedBy(mgr). Named(ControllerName). For(&corev1.Node{}, builder.WithPredicates(r.NodePredicate(), nodePredicate)). - WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 1, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, + }). Complete(r) } diff --git a/pkg/nodeagent/controller/node/reconciler.go b/pkg/nodeagent/controller/node/reconciler.go index aaf87a3cb20..36eaa11627d 100644 --- a/pkg/nodeagent/controller/node/reconciler.go +++ b/pkg/nodeagent/controller/node/reconciler.go @@ -17,7 +17,6 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/gardener/gardener/pkg/controllerutils" nodeagentconfigv1alpha1 "github.com/gardener/gardener/pkg/nodeagent/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/nodeagent/dbus" ) @@ -35,9 +34,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - node := &corev1.Node{} if err := r.Client.Get(ctx, request.NamespacedName, node); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/nodeagent/controller/operatingsystemconfig/add.go b/pkg/nodeagent/controller/operatingsystemconfig/add.go index c0b530ce358..c33e575562d 100644 --- a/pkg/nodeagent/controller/operatingsystemconfig/add.go +++ b/pkg/nodeagent/controller/operatingsystemconfig/add.go @@ -26,6 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" nodeagentconfigv1alpha1 "github.com/gardener/gardener/pkg/nodeagent/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/nodeagent/dbus" @@ -66,7 +67,10 @@ func (r *Reconciler) AddToManager(ctx context.Context, mgr manager.Manager) erro handler.EnqueueRequestsFromMapFunc(r.NodeToSecretMapper()), builder.WithPredicates(r.NodeReadyForUpdate()), ). - WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 1, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, + }). Complete(r) } diff --git a/pkg/nodeagent/controller/operatingsystemconfig/reconciler.go b/pkg/nodeagent/controller/operatingsystemconfig/reconciler.go index 6a314882b56..63d68b18e05 100644 --- a/pkg/nodeagent/controller/operatingsystemconfig/reconciler.go +++ b/pkg/nodeagent/controller/operatingsystemconfig/reconciler.go @@ -46,7 +46,6 @@ import ( extensionsv1alpha1helper "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1/helper" "github.com/gardener/gardener/pkg/client/kubernetes" kubeletcomponent "github.com/gardener/gardener/pkg/component/extensions/operatingsystemconfig/original/components/kubelet" - "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/nodeagent" nodeagentconfigv1alpha1 "github.com/gardener/gardener/pkg/nodeagent/apis/config/v1alpha1" healthcheckcontroller "github.com/gardener/gardener/pkg/nodeagent/controller/healthcheck" @@ -120,11 +119,8 @@ type Reconciler struct { // Reconcile decodes the OperatingSystemConfig resources from secrets and applies the systemd units and files to the // node. -func (r *Reconciler) Reconcile(reconcileCtx context.Context, request reconcile.Request) (reconcile.Result, error) { - log := logf.FromContext(reconcileCtx) - - ctx, cancel := controllerutils.GetMainReconciliationContext(reconcileCtx, controllerutils.DefaultReconciliationTimeout) - defer cancel() +func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log := logf.FromContext(ctx) secret := &corev1.Secret{} if err := r.Client.Get(ctx, request.NamespacedName, secret); err != nil { @@ -186,7 +182,8 @@ func (r *Reconciler) Reconcile(reconcileCtx context.Context, request reconcile.R if isInPlaceUpdate(oscChanges) { // In case of in-place update, we use retries for certain cases like OS update with higher timeouts, // so we need to overwrite the context to use a longer timeout. - ctx, cancel = context.WithTimeout(reconcileCtx, 10*time.Minute) + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 10*time.Minute) defer cancel() if !nodeHasInPlaceUpdateConditionWithReasonReadyForUpdate(node.Status.Conditions) { diff --git a/pkg/nodeagent/controller/operatingsystemconfig/reconciler_test.go b/pkg/nodeagent/controller/operatingsystemconfig/reconciler_test.go index a2df0132da0..06e7e209de4 100644 --- a/pkg/nodeagent/controller/operatingsystemconfig/reconciler_test.go +++ b/pkg/nodeagent/controller/operatingsystemconfig/reconciler_test.go @@ -107,7 +107,6 @@ contexts: name: test-context current-context: test-context kind: Config -preferences: {} ` kubeletCertPath = filepath.Join(kubeletcomponent.PathKubeletDirectory, "pki", "kubelet-client-current.pem") @@ -751,7 +750,6 @@ contexts: name: test-context current-context: test-context kind: Config -preferences: {} users: - name: default-auth user: diff --git a/pkg/nodeagent/controller/token/add.go b/pkg/nodeagent/controller/token/add.go index 184d6f27838..6a8c86a5174 100644 --- a/pkg/nodeagent/controller/token/add.go +++ b/pkg/nodeagent/controller/token/add.go @@ -19,6 +19,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of this controller. @@ -53,7 +55,10 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, channel <-chan event.Type WatchesRawSource( source.TypedChannel(channel, r.EventHandler()), ). - WithOptions(controller.Options{MaxConcurrentReconciles: len(r.Config.SyncConfigs)}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: len(r.Config.SyncConfigs), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, + }). Complete(r) } diff --git a/pkg/nodeagent/controller/token/reconciler.go b/pkg/nodeagent/controller/token/reconciler.go index 6aeed183350..88dc2c58681 100644 --- a/pkg/nodeagent/controller/token/reconciler.go +++ b/pkg/nodeagent/controller/token/reconciler.go @@ -18,7 +18,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" nodeagentconfigv1alpha1 "github.com/gardener/gardener/pkg/nodeagent/apis/config/v1alpha1" ) @@ -35,9 +34,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - secret := &corev1.Secret{} if err := r.APIReader.Get(ctx, request.NamespacedName, secret); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/operator/apis/config/v1alpha1/validation/validation_test.go b/pkg/operator/apis/config/v1alpha1/validation/validation_test.go index ea3c9c1fcca..20fb09e89a5 100644 --- a/pkg/operator/apis/config/v1alpha1/validation/validation_test.go +++ b/pkg/operator/apis/config/v1alpha1/validation/validation_test.go @@ -129,7 +129,7 @@ var _ = Describe("#ValidateOperatorConfiguration", func() { Expect(ValidateOperatorConfiguration(conf)).To(ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), + "Type": Equal(field.ErrorTypeRequired), "Field": Equal("leaderElection.resourceNamespace"), })), )) diff --git a/pkg/operator/controller/extension/care/add.go b/pkg/operator/controller/extension/care/add.go index b60a2ff99a2..db70167b654 100644 --- a/pkg/operator/controller/extension/care/add.go +++ b/pkg/operator/controller/extension/care/add.go @@ -60,6 +60,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, virtualCluster cluster.Cl workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), r.Config.Controllers.ExtensionCare.SyncPeriod.Duration, ), + ReconciliationTimeout: r.Config.Controllers.ExtensionCare.SyncPeriod.Duration, }). Watches( &operatorv1alpha1.Extension{}, diff --git a/pkg/operator/controller/extension/care/reconciler.go b/pkg/operator/controller/extension/care/reconciler.go index 2835aff2b7c..3331e71123d 100644 --- a/pkg/operator/controller/extension/care/reconciler.go +++ b/pkg/operator/controller/extension/care/reconciler.go @@ -36,11 +36,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(reconcileCtx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(reconcileCtx) - // Timeout for all calls (e.g. status updates), give status updates a bit of headroom if health checks - // themselves run into timeouts, so that we will still update the status with that timeout error. - reconcileCtx, cancel := controllerutils.GetMainReconciliationContext(reconcileCtx, r.Config.Controllers.ExtensionCare.SyncPeriod.Duration) - defer cancel() - extension := &operatorv1alpha1.Extension{} if err := r.RuntimeClient.Get(reconcileCtx, request.NamespacedName, extension); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/operator/controller/extension/extension/add.go b/pkg/operator/controller/extension/extension/add.go index f1837c64b0a..02c226cee60 100644 --- a/pkg/operator/controller/extension/extension/add.go +++ b/pkg/operator/controller/extension/extension/add.go @@ -25,6 +25,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/controllerutils/mapper" "github.com/gardener/gardener/pkg/operator/controller/extension/extension/admission" "github.com/gardener/gardener/pkg/operator/controller/extension/extension/controllerregistration" @@ -91,6 +92,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { ))). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.Controllers.Extension.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Watches( &operatorv1alpha1.Garden{}, diff --git a/pkg/operator/controller/extension/extension/reconciler_delete.go b/pkg/operator/controller/extension/extension/reconciler_delete.go index cf85bb96fd2..fa853013281 100644 --- a/pkg/operator/controller/extension/extension/reconciler_delete.go +++ b/pkg/operator/controller/extension/extension/reconciler_delete.go @@ -16,7 +16,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/flow" "github.com/gardener/gardener/pkg/utils/gardener/operator" ) @@ -29,9 +28,6 @@ func (r *Reconciler) delete( reconcile.Result, error, ) { - deleteCtx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - var ( reconcileResult reconcile.Result g = flow.NewGraph("Extension deletion") @@ -62,7 +58,7 @@ func (r *Reconciler) delete( conditions := NewConditions(r.Clock, extension.Status) - if err := g.Compile().Run(deleteCtx, flow.Opts{ + if err := g.Compile().Run(ctx, flow.Opts{ Log: log, }); err != nil { conditions.installed = v1beta1helper.UpdatedConditionWithClock(r.Clock, conditions.installed, gardencorev1beta1.ConditionFalse, ReasonDeleteFailed, err.Error()) diff --git a/pkg/operator/controller/extension/extension/reconciler_reconcile.go b/pkg/operator/controller/extension/extension/reconciler_reconcile.go index 973ce53ce11..931c3269b62 100644 --- a/pkg/operator/controller/extension/extension/reconciler_reconcile.go +++ b/pkg/operator/controller/extension/extension/reconciler_reconcile.go @@ -49,12 +49,9 @@ func (r *Reconciler) reconcile( return reconcile.Result{}, r.updateExtensionStatus(ctx, log, extension, conditions) } - reconcileCtx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - if !controllerutil.ContainsFinalizer(extension, operatorv1alpha1.FinalizerName) { log.Info("Adding finalizer") - if err := controllerutils.AddFinalizers(reconcileCtx, r.RuntimeClientSet.Client(), extension, operatorv1alpha1.FinalizerName); err != nil { + if err := controllerutils.AddFinalizers(ctx, r.RuntimeClientSet.Client(), extension, operatorv1alpha1.FinalizerName); err != nil { return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) } } @@ -120,7 +117,7 @@ func (r *Reconciler) reconcile( }) ) - if err := g.Compile().Run(reconcileCtx, flow.Opts{ + if err := g.Compile().Run(ctx, flow.Opts{ Log: log, }); err != nil { conditions.installed = v1beta1helper.UpdatedConditionWithClock(r.Clock, conditions.installed, gardencorev1beta1.ConditionFalse, ReasonReconcileFailed, err.Error()) diff --git a/pkg/operator/controller/extension/required/runtime/add.go b/pkg/operator/controller/extension/required/runtime/add.go index 7c4507f12c2..8ffd6218dd2 100644 --- a/pkg/operator/controller/extension/required/runtime/add.go +++ b/pkg/operator/controller/extension/required/runtime/add.go @@ -80,6 +80,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { ). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Build(r) if err != nil { diff --git a/pkg/operator/controller/extension/required/runtime/reconciler.go b/pkg/operator/controller/extension/required/runtime/reconciler.go index 81c80a208e8..267fed125bb 100644 --- a/pkg/operator/controller/extension/required/runtime/reconciler.go +++ b/pkg/operator/controller/extension/required/runtime/reconciler.go @@ -22,7 +22,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" operatorconfigv1alpha1 "github.com/gardener/gardener/pkg/operator/apis/config/v1alpha1" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" "github.com/gardener/gardener/pkg/utils/gardener/operator" @@ -45,9 +44,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - for _, ext := range runtimeClusterExtensions { r.Lock.RLock() _, kindProcessed := r.KindToRequiredTypes[ext.objectKind] diff --git a/pkg/operator/controller/extension/required/virtual/add.go b/pkg/operator/controller/extension/required/virtual/add.go index a9b5e75efe8..8c153b15d2b 100644 --- a/pkg/operator/controller/extension/required/virtual/add.go +++ b/pkg/operator/controller/extension/required/virtual/add.go @@ -20,6 +20,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" "github.com/gardener/gardener/pkg/operator/mapper" ) @@ -42,6 +43,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, virtualCluster cluster.Cl Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). For(&operatorv1alpha1.Extension{}, builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create))). WatchesRawSource( diff --git a/pkg/operator/controller/extension/required/virtual/reconciler.go b/pkg/operator/controller/extension/required/virtual/reconciler.go index bb44c79de1b..69b9619edc2 100644 --- a/pkg/operator/controller/extension/required/virtual/reconciler.go +++ b/pkg/operator/controller/extension/required/virtual/reconciler.go @@ -19,7 +19,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" operatorconfigv1alpha1 "github.com/gardener/gardener/pkg/operator/apis/config/v1alpha1" ) @@ -38,9 +37,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - extension := &operatorv1alpha1.Extension{} if err := r.RuntimeClient.Get(ctx, request.NamespacedName, extension); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/operator/controller/garden/care/add.go b/pkg/operator/controller/garden/care/add.go index 155e650fa56..1a9cc22edf8 100644 --- a/pkg/operator/controller/garden/care/add.go +++ b/pkg/operator/controller/garden/care/add.go @@ -58,6 +58,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenClientMap clientmap workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), r.Config.Controllers.GardenCare.SyncPeriod.Duration, ), + ReconciliationTimeout: r.Config.Controllers.GardenCare.SyncPeriod.Duration, }). Watches( &operatorv1alpha1.Garden{}, diff --git a/pkg/operator/controller/garden/care/reconciler.go b/pkg/operator/controller/garden/care/reconciler.go index 4e37667d79e..37d5f206d59 100644 --- a/pkg/operator/controller/garden/care/reconciler.go +++ b/pkg/operator/controller/garden/care/reconciler.go @@ -46,11 +46,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(reconcileCtx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(reconcileCtx) - // Timeout for all calls (e.g. status updates), give status updates a bit of headroom if health checks - // themselves run into timeouts, so that we will still update the status with that timeout error. - reconcileCtx, cancel := controllerutils.GetMainReconciliationContext(reconcileCtx, r.Config.Controllers.GardenCare.SyncPeriod.Duration) - defer cancel() - garden := &operatorv1alpha1.Garden{} if err := r.RuntimeClient.Get(reconcileCtx, req.NamespacedName, garden); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/operator/controller/gardenlet/add.go b/pkg/operator/controller/gardenlet/add.go index 1881cf61800..c03be1c5c20 100644 --- a/pkg/operator/controller/gardenlet/add.go +++ b/pkg/operator/controller/gardenlet/add.go @@ -24,6 +24,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" seedmanagementv1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" gardenletutils "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" "github.com/gardener/gardener/pkg/utils/oci" @@ -67,6 +68,7 @@ func (r *Reconciler) AddToManager(ctx context.Context, mgr manager.Manager, virt Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: pointer.IntDeref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource(source.Kind[client.Object](virtualCluster.GetCache(), &seedmanagementv1alpha1.Gardenlet{}, &handler.EnqueueRequestForObject{}, diff --git a/pkg/operator/controller/gardenlet/reconciler.go b/pkg/operator/controller/gardenlet/reconciler.go index 3f2e86cf85b..d9312e5b4ec 100644 --- a/pkg/operator/controller/gardenlet/reconciler.go +++ b/pkg/operator/controller/gardenlet/reconciler.go @@ -28,7 +28,6 @@ import ( "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1/helper" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/controller/gardenletdeployer" - "github.com/gardener/gardener/pkg/controllerutils" operatorconfigv1alpha1 "github.com/gardener/gardener/pkg/operator/apis/config/v1alpha1" gardenletutils "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" @@ -57,9 +56,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - gardenlet := &seedmanagementv1alpha1.Gardenlet{} if err := r.VirtualClient.Get(ctx, request.NamespacedName, gardenlet); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/resourcemanager/apis/config/v1alpha1/validation/validation_test.go b/pkg/resourcemanager/apis/config/v1alpha1/validation/validation_test.go index 355fb1fe3bf..8a43f7f2c7e 100644 --- a/pkg/resourcemanager/apis/config/v1alpha1/validation/validation_test.go +++ b/pkg/resourcemanager/apis/config/v1alpha1/validation/validation_test.go @@ -140,7 +140,7 @@ var _ = Describe("Validation", func() { Expect(ValidateResourceManagerConfiguration(conf)).To(ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), + "Type": Equal(field.ErrorTypeRequired), "Field": Equal("leaderElection.resourceNamespace"), })), )) diff --git a/pkg/resourcemanager/controller/csrapprover/add.go b/pkg/resourcemanager/controller/csrapprover/add.go index 0f26cc0554f..684e591ae23 100644 --- a/pkg/resourcemanager/controller/csrapprover/add.go +++ b/pkg/resourcemanager/controller/csrapprover/add.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/source" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -36,6 +37,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, sourceCluster, targetClus Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource( source.Kind[client.Object](targetCluster.GetCache(), diff --git a/pkg/resourcemanager/controller/csrapprover/reconciler.go b/pkg/resourcemanager/controller/csrapprover/reconciler.go index 755fdd05db8..ff789560783 100644 --- a/pkg/resourcemanager/controller/csrapprover/reconciler.go +++ b/pkg/resourcemanager/controller/csrapprover/reconciler.go @@ -26,7 +26,6 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/client/kubernetes" - "github.com/gardener/gardener/pkg/controllerutils" resourcemanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/utils" ) @@ -51,9 +50,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - csr := &certificatesv1.CertificateSigningRequest{} if err := r.TargetClient.Get(ctx, request.NamespacedName, csr); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/resourcemanager/controller/garbagecollector/add.go b/pkg/resourcemanager/controller/garbagecollector/add.go index 55078ea5851..a881b6b8d12 100644 --- a/pkg/resourcemanager/controller/garbagecollector/add.go +++ b/pkg/resourcemanager/controller/garbagecollector/add.go @@ -33,6 +33,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, targetCluster cluster.Clu Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: 1, + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). WatchesRawSource(controllerutils.EnqueueOnce). Complete(r) diff --git a/pkg/resourcemanager/controller/garbagecollector/reconciler.go b/pkg/resourcemanager/controller/garbagecollector/reconciler.go index ee9a39eca09..005ff64243d 100644 --- a/pkg/resourcemanager/controller/garbagecollector/reconciler.go +++ b/pkg/resourcemanager/controller/garbagecollector/reconciler.go @@ -26,7 +26,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" resourcemanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references" errorsutils "github.com/gardener/gardener/pkg/utils/errors" @@ -41,11 +40,8 @@ type Reconciler struct { } // Reconcile performs the main reconciliation logic. -func (r *Reconciler) Reconcile(reconcileCtx context.Context, _ reconcile.Request) (reconcile.Result, error) { - log := logf.FromContext(reconcileCtx) - - ctx, cancel := controllerutils.GetMainReconciliationContext(reconcileCtx, r.Config.SyncPeriod.Duration) - defer cancel() +func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + log := logf.FromContext(ctx) log.Info("Starting garbage collection") defer log.Info("Garbage collection finished") diff --git a/pkg/resourcemanager/controller/health/health/add.go b/pkg/resourcemanager/controller/health/health/add.go index 99c93f7cd02..3f3ebb87749 100644 --- a/pkg/resourcemanager/controller/health/health/add.go +++ b/pkg/resourcemanager/controller/health/health/add.go @@ -55,6 +55,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, sourceCluster, targetClus Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). Watches( &resourcesv1alpha1.ManagedResource{}, diff --git a/pkg/resourcemanager/controller/health/health/reconciler.go b/pkg/resourcemanager/controller/health/health/reconciler.go index 55cfbf45a1c..3ace7860199 100644 --- a/pkg/resourcemanager/controller/health/health/reconciler.go +++ b/pkg/resourcemanager/controller/health/health/reconciler.go @@ -47,12 +47,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - // timeout for all calls (e.g. status updates), give status updates a bit of headroom if health checks - // themselves run into timeouts, so that we will still update the status with that timeout error - var cancel context.CancelFunc - ctx, cancel = controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() - mr := &resourcesv1alpha1.ManagedResource{} if err := r.SourceClient.Get(ctx, req.NamespacedName, mr); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/resourcemanager/controller/health/progressing/add.go b/pkg/resourcemanager/controller/health/progressing/add.go index d9f9f2497ec..0d52d7559e7 100644 --- a/pkg/resourcemanager/controller/health/progressing/add.go +++ b/pkg/resourcemanager/controller/health/progressing/add.go @@ -65,6 +65,7 @@ func (r *Reconciler) AddToManager(ctx context.Context, mgr manager.Manager, sour )). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). Build(r) if err != nil { diff --git a/pkg/resourcemanager/controller/health/progressing/reconciler.go b/pkg/resourcemanager/controller/health/progressing/reconciler.go index 2cca0888961..fb22728f3f0 100644 --- a/pkg/resourcemanager/controller/health/progressing/reconciler.go +++ b/pkg/resourcemanager/controller/health/progressing/reconciler.go @@ -44,12 +44,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - // timeout for all calls (e.g. status updates), give status updates a bit of headroom if checks - // themselves run into timeouts, so that we will still update the status with that timeout error - var cancel context.CancelFunc - ctx, cancel = controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() - mr := &resourcesv1alpha1.ManagedResource{} if err := r.SourceClient.Get(ctx, req.NamespacedName, mr); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/resourcemanager/controller/managedresource/add.go b/pkg/resourcemanager/controller/managedresource/add.go index fe41a3bbc94..175176a7e83 100644 --- a/pkg/resourcemanager/controller/managedresource/add.go +++ b/pkg/resourcemanager/controller/managedresource/add.go @@ -56,6 +56,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, sourceCluster, targetClus Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, }). For(&resourcesv1alpha1.ManagedResource{}, builder.WithPredicates( r.ClassFilter, diff --git a/pkg/resourcemanager/controller/managedresource/reconciler.go b/pkg/resourcemanager/controller/managedresource/reconciler.go index 1b54a43cdc9..c3d61bae472 100644 --- a/pkg/resourcemanager/controller/managedresource/reconciler.go +++ b/pkg/resourcemanager/controller/managedresource/reconciler.go @@ -151,15 +151,12 @@ func (r *Reconciler) reconcile(ctx context.Context, log logr.Logger, mr *resourc forceOverwriteAnnotations = *v } - reconcileCtx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() - // Initialize condition based on the current status. conditionResourcesApplied := v1beta1helper.GetOrInitConditionWithClock(r.Clock, mr.Status.Conditions, resourcesv1alpha1.ResourcesApplied) for _, ref := range mr.Spec.SecretRefs { secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: ref.Name, Namespace: mr.Namespace}} - if err := r.SourceClient.Get(reconcileCtx, client.ObjectKeyFromObject(secret), secret); err != nil { + if err := r.SourceClient.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { conditionResourcesApplied = v1beta1helper.UpdatedConditionWithClock(r.Clock, conditionResourcesApplied, gardencorev1beta1.ConditionFalse, "CannotReadSecret", err.Error()) if err := updateConditions(ctx, r.SourceClient, mr, conditionResourcesApplied); err != nil { return reconcile.Result{}, fmt.Errorf("could not update the ManagedResource status: %w", err) @@ -311,7 +308,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log logr.Logger, mr *resourc } } - if deletionPending, err := r.cleanOldResources(reconcileCtx, log, mr, existingResourcesIndex); err != nil { + if deletionPending, err := r.cleanOldResources(ctx, log, mr, existingResourcesIndex); err != nil { var ( reason string status gardencorev1beta1.ConditionStatus @@ -348,7 +345,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log logr.Logger, mr *resourc } injectLabels := mergeMaps(mr.Spec.InjectLabels, map[string]string{resourcesv1alpha1.ManagedBy: *r.Config.ManagedByLabelValue}) - if err := r.applyNewResources(reconcileCtx, log, origin, newResourcesObjects, injectLabels, equivalences); err != nil { + if err := r.applyNewResources(ctx, log, origin, newResourcesObjects, injectLabels, equivalences); err != nil { conditionResourcesApplied = v1beta1helper.UpdatedConditionWithClock(r.Clock, conditionResourcesApplied, gardencorev1beta1.ConditionFalse, resourcesv1alpha1.ConditionApplyFailed, err.Error()) if err := updateConditions(ctx, r.SourceClient, mr, conditionResourcesApplied); err != nil { return reconcile.Result{}, fmt.Errorf("could not update the ManagedResource status: %w", err) @@ -374,9 +371,6 @@ func (r *Reconciler) reconcile(ctx context.Context, log logr.Logger, mr *resourc func (r *Reconciler) delete(ctx context.Context, log logr.Logger, mr *resourcesv1alpha1.ManagedResource) (reconcile.Result, error) { log.Info("Started deleting resources created by ManagedResource") - deleteCtx, cancel := controllerutils.GetMainReconciliationContext(ctx, r.Config.SyncPeriod.Duration) - defer cancel() - if err := r.updateConditionsForDeletion(ctx, mr); err != nil { return reconcile.Result{}, fmt.Errorf("could not update the ManagedResource status: %w", err) } @@ -397,7 +391,7 @@ func (r *Reconciler) delete(ctx context.Context, log logr.Logger, mr *resourcesv return reconcile.Result{}, fmt.Errorf("could not update the ManagedResource status: %w", err) } - if deletionPending, err := r.cleanOldResources(deleteCtx, log, mr, existingResourcesIndex); err != nil { + if deletionPending, err := r.cleanOldResources(ctx, log, mr, existingResourcesIndex); err != nil { var ( reason string status gardencorev1beta1.ConditionStatus diff --git a/pkg/resourcemanager/controller/networkpolicy/add.go b/pkg/resourcemanager/controller/networkpolicy/add.go index 4fd1345383a..e7fecafa665 100644 --- a/pkg/resourcemanager/controller/networkpolicy/add.go +++ b/pkg/resourcemanager/controller/networkpolicy/add.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils" ) // ControllerName is the name of the controller. @@ -75,6 +76,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, targetCluster cluster.Clu Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource(source.Kind[client.Object]( targetCluster.GetCache(), diff --git a/pkg/resourcemanager/controller/networkpolicy/reconciler.go b/pkg/resourcemanager/controller/networkpolicy/reconciler.go index 934d16755cf..9f4fca02fc7 100644 --- a/pkg/resourcemanager/controller/networkpolicy/reconciler.go +++ b/pkg/resourcemanager/controller/networkpolicy/reconciler.go @@ -47,9 +47,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - networkPolicyList := &metav1.PartialObjectMetadataList{} networkPolicyList.SetGroupVersionKind(networkingv1.SchemeGroupVersion.WithKind("NetworkPolicyList")) if err := r.TargetClient.List(ctx, networkPolicyList, client.MatchingLabels{ diff --git a/pkg/resourcemanager/controller/node/agentreconciliationdelay/add.go b/pkg/resourcemanager/controller/node/agentreconciliationdelay/add.go index fd7d869d083..cc52d0a1ee6 100644 --- a/pkg/resourcemanager/controller/node/agentreconciliationdelay/add.go +++ b/pkg/resourcemanager/controller/node/agentreconciliationdelay/add.go @@ -29,7 +29,10 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, targetCluster cluster.Clu return builder. ControllerManagedBy(mgr). Named(ControllerName). - WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 1, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, + }). WatchesRawSource( source.Kind[client.Object](targetCluster.GetCache(), &corev1.Node{}, diff --git a/pkg/resourcemanager/controller/node/agentreconciliationdelay/reconciler.go b/pkg/resourcemanager/controller/node/agentreconciliationdelay/reconciler.go index bd5a00bc4bc..0c05342f9f3 100644 --- a/pkg/resourcemanager/controller/node/agentreconciliationdelay/reconciler.go +++ b/pkg/resourcemanager/controller/node/agentreconciliationdelay/reconciler.go @@ -17,7 +17,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/gardener/gardener/pkg/controllerutils" resourcemanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/utils/flow" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" @@ -34,11 +33,8 @@ type Reconciler struct { // Reconcile computes a time.Duration that can be used to delay reconciliations by using a simple linear mapping // approach based on the indices of the nodes in the list of all nodes in the cluster. This way, the delays of all // instances of gardener-node-agent are distributed evenly. -func (r *Reconciler) Reconcile(reconcileCtx context.Context, _ reconcile.Request) (reconcile.Result, error) { - log := logf.FromContext(reconcileCtx) - - ctx, cancel := controllerutils.GetMainReconciliationContext(reconcileCtx, controllerutils.DefaultReconciliationTimeout) - defer cancel() +func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + log := logf.FromContext(ctx) nodeList := &corev1.NodeList{} if err := r.TargetClient.List(ctx, nodeList); err != nil { diff --git a/pkg/resourcemanager/controller/node/criticalcomponents/add.go b/pkg/resourcemanager/controller/node/criticalcomponents/add.go index e037be77bba..68aedcd3dd7 100644 --- a/pkg/resourcemanager/controller/node/criticalcomponents/add.go +++ b/pkg/resourcemanager/controller/node/criticalcomponents/add.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -37,6 +38,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, targetCluster cluster.Clu Named(ControllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). WatchesRawSource( source.Kind[client.Object](targetCluster.GetCache(), diff --git a/pkg/resourcemanager/controller/node/criticalcomponents/reconciler.go b/pkg/resourcemanager/controller/node/criticalcomponents/reconciler.go index 873aa8d7f5a..428fe7bafca 100644 --- a/pkg/resourcemanager/controller/node/criticalcomponents/reconciler.go +++ b/pkg/resourcemanager/controller/node/criticalcomponents/reconciler.go @@ -25,7 +25,6 @@ import ( "github.com/gardener/gardener/pkg/api/indexer" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/gardener/gardener/pkg/controllerutils" resourcemanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/resourcemanager/controller/node/criticalcomponents/helper" "github.com/gardener/gardener/pkg/utils/kubernetes/health" @@ -40,11 +39,8 @@ type Reconciler struct { } // Reconcile checks if the critical components not ready taint can be removed from the Node object. -func (r *Reconciler) Reconcile(reconcileCtx context.Context, req reconcile.Request) (reconcile.Result, error) { - log := logf.FromContext(reconcileCtx) - - ctx, cancel := controllerutils.GetMainReconciliationContext(reconcileCtx, controllerutils.DefaultReconciliationTimeout) - defer cancel() +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + log := logf.FromContext(ctx) node := &corev1.Node{} if err := r.TargetClient.Get(ctx, req.NamespacedName, node); err != nil { diff --git a/pkg/resourcemanager/webhook/nodeagentauthorizer/authorizer_test.go b/pkg/resourcemanager/webhook/nodeagentauthorizer/authorizer_test.go index c2fe1f2ec00..c43d5f686dd 100644 --- a/pkg/resourcemanager/webhook/nodeagentauthorizer/authorizer_test.go +++ b/pkg/resourcemanager/webhook/nodeagentauthorizer/authorizer_test.go @@ -19,6 +19,8 @@ import ( "k8s.io/apimachinery/pkg/selection" "k8s.io/apiserver/pkg/authentication/user" auth "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" logzap "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -55,7 +57,10 @@ var _ = Describe("Authorizer", func() { ctx = context.Background() log = logger.MustNewZapLogger(logger.DebugLevel, logger.FormatJSON, logzap.WriteTo(GinkgoWriter)) - sourceClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder()) + sourceClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithObjectTracker(objectTracker).Build() targetClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.ShootScheme).Build() machineNamespace = "shoot--foo" authorizer = NewAuthorizer(log, sourceClient, targetClient, &machineNamespace, true) diff --git a/pkg/scheduler/apis/config/v1alpha1/validation/validation_test.go b/pkg/scheduler/apis/config/v1alpha1/validation/validation_test.go index b31c7af8960..1a9f3926eb2 100644 --- a/pkg/scheduler/apis/config/v1alpha1/validation/validation_test.go +++ b/pkg/scheduler/apis/config/v1alpha1/validation/validation_test.go @@ -98,7 +98,7 @@ var _ = Describe("#ValidateConfiguration", func() { Expect(ValidateConfiguration(conf)).To(ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), + "Type": Equal(field.ErrorTypeRequired), "Field": Equal("leaderElection.resourceNamespace"), })), )) diff --git a/pkg/scheduler/controller/shoot/add.go b/pkg/scheduler/controller/shoot/add.go index c81e32cd207..39e17b64d4c 100644 --- a/pkg/scheduler/controller/shoot/add.go +++ b/pkg/scheduler/controller/shoot/add.go @@ -17,6 +17,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/controllerutils" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" ) @@ -46,6 +47,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { )). WithOptions(controller.Options{ MaxConcurrentReconciles: r.Config.ConcurrentSyncs, + ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). Complete(r) } diff --git a/pkg/scheduler/controller/shoot/reconciler.go b/pkg/scheduler/controller/shoot/reconciler.go index 7d4bac36ead..38470648b86 100644 --- a/pkg/scheduler/controller/shoot/reconciler.go +++ b/pkg/scheduler/controller/shoot/reconciler.go @@ -28,7 +28,6 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" - "github.com/gardener/gardener/pkg/controllerutils" schedulerconfigv1alpha1 "github.com/gardener/gardener/pkg/scheduler/apis/config/v1alpha1" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" cidrvalidation "github.com/gardener/gardener/pkg/utils/validation/cidr" @@ -46,9 +45,6 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) - defer cancel() - shoot := &gardencorev1beta1.Shoot{} if err := r.Client.Get(ctx, request.NamespacedName, shoot); err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/utils/gardener/machines_test.go b/pkg/utils/gardener/machines_test.go index e30e9f37bfd..2be9fdc4adf 100644 --- a/pkg/utils/gardener/machines_test.go +++ b/pkg/utils/gardener/machines_test.go @@ -13,6 +13,8 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -153,7 +155,10 @@ var _ = Describe("Machines", func() { BeforeEach(func() { fakeOps = &retryfake.Ops{MaxAttempts: 1} - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder()) + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithObjectTracker(objectTracker).Build() DeferCleanup(test.WithVars( &retry.Until, fakeOps.Until, diff --git a/pkg/utils/gardener/shootstate/shootstate_test.go b/pkg/utils/gardener/shootstate/shootstate_test.go index d9266760295..a3afcf61b94 100644 --- a/pkg/utils/gardener/shootstate/shootstate_test.go +++ b/pkg/utils/gardener/shootstate/shootstate_test.go @@ -15,6 +15,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" "k8s.io/utils/clock" testclock "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" @@ -45,7 +47,10 @@ var _ = Describe("ShootState", func() { BeforeEach(func() { fakeGardenClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).Build() - fakeSeedClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + // Starting with controller-runtime v0.22.0, the default object tracker does not work with resources which include + // structs directly as pointer, e.g. *MachineConfiguration in Machine resource. Hence, use the old one instead. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, scheme.Codecs.UniversalDecoder()) + fakeSeedClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithObjectTracker(objectTracker).Build() fakeClock = testclock.NewFakeClock(time.Now()) shoot = &gardencorev1beta1.Shoot{ @@ -128,7 +133,7 @@ var _ = Describe("ShootState", func() { { Name: "machine-state", Type: "machine-state", - Data: runtime.RawExtension{Raw: []byte(`{"state":"H4sIAAAAAAAC/+yUvW7jMAzH34WzPNjJLV4vy003JDcdMrASgbjQFySmQBDo3QvJdpKiKJo6CRC08WKJpEj6R/m/B4Ny01lakNduZ8hyhHYPqmzrvAzkdScxQjsTY/SSctj/PRhiVMiYAy0agnY8WkXiGkSxRo8yu+LGOa4qs6t8cM8k+3WxggAZCLlzdtUZiozGQ2u3WgtAax0XV+ktulLnaIWUBERPMnuHDn9rjDk6CWAyXiNT8Z70+0G9k1zymMQ6RaupiVJ+BERG3pZP0Bj5r6fQtz8Y/nmFTDnJcD4l8Rnh5jqENT6Rju9LQPrJ9NeH+37OZW+qIfjeZnLC0Qf30ikKfxbQHjZVp+ByxF8HLEBuQyDLyzcHp/wG9ci+vjp7p3Ke/CplL4B/z4TXSQw8m+ma3+RRzB6afzvNL4TnN9WX5qH5Z2l+P4pRd37d2Uy+t+b3QjOynz80f6Lmp/QaAAD//31qkngADAAA"}`)}, + Data: runtime.RawExtension{Raw: []byte(`{"state":"H4sIAAAAAAAC/+yUPWv7MBDGv8vN8mAn/8XrP0unDkmnkuEqHcRFb0iXQgj67kWynaS0kJcWUtJ4sXR6Tjz80D1bMChXnaUZee02hixHaLegyrbOy0BedxIjtBMxqueUZc9bMMSokDELLRqCdmytInENolSjR5mP4so5riqzqXxwryT7damCALTWMXLnbLEQXbluX4WUBERPMp8ORv5rjFmdBDAZr5GpnO5tHfTIvdg6RYtjDSl/AiIjr4sljZEfPYXezlB48gqZFl12a9da555jYJqzwGh8IR0/3wTplqAtd6/rlKfVVIP4SigPsPjg3jpF4WEG7W5TdQpOJ3Y+LwFyHQJZnn9ovOQx1iPK+lKUTmV5/pXbv8HyNwBbJjHgaS7PvyaTndzz72sw058Y2uaP5l9PcBzaf9dBeRv510/piHJ6z78+/1J6DwAA//+Ak3pfGAoAAA=="}`)}, }, }, Extensions: []gardencorev1beta1.ExtensionResourceState{ diff --git a/pkg/utils/managedresources/registry_test.go b/pkg/utils/managedresources/registry_test.go index 5db71609962..d2308d0d7e9 100644 --- a/pkg/utils/managedresources/registry_test.go +++ b/pkg/utils/managedresources/registry_test.go @@ -62,7 +62,6 @@ metadata: foo.bar/test-07679f5e: "5" foo.bar/test-d2718f1d: "6" foo.bar/test-ea8edc28: "7" - creationTimestamp: null labels: foo.bar/test-9dca243c: "1" foo.bar/test-47fc132b: "2" @@ -84,7 +83,6 @@ metadata: roleBindingSerialized = `apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - creationTimestamp: null name: ` + roleBinding.Name + ` namespace: ` + roleBinding.Namespace + ` roleRef: diff --git a/pkg/utils/validation/componentbaseconfig_test.go b/pkg/utils/validation/componentbaseconfig_test.go index 4b497844b0d..89356c433a6 100644 --- a/pkg/utils/validation/componentbaseconfig_test.go +++ b/pkg/utils/validation/componentbaseconfig_test.go @@ -92,7 +92,7 @@ var _ = Describe("componentbaseconfig validation helpers", func() { Expect(ValidateLeaderElectionConfiguration(config, fldPath)).To(ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), + "Type": Equal(field.ErrorTypeRequired), "Field": Equal(fldPath.Child("resourceNamespace").String()), })), )) diff --git a/pkg/webhook/authorizer/handler_test.go b/pkg/webhook/authorizer/handler_test.go index 600abf9aaae..24169438b1a 100644 --- a/pkg/webhook/authorizer/handler_test.go +++ b/pkg/webhook/authorizer/handler_test.go @@ -45,7 +45,7 @@ var _ = Describe("Handler", func() { handler.ServeHTTP(respRecorder, req) - Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{"creationTimestamp":null},"spec":{},"status":{"allowed":false,"evaluationError":"422 request body is empty"}} + Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{},"spec":{},"status":{"allowed":false,"evaluationError":"422 request body is empty"}} `)) }) @@ -57,7 +57,7 @@ var _ = Describe("Handler", func() { handler.ServeHTTP(respRecorder, req) - Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{"creationTimestamp":null},"spec":{},"status":{"allowed":false,"evaluationError":"400 fake-err"}} + Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{},"spec":{},"status":{"allowed":false,"evaluationError":"400 fake-err"}} `)) }) @@ -69,7 +69,7 @@ var _ = Describe("Handler", func() { handler.ServeHTTP(respRecorder, req) - Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{"creationTimestamp":null},"spec":{},"status":{"allowed":false,"evaluationError":"400 contentType=foo, expected application/json"}} + Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{},"spec":{},"status":{"allowed":false,"evaluationError":"400 contentType=foo, expected application/json"}} `)) }) @@ -81,7 +81,7 @@ var _ = Describe("Handler", func() { handler.ServeHTTP(respRecorder, req) - Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{"creationTimestamp":null},"spec":{},"status":{"allowed":false,"evaluationError":"400 couldn't get version/kind; json parse error: unexpected end of JSON input"}} + Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{},"spec":{},"status":{"allowed":false,"evaluationError":"400 couldn't get version/kind; json parse error: unexpected end of JSON input"}} `)) }) @@ -97,7 +97,7 @@ var _ = Describe("Handler", func() { handler = &Handler{Logger: log, Authorizer: &fakeAuthorizer{fn: fn}} handler.ServeHTTP(respRecorder, req) - Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{"creationTimestamp":null},"spec":{},"status":{` + expectedStatus + `}} + Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1","metadata":{},"spec":{},"status":{` + expectedStatus + `}} `)) }, @@ -117,7 +117,7 @@ var _ = Describe("Handler", func() { handler.ServeHTTP(respRecorder, req) - Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","metadata":{"creationTimestamp":null},"spec":{},"status":{"allowed":true}} + Expect(respRecorder.Body.String()).To(Equal(`{"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","metadata":{},"spec":{},"status":{"allowed":true}} `)) }) }) diff --git a/plugin/pkg/managedseed/validator/admission_test.go b/plugin/pkg/managedseed/validator/admission_test.go index ed65c1fc8aa..80e29e6b536 100644 --- a/plugin/pkg/managedseed/validator/admission_test.go +++ b/plugin/pkg/managedseed/validator/admission_test.go @@ -692,7 +692,7 @@ var _ = Describe("ManagedSeed", func() { PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.gardenlet.config.seedConfig.spec.provider.zones"), - "Detail": ContainSubstring("[]string{\"foo\", \"bar\"}: cannot use zone in seed provider that is not available in referenced shoot"), + "Detail": ContainSubstring("[\"foo\",\"bar\"]: cannot use zone in seed provider that is not available in referenced shoot"), })), )) }) @@ -911,7 +911,7 @@ var _ = Describe("ManagedSeed", func() { PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.gardenlet.config.seedConfig.spec.provider.zones"), - "Detail": ContainSubstring("[]string{\"zone-foobar\"}: added zones must match zone names configured for workers in the referenced shoot cluster"), + "Detail": ContainSubstring("[\"zone-foobar\"]: added zones must match zone names configured for workers in the referenced shoot cluster"), })), )) }) diff --git a/test/integration/nodeagent/operatingsystemconfig/operatingsystemconfig_test.go b/test/integration/nodeagent/operatingsystemconfig/operatingsystemconfig_test.go index b2d3e8faa66..01e9ec7c686 100644 --- a/test/integration/nodeagent/operatingsystemconfig/operatingsystemconfig_test.go +++ b/test/integration/nodeagent/operatingsystemconfig/operatingsystemconfig_test.go @@ -500,7 +500,7 @@ inPlaceUpdates: operatingSystem: false serviceAccountKeyRotation: false mustRestartNodeAgent: false -operatingSystemConfigChecksum: 9f99a06a84314322dfbe80920a71938f4ac301874fc0e21c8fa5ad0d09baa98c +operatingSystemConfigChecksum: ddafef1ed407f75f5fc6a8a075ff430fb03b348a4c00474978e2650a22edd9ff units: {} `, 0600) @@ -1238,7 +1238,6 @@ contexts: name: test-context current-context: test-context kind: Config -preferences: {} users: - name: default-auth user: diff --git a/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_certificaterevocations.yaml b/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_certificaterevocations.yaml index 21e9267458b..2d859cef082 100644 --- a/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_certificaterevocations.yaml +++ b/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_certificaterevocations.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: certificaterevocations.cert.gardener.cloud spec: group: cert.gardener.cloud diff --git a/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_certificates.yaml b/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_certificates.yaml index a8d2b94484f..838d9439e8d 100644 --- a/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_certificates.yaml +++ b/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_certificates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: certificates.cert.gardener.cloud spec: group: cert.gardener.cloud diff --git a/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_issuers.yaml b/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_issuers.yaml index 31cbd8a8845..d0fe1d61589 100644 --- a/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_issuers.yaml +++ b/test/integration/resourcemanager/health/crds/10-crd-cert.gardener.cloud_issuers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.19.0 name: issuers.cert.gardener.cloud spec: group: cert.gardener.cloud diff --git a/third_party/mock/client-go/kubernetes/mocks.go b/third_party/mock/client-go/kubernetes/mocks.go index 8b375db1c96..8b515ee3a20 100644 --- a/third_party/mock/client-go/kubernetes/mocks.go +++ b/third_party/mock/client-go/kubernetes/mocks.go @@ -49,26 +49,26 @@ import ( v1beta20 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2" v1beta3 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3" v111 "k8s.io/client-go/kubernetes/typed/networking/v1" - v1alpha13 "k8s.io/client-go/kubernetes/typed/networking/v1alpha1" v1beta110 "k8s.io/client-go/kubernetes/typed/networking/v1beta1" v112 "k8s.io/client-go/kubernetes/typed/node/v1" - v1alpha14 "k8s.io/client-go/kubernetes/typed/node/v1alpha1" + v1alpha13 "k8s.io/client-go/kubernetes/typed/node/v1alpha1" v1beta111 "k8s.io/client-go/kubernetes/typed/node/v1beta1" v113 "k8s.io/client-go/kubernetes/typed/policy/v1" v1beta112 "k8s.io/client-go/kubernetes/typed/policy/v1beta1" v114 "k8s.io/client-go/kubernetes/typed/rbac/v1" - v1alpha15 "k8s.io/client-go/kubernetes/typed/rbac/v1alpha1" + v1alpha14 "k8s.io/client-go/kubernetes/typed/rbac/v1alpha1" v1beta113 "k8s.io/client-go/kubernetes/typed/rbac/v1beta1" + v115 "k8s.io/client-go/kubernetes/typed/resource/v1" v1alpha3 "k8s.io/client-go/kubernetes/typed/resource/v1alpha3" v1beta114 "k8s.io/client-go/kubernetes/typed/resource/v1beta1" v1beta21 "k8s.io/client-go/kubernetes/typed/resource/v1beta2" - v115 "k8s.io/client-go/kubernetes/typed/scheduling/v1" - v1alpha16 "k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1" + v116 "k8s.io/client-go/kubernetes/typed/scheduling/v1" + v1alpha15 "k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1" v1beta115 "k8s.io/client-go/kubernetes/typed/scheduling/v1beta1" - v116 "k8s.io/client-go/kubernetes/typed/storage/v1" - v1alpha17 "k8s.io/client-go/kubernetes/typed/storage/v1alpha1" + v117 "k8s.io/client-go/kubernetes/typed/storage/v1" + v1alpha16 "k8s.io/client-go/kubernetes/typed/storage/v1alpha1" v1beta116 "k8s.io/client-go/kubernetes/typed/storage/v1beta1" - v1alpha18 "k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1" + v1alpha17 "k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1" ) // MockInterface is a mock of Interface interface. @@ -599,20 +599,6 @@ func (mr *MockInterfaceMockRecorder) NetworkingV1() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkingV1", reflect.TypeOf((*MockInterface)(nil).NetworkingV1)) } -// NetworkingV1alpha1 mocks base method. -func (m *MockInterface) NetworkingV1alpha1() v1alpha13.NetworkingV1alpha1Interface { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NetworkingV1alpha1") - ret0, _ := ret[0].(v1alpha13.NetworkingV1alpha1Interface) - return ret0 -} - -// NetworkingV1alpha1 indicates an expected call of NetworkingV1alpha1. -func (mr *MockInterfaceMockRecorder) NetworkingV1alpha1() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkingV1alpha1", reflect.TypeOf((*MockInterface)(nil).NetworkingV1alpha1)) -} - // NetworkingV1beta1 mocks base method. func (m *MockInterface) NetworkingV1beta1() v1beta110.NetworkingV1beta1Interface { m.ctrl.T.Helper() @@ -642,10 +628,10 @@ func (mr *MockInterfaceMockRecorder) NodeV1() *gomock.Call { } // NodeV1alpha1 mocks base method. -func (m *MockInterface) NodeV1alpha1() v1alpha14.NodeV1alpha1Interface { +func (m *MockInterface) NodeV1alpha1() v1alpha13.NodeV1alpha1Interface { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NodeV1alpha1") - ret0, _ := ret[0].(v1alpha14.NodeV1alpha1Interface) + ret0, _ := ret[0].(v1alpha13.NodeV1alpha1Interface) return ret0 } @@ -712,10 +698,10 @@ func (mr *MockInterfaceMockRecorder) RbacV1() *gomock.Call { } // RbacV1alpha1 mocks base method. -func (m *MockInterface) RbacV1alpha1() v1alpha15.RbacV1alpha1Interface { +func (m *MockInterface) RbacV1alpha1() v1alpha14.RbacV1alpha1Interface { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RbacV1alpha1") - ret0, _ := ret[0].(v1alpha15.RbacV1alpha1Interface) + ret0, _ := ret[0].(v1alpha14.RbacV1alpha1Interface) return ret0 } @@ -739,6 +725,20 @@ func (mr *MockInterfaceMockRecorder) RbacV1beta1() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RbacV1beta1", reflect.TypeOf((*MockInterface)(nil).RbacV1beta1)) } +// ResourceV1 mocks base method. +func (m *MockInterface) ResourceV1() v115.ResourceV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResourceV1") + ret0, _ := ret[0].(v115.ResourceV1Interface) + return ret0 +} + +// ResourceV1 indicates an expected call of ResourceV1. +func (mr *MockInterfaceMockRecorder) ResourceV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourceV1", reflect.TypeOf((*MockInterface)(nil).ResourceV1)) +} + // ResourceV1alpha3 mocks base method. func (m *MockInterface) ResourceV1alpha3() v1alpha3.ResourceV1alpha3Interface { m.ctrl.T.Helper() @@ -782,10 +782,10 @@ func (mr *MockInterfaceMockRecorder) ResourceV1beta2() *gomock.Call { } // SchedulingV1 mocks base method. -func (m *MockInterface) SchedulingV1() v115.SchedulingV1Interface { +func (m *MockInterface) SchedulingV1() v116.SchedulingV1Interface { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SchedulingV1") - ret0, _ := ret[0].(v115.SchedulingV1Interface) + ret0, _ := ret[0].(v116.SchedulingV1Interface) return ret0 } @@ -796,10 +796,10 @@ func (mr *MockInterfaceMockRecorder) SchedulingV1() *gomock.Call { } // SchedulingV1alpha1 mocks base method. -func (m *MockInterface) SchedulingV1alpha1() v1alpha16.SchedulingV1alpha1Interface { +func (m *MockInterface) SchedulingV1alpha1() v1alpha15.SchedulingV1alpha1Interface { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SchedulingV1alpha1") - ret0, _ := ret[0].(v1alpha16.SchedulingV1alpha1Interface) + ret0, _ := ret[0].(v1alpha15.SchedulingV1alpha1Interface) return ret0 } @@ -824,10 +824,10 @@ func (mr *MockInterfaceMockRecorder) SchedulingV1beta1() *gomock.Call { } // StorageV1 mocks base method. -func (m *MockInterface) StorageV1() v116.StorageV1Interface { +func (m *MockInterface) StorageV1() v117.StorageV1Interface { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "StorageV1") - ret0, _ := ret[0].(v116.StorageV1Interface) + ret0, _ := ret[0].(v117.StorageV1Interface) return ret0 } @@ -838,10 +838,10 @@ func (mr *MockInterfaceMockRecorder) StorageV1() *gomock.Call { } // StorageV1alpha1 mocks base method. -func (m *MockInterface) StorageV1alpha1() v1alpha17.StorageV1alpha1Interface { +func (m *MockInterface) StorageV1alpha1() v1alpha16.StorageV1alpha1Interface { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "StorageV1alpha1") - ret0, _ := ret[0].(v1alpha17.StorageV1alpha1Interface) + ret0, _ := ret[0].(v1alpha16.StorageV1alpha1Interface) return ret0 } @@ -866,10 +866,10 @@ func (mr *MockInterfaceMockRecorder) StorageV1beta1() *gomock.Call { } // StoragemigrationV1alpha1 mocks base method. -func (m *MockInterface) StoragemigrationV1alpha1() v1alpha18.StoragemigrationV1alpha1Interface { +func (m *MockInterface) StoragemigrationV1alpha1() v1alpha17.StoragemigrationV1alpha1Interface { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "StoragemigrationV1alpha1") - ret0, _ := ret[0].(v1alpha18.StoragemigrationV1alpha1Interface) + ret0, _ := ret[0].(v1alpha17.StoragemigrationV1alpha1Interface) return ret0 } diff --git a/third_party/mock/controller-runtime/client/mocks.go b/third_party/mock/controller-runtime/client/mocks.go index 4af03026ea6..851ec41129e 100644 --- a/third_party/mock/controller-runtime/client/mocks.go +++ b/third_party/mock/controller-runtime/client/mocks.go @@ -44,6 +44,25 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { return m.recorder } +// Apply mocks base method. +func (m *MockClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + m.ctrl.T.Helper() + varargs := []any{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Apply", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Apply indicates an expected call of Apply. +func (mr *MockClientMockRecorder) Apply(ctx, obj any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockClient)(nil).Apply), varargs...) +} + // Create mocks base method. func (m *MockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { m.ctrl.T.Helper() @@ -430,6 +449,25 @@ func (m *MockWriter) EXPECT() *MockWriterMockRecorder { return m.recorder } +// Apply mocks base method. +func (m *MockWriter) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + m.ctrl.T.Helper() + varargs := []any{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Apply", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Apply indicates an expected call of Apply. +func (mr *MockWriterMockRecorder) Apply(ctx, obj any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockWriter)(nil).Apply), varargs...) +} + // Create mocks base method. func (m *MockWriter) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { m.ctrl.T.Helper() From 0bafbc744ba41ea3dd826df82874c61ae0e07357 Mon Sep 17 00:00:00 2001 From: Vladimir Nachev Date: Fri, 31 Oct 2025 00:05:55 +0200 Subject: [PATCH 003/176] Ensure already migrated CRBs are not duplicated in the gardeneraccess MR (#13307) --- cmd/gardenlet/app/migration.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/gardenlet/app/migration.go b/cmd/gardenlet/app/migration.go index 67ad5dc6ff5..e378181acd9 100644 --- a/cmd/gardenlet/app/migration.go +++ b/cmd/gardenlet/app/migration.go @@ -117,6 +117,10 @@ func migrateAdminViewerKubeconfigClusterRoleBindings(ctx context.Context, log lo return fmt.Errorf("failed to get objects for ManagedResource %q: %w", gardenerAccessKey, err) } + gardenerAccessObjects = slices.DeleteFunc(gardenerAccessObjects, func(obj client.Object) bool { + return slices.Contains(crbs, obj.GetName()) + }) + gardenerAccessObjects = append(gardenerAccessObjects, gardeneraccess.ShootAccessClusterRoleBindings()...) gardenerAccessRegistry := managedresources.NewRegistry(kubernetes.ShootScheme, kubernetes.ShootCodec, kubernetes.ShootSerializer) gardenerAccessResources, err := gardenerAccessRegistry.AddAllAndSerialize(gardenerAccessObjects...) From eba00b168c5d81b98fd467361ce4a01a572f3bff Mon Sep 17 00:00:00 2001 From: Marco Voelz Date: Fri, 31 Oct 2025 10:03:05 +0100 Subject: [PATCH 004/176] Increase client-side rate limits for KCM (#13251) * Increase client-side rate limits for KCM * Move qps and burst settings to first args block --- pkg/component/kubernetes/controllermanager/controllermanager.go | 2 ++ .../kubernetes/controllermanager/controllermanager_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pkg/component/kubernetes/controllermanager/controllermanager.go b/pkg/component/kubernetes/controllermanager/controllermanager.go index 7370b480289..5f3db5c1047 100644 --- a/pkg/component/kubernetes/controllermanager/controllermanager.go +++ b/pkg/component/kubernetes/controllermanager/controllermanager.go @@ -653,6 +653,8 @@ func (k *kubeControllerManager) computeCommand(port int32) []string { "--authentication-kubeconfig=" + gardenerutils.PathGenericKubeconfig, "--authorization-kubeconfig=" + gardenerutils.PathGenericKubeconfig, "--kubeconfig=" + gardenerutils.PathGenericKubeconfig, + "--kube-api-qps=100", + "--kube-api-burst=200", } controllersToEnable = sets.New("*", "bootstrapsigner", "tokencleaner") diff --git a/pkg/component/kubernetes/controllermanager/controllermanager_test.go b/pkg/component/kubernetes/controllermanager/controllermanager_test.go index 5dd09dbf0b6..a0fac419b58 100644 --- a/pkg/component/kubernetes/controllermanager/controllermanager_test.go +++ b/pkg/component/kubernetes/controllermanager/controllermanager_test.go @@ -1000,6 +1000,8 @@ func commandForKubernetesVersion( "--authentication-kubeconfig=/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig/kubeconfig", "--authorization-kubeconfig=/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig/kubeconfig", "--kubeconfig=/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig/kubeconfig", + "--kube-api-qps=100", + "--kube-api-burst=200", ) if !isWorkerless { From 0629b6cc9caa8e272b437e6bf9b604a203d68170 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Fri, 31 Oct 2025 10:03:12 +0100 Subject: [PATCH 005/176] [GEP-28] Clean up some TODOs (#13290) * Deploy machine CRDs in shoots with managed infrastructure Previously, we only deployed the machine CRDs in `gardenadm bootstrap`. With this, we also deploy it in `gardenadm init` for shoots with managed infrastructure. * Disable high-availability-config webhook in bootstrap cluster Disable the webhook in the kind cluster during `gardenadm bootstrap`, as it is not needed. This also disables the webhook from the virtual garden, where it isn't needed either. * Adapt CRDs unit test --- .../gardener/resourcemanager/resource_manager.go | 7 +++++-- .../resourcemanager/resource_manager_test.go | 3 ++- .../botanist/customresourcedefinitions.go | 5 +---- .../botanist/customresourcedefinitions_test.go | 14 ++++++++++---- pkg/gardenlet/controller/seed/seed/components.go | 1 + .../operation/botanist/resource_manager.go | 3 ++- .../controller/garden/garden/components.go | 1 + 7 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pkg/component/gardener/resourcemanager/resource_manager.go b/pkg/component/gardener/resourcemanager/resource_manager.go index bacadfa937d..42a585e4b20 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager.go +++ b/pkg/component/gardener/resourcemanager/resource_manager.go @@ -249,6 +249,8 @@ type Values struct { ClusterIdentity *string // ConcurrentSyncs are the number of worker threads for concurrent reconciliation of resources ConcurrentSyncs *int + // HighAvailabilityConfigWebhookEnabled controls whether the high availability config webhook is enabled. + HighAvailabilityConfigWebhookEnabled bool // DefaultNotReadyTolerationSeconds indicates the tolerationSeconds of the toleration for notReady:NoExecute DefaultNotReadyToleration *int64 // DefaultUnreachableTolerationSeconds indicates the tolerationSeconds of the toleration for unreachable:NoExecute @@ -595,7 +597,7 @@ func (r *resourceManager) ensureConfigMap(ctx context.Context, configMap *corev1 Enabled: r.values.EndpointSliceHintsEnabled, }, HighAvailabilityConfig: resourcemanagerconfigv1alpha1.HighAvailabilityConfigWebhookConfig{ - Enabled: !r.values.BootstrapControlPlaneNode, + Enabled: r.values.HighAvailabilityConfigWebhookEnabled, DefaultNotReadyTolerationSeconds: r.values.DefaultNotReadyToleration, DefaultUnreachableTolerationSeconds: r.values.DefaultUnreachableToleration, }, @@ -1358,7 +1360,7 @@ func (r *resourceManager) newMutatingWebhookConfigurationWebhooks( r.newProjectedTokenMountMutatingWebhook(namespaceSelector, secretServerCA, buildClientConfigFn), } - if !r.values.BootstrapControlPlaneNode { + if r.values.HighAvailabilityConfigWebhookEnabled { webhooks = append(webhooks, NewHighAvailabilityConfigMutatingWebhook(namespaceSelector, objectSelector, secretServerCA, buildClientConfigFn)) } @@ -2099,6 +2101,7 @@ func (r *resourceManager) GetValues() Values { return r.values } // SetBootstrapControlPlaneNode sets the BootstrapControlPlaneNode field in the Values. func (r *resourceManager) SetBootstrapControlPlaneNode(b bool) { r.values.BootstrapControlPlaneNode = b + r.values.HighAvailabilityConfigWebhookEnabled = !b } func (r *resourceManager) skipStaticPods(webhooks []admissionregistrationv1.MutatingWebhook) { diff --git a/pkg/component/gardener/resourcemanager/resource_manager_test.go b/pkg/component/gardener/resourcemanager/resource_manager_test.go index 6449c427a86..c761b4ff1f1 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager_test.go +++ b/pkg/component/gardener/resourcemanager/resource_manager_test.go @@ -304,6 +304,7 @@ var _ = Describe("ResourceManager", func() { AlwaysUpdate: &alwaysUpdate, ClusterIdentity: &clusterIdentity, ConcurrentSyncs: &concurrentSyncs, + HighAvailabilityConfigWebhookEnabled: true, DefaultNotReadyToleration: defaultNotReadyTolerationSeconds, DefaultUnreachableToleration: defaultUnreachableTolerationSeconds, NetworkPolicyAdditionalNamespaceSelectors: additionalNetworkPolicyNamespaceSelectors, @@ -2679,9 +2680,9 @@ subjects: })) configMap = configMapFor(&watchedNamespace, ForSourceAndTarget, false, true) - cfg.BootstrapControlPlaneNode = true resourceManager = New(c, deployNamespace, sm, cfg) resourceManager.SetSecrets(secrets) + resourceManager.SetBootstrapControlPlaneNode(true) deployment = deploymentFor(configMap.Name, false, nil, true) diff --git a/pkg/gardenadm/botanist/customresourcedefinitions.go b/pkg/gardenadm/botanist/customresourcedefinitions.go index 31b2d2f737e..f350eb2d3bf 100644 --- a/pkg/gardenadm/botanist/customresourcedefinitions.go +++ b/pkg/gardenadm/botanist/customresourcedefinitions.go @@ -55,10 +55,7 @@ func (b *AutonomousBotanist) ReconcileCustomResourceDefinitions(ctx context.Cont "ETCD": etcdCRDDeployer, } - // For now, we only deploy the machine CRDs in `gardenadm bootstrap`. - // See https://github.com/gardener/gardener/pull/12152#discussion_r2101790385 - // TODO(timebertt): distinguish between scenarios in `gardenadm init` - if !b.Shoot.RunsControlPlane() { + if b.Shoot.HasManagedInfrastructure() { deployers["Machine"], err = machinecontrollermanager.NewCRD(b.SeedClientSet.Client()) if err != nil { return fmt.Errorf("failed creating machine CRD deployer: %w", err) diff --git a/pkg/gardenadm/botanist/customresourcedefinitions_test.go b/pkg/gardenadm/botanist/customresourcedefinitions_test.go index b4141e1400e..f37373e8359 100644 --- a/pkg/gardenadm/botanist/customresourcedefinitions_test.go +++ b/pkg/gardenadm/botanist/customresourcedefinitions_test.go @@ -15,9 +15,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" "github.com/gardener/gardener/pkg/client/kubernetes" fakekubernetes "github.com/gardener/gardener/pkg/client/kubernetes/fake" . "github.com/gardener/gardener/pkg/gardenadm/botanist" @@ -60,9 +62,9 @@ var _ = Describe("CustomResourceDefinitions", func() { }) Describe("#ReconcileCustomResourceDefinitions", func() { - When("running the control plane", func() { + When("infrastructure is not managed by Gardener", func() { BeforeEach(func() { - b.Shoot.ControlPlaneNamespace = metav1.NamespaceSystem + b.Shoot.SetInfo(&gardencorev1beta1.Shoot{}) }) It("should reconcile all expected CRDs", func() { @@ -114,9 +116,13 @@ var _ = Describe("CustomResourceDefinitions", func() { }) }) - When("not running the control plane", func() { + When("infrastructure is managed by Gardener", func() { BeforeEach(func() { - b.Shoot.ControlPlaneNamespace = "shoot--foo--bar" + b.Shoot.SetInfo(&gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + CredentialsBindingName: ptr.To("some-binding"), + }, + }) }) It("should deploy the additional CRDs for the medium-touch scenario", func() { diff --git a/pkg/gardenlet/controller/seed/seed/components.go b/pkg/gardenlet/controller/seed/seed/components.go index 2a0d00c3f87..07c359d637c 100644 --- a/pkg/gardenlet/controller/seed/seed/components.go +++ b/pkg/gardenlet/controller/seed/seed/components.go @@ -294,6 +294,7 @@ func (r *Reconciler) newGardenerResourceManager(seed *gardencorev1beta1.Seed, se return sharedcomponent.NewRuntimeGardenerResourceManager(r.SeedClientSet.Client(), r.GardenNamespace, secretsManager, resourcemanager.Values{ DefaultSeccompProfileEnabled: features.DefaultFeatureGate.Enabled(features.DefaultSeccompProfile), + HighAvailabilityConfigWebhookEnabled: true, DefaultNotReadyToleration: defaultNotReadyTolerationSeconds, DefaultUnreachableToleration: defaultUnreachableTolerationSeconds, EndpointSliceHintsEnabled: endpointSliceHintsEnabled, diff --git a/pkg/gardenlet/operation/botanist/resource_manager.go b/pkg/gardenlet/operation/botanist/resource_manager.go index 609ec3a3f61..7521df47f16 100644 --- a/pkg/gardenlet/operation/botanist/resource_manager.go +++ b/pkg/gardenlet/operation/botanist/resource_manager.go @@ -35,6 +35,7 @@ func (b *Botanist) DefaultResourceManager() (resourcemanager.Interface, error) { values = resourcemanager.Values{ ClusterIdentity: b.Seed.GetInfo().Status.ClusterIdentity, + HighAvailabilityConfigWebhookEnabled: true, DefaultNotReadyToleration: defaultNotReadyTolerationSeconds, DefaultUnreachableToleration: defaultUnreachableTolerationSeconds, IsWorkerless: b.Shoot.IsWorkerless, @@ -65,7 +66,7 @@ func (b *Botanist) DefaultResourceManager() (resourcemanager.Interface, error) { values.TargetNamespaces = nil } else { newFunc = shared.NewRuntimeGardenerResourceManager - // TODO(timebertt): consider disabling the highavailabilityconfig webhook + values.HighAvailabilityConfigWebhookEnabled = false values.PriorityClassName = v1beta1constants.PriorityClassNameSeedSystemCritical } } diff --git a/pkg/operator/controller/garden/garden/components.go b/pkg/operator/controller/garden/garden/components.go index e3b5354be27..7aa5cee5806 100644 --- a/pkg/operator/controller/garden/garden/components.go +++ b/pkg/operator/controller/garden/garden/components.go @@ -396,6 +396,7 @@ func (r *Reconciler) newGardenerResourceManager(garden *operatorv1alpha1.Garden, return sharedcomponent.NewRuntimeGardenerResourceManager(r.RuntimeClientSet.Client(), r.GardenNamespace, secretsManager, resourcemanager.Values{ DefaultSeccompProfileEnabled: features.DefaultFeatureGate.Enabled(features.DefaultSeccompProfile), + HighAvailabilityConfigWebhookEnabled: true, DefaultNotReadyToleration: defaultNotReadyTolerationSeconds, DefaultUnreachableToleration: defaultUnreachableTolerationSeconds, EndpointSliceHintsEnabled: endpointSliceHintsEnabled, From 234659f22dfa0c38d32b8b197fd4172afdc12541 Mon Sep 17 00:00:00 2001 From: Jonas Brand <101177682+8R0WNI3@users.noreply.github.com> Date: Fri, 31 Oct 2025 12:05:05 +0100 Subject: [PATCH 006/176] Add default `.ocm/branch-info.yaml` file (#13281) --- .ocm/branch-info.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .ocm/branch-info.yaml diff --git a/.ocm/branch-info.yaml b/.ocm/branch-info.yaml new file mode 100644 index 00000000000..94491fa6302 --- /dev/null +++ b/.ocm/branch-info.yaml @@ -0,0 +1,5 @@ +release-branch-template: release-v$major.$minor # e.g. release-v1.0 +branch-policy: + significant-part: minor # major, minor, patch + supported-versions-count: 3 + release-cadence: 2w # d (days), w (weeks), y | yr (years) From 87991d92e4ef759a0760b5463fc73b0a5874fa09 Mon Sep 17 00:00:00 2001 From: Plamen Kokanov <35485709+plkokanov@users.noreply.github.com> Date: Fri, 31 Oct 2025 13:05:14 +0200 Subject: [PATCH 007/176] Fix flake in `NodeAgentAuthorizer` integration test (#13287) --- .../nodeagentauthorizer_suite_test.go | 137 ++++++++++++++++-- .../nodeagentauthorizer_test.go | 58 -------- 2 files changed, 124 insertions(+), 71 deletions(-) diff --git a/test/integration/resourcemanager/nodeagentauthorizer/nodeagentauthorizer_suite_test.go b/test/integration/resourcemanager/nodeagentauthorizer/nodeagentauthorizer_suite_test.go index c7f789cc0dd..8823cb3f247 100644 --- a/test/integration/resourcemanager/nodeagentauthorizer/nodeagentauthorizer_suite_test.go +++ b/test/integration/resourcemanager/nodeagentauthorizer/nodeagentauthorizer_suite_test.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "net" + "net/http" "os" "path/filepath" "testing" @@ -25,14 +26,20 @@ import ( clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest" clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + controllerconfig "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/component/kubernetes/apiserver" "github.com/gardener/gardener/pkg/logger" + resourcemanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1" resourcemanagerclient "github.com/gardener/gardener/pkg/resourcemanager/client" "github.com/gardener/gardener/pkg/resourcemanager/webhook/nodeagentauthorizer" "github.com/gardener/gardener/pkg/utils" @@ -77,20 +84,30 @@ var _ = BeforeSuite(func() { // determine a unique namespace name for test environment testNamespaceName := testID + "-" + testRunID[:8] - By("Create kubeconfig file for the authorization webhook") + By("Create kubeconfig files for the authorization webhooks") webhookAddress, err := net.ResolveTCPAddr("tcp", net.JoinHostPort("localhost", "0")) Expect(err).NotTo(HaveOccurred()) - webhookPort, _, err := netutils.SuggestPort("") + webhookPortNodeAgentMachine, _, err := netutils.SuggestPort("") Expect(err).ToNot(HaveOccurred()) - kubeconfigFileName, err := createKubeconfigFileForAuthorizationWebhook(webhookAddress.IP.String(), webhookPort) + webhookPortNodeAgentNode, _, err := netutils.SuggestPort("") + Expect(err).ToNot(HaveOccurred()) + + kubeconfigFileNameMachine, err := createKubeconfigFileForAuthorizationWebhook(webhookAddress.IP.String(), webhookPortNodeAgentMachine) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { + By("Delete kubeconfig file for machine authorization webhook") + Expect(os.Remove(kubeconfigFileNameMachine)).To(Succeed()) + }) + + kubeconfigFileNameNode, err := createKubeconfigFileForAuthorizationWebhook(webhookAddress.IP.String(), webhookPortNodeAgentNode) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { - By("Delete kubeconfig file for authorization webhook") - Expect(os.Remove(kubeconfigFileName)).To(Succeed()) + By("Delete kubeconfig file for node authorization webhook") + Expect(os.Remove(kubeconfigFileNameNode)).To(Succeed()) }) By("Create authorization configuration file") - authorizerConfigFileName, err := createAuthorizationConfigurationFile(kubeconfigFileName) + authorizerConfigFileName, err := createAuthorizationConfigurationFile(kubeconfigFileNameMachine, kubeconfigFileNameNode) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { By("Delete authorization configuration file") @@ -98,7 +115,8 @@ var _ = BeforeSuite(func() { }) By("Start test environment") - Expect(framework.FileExists(kubeconfigFileName)).To(BeTrue()) + Expect(framework.FileExists(kubeconfigFileNameMachine)).To(BeTrue()) + Expect(framework.FileExists(kubeconfigFileNameNode)).To(BeTrue()) testAPIServer := &envtest.APIServer{} testAPIServer.Configure(). Set("authorization-config", authorizerConfigFileName). @@ -118,7 +136,6 @@ var _ = BeforeSuite(func() { ErrorIfCRDPathMissing: true, WebhookInstallOptions: envtest.WebhookInstallOptions{ LocalServingHost: webhookAddress.IP.String(), - LocalServingPort: webhookPort, }, } @@ -171,13 +188,86 @@ var _ = BeforeSuite(func() { By("Delete test Namespace from test cluster") Expect(testClient.Delete(ctx, testNamespace)).To(Or(Succeed(), BeNotFoundError())) }) + + By("Setup managers") + mgrMachine, err := manager.New(testRestConfig, manager.Options{ + WebhookServer: webhook.NewServer(webhook.Options{ + Port: webhookPortNodeAgentMachine, + Host: testEnv.WebhookInstallOptions.LocalServingHost, + CertDir: testEnv.WebhookInstallOptions.LocalServingCertDir, + }), + Metrics: metricsserver.Options{BindAddress: "0"}, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespace.Name: {}}, + }, + Controller: controllerconfig.Controller{ + SkipNameValidation: ptr.To(true), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + mgrNode, err := manager.New(testRestConfig, manager.Options{ + WebhookServer: webhook.NewServer(webhook.Options{ + Port: webhookPortNodeAgentNode, + Host: testEnv.WebhookInstallOptions.LocalServingHost, + CertDir: testEnv.WebhookInstallOptions.LocalServingCertDir, + }), + Metrics: metricsserver.Options{BindAddress: "0"}, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespace.Name: {}}, + }, + Controller: controllerconfig.Controller{ + SkipNameValidation: ptr.To(true), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Register webhooks") + nodeAgentMachineAuthorizer := &nodeagentauthorizer.Webhook{ + Logger: log, + Config: resourcemanagerconfigv1alpha1.NodeAgentAuthorizerWebhookConfig{ + Enabled: true, + MachineNamespace: &testNamespace.Name, + }, + } + Expect(nodeAgentMachineAuthorizer.AddToManager(mgrMachine, testClient, testClient)).To(Succeed()) + + nodeAgentNodeAuthorizer := &nodeagentauthorizer.Webhook{ + Logger: log, + Config: resourcemanagerconfigv1alpha1.NodeAgentAuthorizerWebhookConfig{ + Enabled: true, + MachineNamespace: nil, + }, + } + Expect(nodeAgentNodeAuthorizer.AddToManager(mgrNode, testClient, testClient)).To(Succeed()) + + By("Start managers") + for _, mgr := range []manager.Manager{mgrMachine, mgrNode} { + mgrContext, mgrCancel := context.WithCancel(ctx) + + go func() { + defer GinkgoRecover() + Expect(mgr.Start(mgrContext)).To(Succeed()) + }() + + // Wait for the webhook server to start + Eventually(func() error { + checker := mgr.GetWebhookServer().StartedChecker() + return checker(&http.Request{}) + }).Should(Succeed()) + + DeferCleanup(func() { + By("Stop manager") + mgrCancel() + }) + } }) func createKubeconfigFileForAuthorizationWebhook(address string, port int) (string, error) { kubeconfig, err := runtime.Encode(clientcmdlatest.Codec, kubernetesutils.NewKubeconfig( "authorization-webhook", clientcmdv1.Cluster{ - Server: fmt.Sprintf("https://%s:%d%s", address, port, nodeagentauthorizer.WebhookPath), + Server: fmt.Sprintf("https://%s:%d%s", address, port, "/webhooks/auth/nodeagent"), InsecureSkipTLSVerify: true, }, clientcmdv1.AuthInfo{}, @@ -194,7 +284,7 @@ func createKubeconfigFileForAuthorizationWebhook(address string, port int) (stri return kubeConfigFile.Name(), os.WriteFile(kubeConfigFile.Name(), kubeconfig, 0600) } -func createAuthorizationConfigurationFile(kubeconfigFileName string) (string, error) { +func createAuthorizationConfigurationFile(kubeconfigFileNameMachine, kubeconfigFileNameNode string) (string, error) { authorizationConfiguration := &apiserverv1beta1.AuthorizationConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: apiserverv1beta1.ConfigSchemeGroupVersion.String(), @@ -204,7 +294,28 @@ func createAuthorizationConfigurationFile(kubeconfigFileName string) (string, er {Type: "RBAC", Name: "rbac"}, { Type: "Webhook", - Name: "node-agent-authorizer", + Name: "node-agent-authorizer-machine", + Webhook: &apiserverv1beta1.WebhookConfiguration{ + // Set TTL to a very low value since it cannot be set to 0 because of defaulting. + // See https://github.com/kubernetes/apiserver/blob/3658357fea9fa8b36173d072f2d548f135049e05/pkg/apis/apiserver/v1/defaults.go#L52-L59 + AuthorizedTTL: metav1.Duration{Duration: 1 * time.Nanosecond}, + UnauthorizedTTL: metav1.Duration{Duration: 1 * time.Nanosecond}, + Timeout: metav1.Duration{Duration: 1 * time.Second}, + FailurePolicy: apiserverv1beta1.FailurePolicyDeny, + SubjectAccessReviewVersion: "v1", + MatchConditionSubjectAccessReviewVersion: "v1", + MatchConditions: []apiserverv1beta1.WebhookMatchCondition{{ + Expression: fmt.Sprintf("'%s' in request.groups && request.user == 'gardener.cloud:node-agent:machine:%s'", v1beta1constants.NodeAgentsGroup, machineName), + }}, + ConnectionInfo: apiserverv1beta1.WebhookConnectionInfo{ + Type: apiserverv1beta1.AuthorizationWebhookConnectionInfoTypeKubeConfigFile, + KubeConfigFile: ptr.To(kubeconfigFileNameMachine), + }, + }, + }, + { + Type: "Webhook", + Name: "node-agent-authorizer-node", Webhook: &apiserverv1beta1.WebhookConfiguration{ // Set TTL to a very low value since it cannot be set to 0 because of defaulting. // See https://github.com/kubernetes/apiserver/blob/3658357fea9fa8b36173d072f2d548f135049e05/pkg/apis/apiserver/v1/defaults.go#L52-L59 @@ -215,11 +326,11 @@ func createAuthorizationConfigurationFile(kubeconfigFileName string) (string, er SubjectAccessReviewVersion: "v1", MatchConditionSubjectAccessReviewVersion: "v1", MatchConditions: []apiserverv1beta1.WebhookMatchCondition{{ - Expression: fmt.Sprintf("'%s' in request.groups", v1beta1constants.NodeAgentsGroup), + Expression: fmt.Sprintf("'%s' in request.groups && request.user == 'gardener.cloud:node-agent:machine:%s'", v1beta1constants.NodeAgentsGroup, nodeName), }}, ConnectionInfo: apiserverv1beta1.WebhookConnectionInfo{ Type: apiserverv1beta1.AuthorizationWebhookConnectionInfoTypeKubeConfigFile, - KubeConfigFile: ptr.To(kubeconfigFileName), + KubeConfigFile: ptr.To(kubeconfigFileNameNode), }, }, }, diff --git a/test/integration/resourcemanager/nodeagentauthorizer/nodeagentauthorizer_test.go b/test/integration/resourcemanager/nodeagentauthorizer/nodeagentauthorizer_test.go index 6955d78f3a5..a2a004b3456 100644 --- a/test/integration/resourcemanager/nodeagentauthorizer/nodeagentauthorizer_test.go +++ b/test/integration/resourcemanager/nodeagentauthorizer/nodeagentauthorizer_test.go @@ -5,11 +5,9 @@ package nodeagentauthorizer_test import ( - "context" "crypto/rand" "crypto/x509/pkix" "fmt" - "net/http" machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" . "github.com/onsi/ginkgo/v2" @@ -23,17 +21,9 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" certutil "k8s.io/client-go/util/cert" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - controllerconfig "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/manager" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - resourcemanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/resourcemanager/webhook/nodeagentauthorizer" "github.com/gardener/gardener/pkg/utils" secretsutils "github.com/gardener/gardener/pkg/utils/secrets" . "github.com/gardener/gardener/pkg/utils/test/matchers" @@ -58,54 +48,6 @@ var _ = Describe("NodeAgentAuthorizer tests", func() { ) runTests := func() { - BeforeEach(func() { - By("Setup manager") - mgr, err := manager.New(testRestConfig, manager.Options{ - WebhookServer: webhook.NewServer(webhook.Options{ - Port: testEnv.WebhookInstallOptions.LocalServingPort, - Host: testEnv.WebhookInstallOptions.LocalServingHost, - CertDir: testEnv.WebhookInstallOptions.LocalServingCertDir, - }), - Metrics: metricsserver.Options{BindAddress: "0"}, - Cache: cache.Options{ - DefaultNamespaces: map[string]cache.Config{testNamespace.Name: {}}, - }, - Controller: controllerconfig.Controller{ - SkipNameValidation: ptr.To(true), - }, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Register webhook") - nodeAgentAuthorizer := &nodeagentauthorizer.Webhook{ - Logger: log, - Config: resourcemanagerconfigv1alpha1.NodeAgentAuthorizerWebhookConfig{ - Enabled: true, - MachineNamespace: machineNamespace, - }, - } - Expect(nodeAgentAuthorizer.AddToManager(mgr, testClient, testClient)).To(Succeed()) - - By("Start manager") - mgrContext, mgrCancel := context.WithCancel(ctx) - - go func() { - defer GinkgoRecover() - Expect(mgr.Start(mgrContext)).To(Succeed()) - }() - - // Wait for the webhook server to start - Eventually(func() error { - checker := mgr.GetWebhookServer().StartedChecker() - return checker(&http.Request{}) - }).Should(Succeed()) - - DeferCleanup(func() { - By("Stop manager") - mgrCancel() - }) - }) - Describe("#CertificateSigningRequests", func() { var ( csr *certificatesv1.CertificateSigningRequest From d1677e04ed93b4ac6a0768e4f425a91e1543e56e Mon Sep 17 00:00:00 2001 From: Sonu Kumar Singh Date: Fri, 31 Oct 2025 16:35:21 +0530 Subject: [PATCH 008/176] Update log message for certificate reload (#13295) --- extensions/pkg/webhook/certificates/reloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/pkg/webhook/certificates/reloader.go b/extensions/pkg/webhook/certificates/reloader.go index 5a683a8905e..ea6ce46b640 100644 --- a/extensions/pkg/webhook/certificates/reloader.go +++ b/extensions/pkg/webhook/certificates/reloader.go @@ -113,7 +113,7 @@ func (r *reloader) Reconcile(ctx context.Context, _ reconcile.Request) (reconcil "certDir", r.certDir, ) - log.V(1).Info("Reloading server certificate from secret") + log.V(1).Info("Checking for updated webhook server certificate in secret") secretName, serverCert, serverKey, err := r.getServerCert(ctx, log, r.reader) if err != nil { From cd77ac132230ab5cb346d999478d088c08f2ce4a Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 31 Oct 2025 12:05:27 +0100 Subject: [PATCH 009/176] Update prometheus to v3.7.3 (#13296) --- hack/tools.mk | 2 +- imagevector/containers.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/tools.mk b/hack/tools.mk index cd51659bf59..69b5c6b0515 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -73,7 +73,7 @@ KUBECTL_VERSION ?= v1.34.1 # renovate: datasource=github-releases depName=kubernetes-sigs/kustomize KUSTOMIZE_VERSION ?= v5.3.0 # renovate: datasource=github-releases depName=prometheus/prometheus -PROMTOOL_VERSION ?= 3.7.2 +PROMTOOL_VERSION ?= 3.7.3 # renovate: datasource=github-releases depName=protocolbuffers/protobuf PROTOC_VERSION ?= v33.0 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index f83ce622b26..fdbaa6c03d3 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -229,7 +229,7 @@ images: - name: prometheus sourceRepository: github.com/prometheus/prometheus repository: quay.io/prometheus/prometheus - tag: v3.7.2 + tag: v3.7.3 labels: - name: gardener.cloud/cve-categorisation value: From 337792ff0c1f56e3190b1588b1ea353beb2d8dec Mon Sep 17 00:00:00 2001 From: Tobias Schlicht <155947440+tobschli@users.noreply.github.com> Date: Fri, 31 Oct 2025 12:05:39 +0100 Subject: [PATCH 010/176] Do not wait for VPA CRDs when vpa is not enabled (#13314) There is a check already at the beginning of the reconciliation https://github.com/gardener/gardener/blob/b552e4362f99ab968cc259a245b5ebd48ed6ef8c/pkg/operator/controller/garden/garden/reconciler_reconcile.go\#L99-L105 --- pkg/operator/controller/garden/garden/reconciler_reconcile.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/operator/controller/garden/garden/reconciler_reconcile.go b/pkg/operator/controller/garden/garden/reconciler_reconcile.go index 09eccf8c5f0..f907db9b1cf 100644 --- a/pkg/operator/controller/garden/garden/reconciler_reconcile.go +++ b/pkg/operator/controller/garden/garden/reconciler_reconcile.go @@ -709,6 +709,7 @@ func (r *Reconciler) runRuntimeSetupFlow(ctx context.Context, log logr.Logger, g Name: "Waiting for custom resource definitions for VPA", Fn: c.vpaCRD.Wait, Dependencies: flow.NewTaskIDs(deployVPACRD), + SkipIf: !vpaEnabled(garden.Spec.RuntimeCluster.Settings), }) waitForIstioCRD = g.Add(flow.Task{ From 103a6f6f8624eb8c0cfbf37420773014109d7419 Mon Sep 17 00:00:00 2001 From: Dominik Froehlich Date: Fri, 31 Oct 2025 14:23:06 +0100 Subject: [PATCH 011/176] Fix typo in logging documentation (#13317) --- docs/usage/observability/logging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/observability/logging.md b/docs/usage/observability/logging.md index f6dd00e74ad..2118396b39e 100644 --- a/docs/usage/observability/logging.md +++ b/docs/usage/observability/logging.md @@ -66,7 +66,7 @@ If you are searching for logs for the past one hour, do not expect to see labels By clicking on a value, Plutono automatically eliminates all other labels and/or values with which no valid log stream can be made. After choosing the right labels and their values, click on the `Show logs` button. This will build `Log query` and execute it. -This approach is convenient when you don't know the labels names or they values. +This approach is convenient when you don't know the label names or their values. ![](./images/explore-button-usage.png) Once you feel comfortable, you can start to use the [LogQL](https://github.com/credativ/plutono) language to search for logs. From 9267010aa24a0fef8fb15f8561aaaea643a897f8 Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Fri, 31 Oct 2025 17:37:04 +0100 Subject: [PATCH 012/176] Rename "Autonomous Shoot Clusters" to "Self-Hosted Shoot Clusters" (#13273) * Rename "Autonomous Shoot" to "Self-Hosted Shoot" * Rename "medium touch" to "managed infrastructure" and "high touch" to "unmanaged infrastructure" * [make generate] * Address PR review feedback --- Makefile | 8 +- .../application/templates/rbac-user.yaml | 2 +- charts/gardener/gardenlet/values.yaml | 4 +- .../provider-local/templates/_helpers.tpl | 2 +- .../provider-local/templates/deployment.yaml | 4 +- charts/gardener/provider-local/values.yaml | 2 +- cmd/gardenadm/app/app.go | 4 +- .../app/app.go | 4 +- cmd/gardenlet/app/app.go | 50 +++---- dev-setup/gardenadm.sh | 10 +- dev-setup/gardenadm/machines/ippool.yaml | 2 +- dev-setup/gardenadm/machines/machine.yaml | 6 +- dev-setup/gardenadm/machines/namespace.yaml | 2 +- .../{high-touch => managed-infra}/.gitkeep | 0 .../.gitkeep | 0 .../kustomization.yaml | 2 +- .../patch-shoot-managed-infra.yaml} | 0 .../kustomization.yaml | 0 docs/README.md | 4 +- docs/api-reference/core.md | 2 +- docs/cli-reference/gardenadm/gardenadm.md | 4 +- .../gardenadm/gardenadm_bootstrap.md | 6 +- .../gardenadm/gardenadm_connect.md | 4 +- .../gardenadm/gardenadm_discover.md | 2 +- .../cli-reference/gardenadm/gardenadm_init.md | 2 +- .../cli-reference/gardenadm/gardenadm_join.md | 4 +- .../gardenadm/gardenadm_token.md | 2 +- .../gardenadm/gardenadm_token_generate.md | 2 +- .../gardenadm/gardenadm_version.md | 2 +- docs/concepts/apiserver-admission-plugins.md | 2 +- docs/concepts/controller-manager.md | 6 +- docs/concepts/gardenadm.md | 28 ++-- .../getting_started_locally_with_gardenadm.md | 68 +++++----- docs/extensions/provider-local.md | 2 +- docs/extensions/registration.md | 2 +- docs/extensions/shoot-webhooks.md | 6 +- ...rs.md => 28-self-hosted-shoot-clusters.md} | 128 +++++++++--------- example/90-shoot.yaml | 2 +- extensions/pkg/controller/cmd/options.go | 18 +-- .../controlplane/genericactuator/actuator.go | 2 +- .../pkg/controller/worker/controller.go | 10 +- hack/ci-e2e-kind-gardenadm.sh | 20 +-- .../gardenletidentity/shoot/identity.go | 14 +- .../gardenletidentity/shoot/identity_test.go | 18 +-- .../admission/shootrestriction/handler.go | 10 +- .../webhook/auth/shoot/authorizer.go | 8 +- .../webhook/auth/shoot/authorizer_test.go | 10 +- .../webhook/auth/shoot/gardenadm.go | 2 +- pkg/apis/core/helper/shoot.go | 6 +- pkg/apis/core/helper/shoot_test.go | 10 +- pkg/apis/core/types_shoot.go | 2 +- .../core/v1beta1/constants/types_constants.go | 8 +- pkg/apis/core/v1beta1/generated.proto | 2 +- pkg/apis/core/v1beta1/helper/shoot.go | 15 +- pkg/apis/core/v1beta1/helper/shoot_test.go | 16 +-- pkg/apis/core/v1beta1/types_shoot.go | 2 +- pkg/apis/core/validation/shoot.go | 14 +- pkg/apis/core/validation/shoot_test.go | 12 +- pkg/apiserver/openapi/openapi_generated.go | 2 +- .../operatingsystemconfig.go | 2 +- .../garden/system/virtual/virtual.go | 2 +- .../garden/system/virtual/virtual_test.go | 2 +- .../resourcemanager/resource_manager.go | 2 +- .../machine_controller_manager.go | 18 +-- .../machine_controller_manager_test.go | 8 +- .../machinecontrollermanager/provider.go | 6 +- .../machinecontrollermanager/provider_test.go | 4 +- pkg/component/shared/resourcemanager.go | 2 +- pkg/controller/gardenletdeployer/actuator.go | 6 +- .../gardenletdeployer/actuator_test.go | 6 +- .../certificatesigningrequest/reconciler.go | 2 +- .../reconciler_test.go | 10 +- pkg/gardenadm/botanist/bastion.go | 2 +- pkg/gardenadm/botanist/botanist.go | 62 ++++----- pkg/gardenadm/botanist/botanist_test.go | 42 +++--- pkg/gardenadm/botanist/controlplane.go | 32 ++--- pkg/gardenadm/botanist/controlplane_test.go | 4 +- .../botanist/customresourcedefinitions.go | 4 +- .../customresourcedefinitions_test.go | 6 +- pkg/gardenadm/botanist/dnsrecord.go | 4 +- pkg/gardenadm/botanist/etcd.go | 14 +- pkg/gardenadm/botanist/extensions.go | 20 ++- pkg/gardenadm/botanist/extensions_test.go | 8 +- pkg/gardenadm/botanist/kubelet.go | 2 +- pkg/gardenadm/botanist/kubelet_test.go | 4 +- pkg/gardenadm/botanist/machines.go | 4 +- pkg/gardenadm/botanist/network.go | 4 +- pkg/gardenadm/botanist/network_test.go | 4 +- pkg/gardenadm/botanist/nodeagent.go | 12 +- pkg/gardenadm/botanist/nodeagent_test.go | 4 +- .../botanist/operatingsystemconfig.go | 20 +-- .../botanist/operatingsystemconfig_test.go | 4 +- pkg/gardenadm/botanist/secrets.go | 2 +- pkg/gardenadm/botanist/secrets_test.go | 4 +- pkg/gardenadm/botanist/ssh.go | 10 +- pkg/gardenadm/botanist/system.go | 4 +- pkg/gardenadm/cmd/bootstrap/bootstrap.go | 14 +- pkg/gardenadm/cmd/connect/connect.go | 28 ++-- pkg/gardenadm/cmd/connect/options.go | 4 +- pkg/gardenadm/cmd/init/init.go | 8 +- pkg/gardenadm/cmd/join/join.go | 6 +- pkg/gardenadm/cmd/token/create/options.go | 6 +- .../cmd/token/create/options_test.go | 4 +- pkg/gardenadm/cmd/token/generate/generate.go | 2 +- pkg/gardenadm/cmd/token/utils/clientset.go | 4 +- pkg/gardenlet/bootstrap/bootstrap.go | 12 +- pkg/gardenlet/bootstrap/bootstrap_test.go | 16 +-- .../certificate/certificate_rotation.go | 12 +- .../bootstrappers/garden_kubeconfig.go | 8 +- pkg/gardenlet/controller/add.go | 6 +- .../controllerinstallation/reconciler.go | 10 +- pkg/gardenlet/controller/gardenlet/add.go | 2 +- .../controller/gardenlet/reconciler.go | 2 +- .../botanist/machinecontrollermanager.go | 2 +- .../operation/botanist/resource_manager.go | 2 +- pkg/gardenlet/operation/botanist/secrets.go | 8 +- pkg/gardenlet/operation/botanist/worker.go | 4 +- pkg/gardenlet/operation/shoot/shoot.go | 11 +- pkg/gardenlet/operation/shoot/shoot_test.go | 10 +- pkg/gardenlet/operation/shoot/types.go | 2 +- .../controller/garden/garden/components.go | 2 +- pkg/operator/controller/gardenlet/add.go | 2 +- pkg/operator/controller/gardenlet/add_test.go | 4 +- .../controller/dnsrecord/actuator.go | 10 +- .../controller/dnsrecord/actuator_test.go | 4 +- pkg/provider-local/controller/worker/add.go | 6 +- pkg/provider-local/webhook/dnsconfig/add.go | 4 +- pkg/scheduler/controller/shoot/add.go | 8 +- pkg/scheduler/controller/shoot/add_test.go | 8 +- pkg/utils/gardener/csr.go | 10 +- pkg/utils/gardener/gardenlet/gardenlet.go | 14 +- .../gardener/gardenlet/gardenlet_test.go | 22 +-- pkg/utils/gardener/shoot.go | 6 +- pkg/utils/gardener/shoot_test.go | 4 +- pkg/utils/graph/eventhandler_gardenlet.go | 10 +- pkg/utils/graph/eventhandler_shoot.go | 2 +- pkg/utils/graph/graph.go | 8 +- pkg/utils/graph/graph_test.go | 2 +- .../bootstraptoken/bootstraptoken.go | 6 +- .../global/customverbauthorizer/admission.go | 8 +- .../customverbauthorizer/admission_test.go | 14 +- skaffold-gardenadm.yaml | 12 +- test/e2e/gardenadm/e2e_suite_test.go | 4 +- .../{mediumtouch => managedinfra}/exec.go | 2 +- .../gardenadm.go | 10 +- .../gardenadm.go | 8 +- .../{hightouch => unmanagedinfra}/helper.go | 4 +- .../certificatesigningrequest_test.go | 4 +- .../controllerinstallation_test.go | 12 +- 149 files changed, 658 insertions(+), 662 deletions(-) rename dev-setup/gardenadm/resources/generated/{high-touch => managed-infra}/.gitkeep (100%) rename dev-setup/gardenadm/resources/generated/{medium-touch => unmanaged-infra}/.gitkeep (100%) rename dev-setup/gardenadm/resources/overlays/{medium-touch => managed-infra}/kustomization.yaml (91%) rename dev-setup/gardenadm/resources/overlays/{medium-touch/patch-shoot-medium-touch.yaml => managed-infra/patch-shoot-managed-infra.yaml} (100%) rename dev-setup/gardenadm/resources/overlays/{high-touch => unmanaged-infra}/kustomization.yaml (100%) rename docs/proposals/{28-autonomous-shoot-clusters.md => 28-self-hosted-shoot-clusters.md} (84%) rename test/e2e/gardenadm/{mediumtouch => managedinfra}/exec.go (99%) rename test/e2e/gardenadm/{mediumtouch => managedinfra}/gardenadm.go (97%) rename test/e2e/gardenadm/{hightouch => unmanagedinfra}/gardenadm.go (97%) rename test/e2e/gardenadm/{hightouch => unmanagedinfra}/helper.go (80%) diff --git a/Makefile b/Makefile index 3e3eb959264..3c5c917dd27 100644 --- a/Makefile +++ b/Makefile @@ -449,10 +449,10 @@ test-e2e-local-ha-multi-zone: $(GINKGO) SHOOT_FAILURE_TOLERANCE_TYPE=zone USE_PROVIDER_LOCAL_COREDNS_SERVER=true ./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter "basic || (high-availability && update-to-zone)" ./test/e2e/gardener/... test-e2e-local-operator: $(GINKGO) ./hack/test-e2e-local.sh operator --procs=1 --label-filter="default" ./test/e2e/operator/... -test-e2e-local-gardenadm-medium-touch: $(GINKGO) - ./hack/test-e2e-local.sh gardenadm --procs=1 --label-filter="medium-touch" ./test/e2e/gardenadm/... -test-e2e-local-gardenadm-high-touch: $(GINKGO) - ./hack/test-e2e-local.sh gardenadm --procs=1 --label-filter="high-touch" ./test/e2e/gardenadm/... +test-e2e-local-gardenadm-managed-infra: $(GINKGO) + ./hack/test-e2e-local.sh gardenadm --procs=1 --label-filter="managed-infra" ./test/e2e/gardenadm/... +test-e2e-local-gardenadm-unmanaged-infra: $(GINKGO) + ./hack/test-e2e-local.sh gardenadm --procs=1 --label-filter="unmanaged-infra" ./test/e2e/gardenadm/... test-non-ha-pre-upgrade: $(GINKGO) ./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter="pre-upgrade && !high-availability" ./test/e2e/gardener/... diff --git a/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml b/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml index 9f3a8e91c7d..2480625ee7a 100644 --- a/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml +++ b/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml @@ -25,7 +25,7 @@ rules: - modify-spec-kubernetes - modify-spec-machineimages - modify-spec-providerconfig - - mark-autonomous + - mark-self-hosted - apiGroups: - seedmanagement.gardener.cloud - dashboard.gardener.cloud diff --git a/charts/gardener/gardenlet/values.yaml b/charts/gardener/gardenlet/values.yaml index 7c8e839d227..34e95881590 100644 --- a/charts/gardener/gardenlet/values.yaml +++ b/charts/gardener/gardenlet/values.yaml @@ -221,5 +221,5 @@ nodeToleration: # helm: # ociRepository: # ref: -# name: name-of-autonomous-shoot -# namespace: namespace-of-autonomous-shoot +# name: name-of-self-hosted-shoot +# namespace: namespace-of-self-hosted-shoot diff --git a/charts/gardener/provider-local/templates/_helpers.tpl b/charts/gardener/provider-local/templates/_helpers.tpl index 0526caf71e6..207e1857f98 100644 --- a/charts/gardener/provider-local/templates/_helpers.tpl +++ b/charts/gardener/provider-local/templates/_helpers.tpl @@ -23,7 +23,7 @@ policy/v1 {{- end -}} {{- define "coredns.enabled" -}} -{{- if or .Values.gardener.runtimeCluster.enabled .Values.gardener.autonomousShootCluster -}} +{{- if or .Values.gardener.runtimeCluster.enabled .Values.gardener.selfHostedShootCluster -}} false {{- else -}} {{ .Values.coredns.enabled }} diff --git a/charts/gardener/provider-local/templates/deployment.yaml b/charts/gardener/provider-local/templates/deployment.yaml index f6d065614ab..24ebabdc51e 100644 --- a/charts/gardener/provider-local/templates/deployment.yaml +++ b/charts/gardener/provider-local/templates/deployment.yaml @@ -95,8 +95,8 @@ spec: {{- if .Values.gardener.version }} - --gardener-version={{ .Values.gardener.version }} {{- end }} - {{- if .Values.gardener.autonomousShootCluster }} - - --autonomous-shoot-cluster={{ .Values.gardener.autonomousShootCluster }} + {{- if .Values.gardener.selfHostedShootCluster }} + - --self-hosted-shoot-cluster={{ .Values.gardener.selfHostedShootCluster }} {{- end }} - --log-level={{ .Values.logLevel | default "info" }} - --log-format={{ .Values.logFormat | default "json" }} diff --git a/charts/gardener/provider-local/values.yaml b/charts/gardener/provider-local/values.yaml index 03d98cf5b47..8eb662051ec 100644 --- a/charts/gardener/provider-local/values.yaml +++ b/charts/gardener/provider-local/values.yaml @@ -99,7 +99,7 @@ gardener: provider: local runtimeCluster: enabled: false - autonomousShootCluster: false + selfHostedShootCluster: false usablePorts: - 8080 # metrics diff --git a/cmd/gardenadm/app/app.go b/cmd/gardenadm/app/app.go index 7ecfc079331..f6a2d52a022 100644 --- a/cmd/gardenadm/app/app.go +++ b/cmd/gardenadm/app/app.go @@ -31,7 +31,7 @@ func NewCommand() *cobra.Command { cmd := &cobra.Command{ Use: Name, - Short: Name + " bootstraps and manages autonomous shoot clusters in the Gardener project.", + Short: Name + " bootstraps and manages self-hosted shoot clusters in the Gardener project.", PersistentPreRunE: func(_ *cobra.Command, _ []string) error { if err := opts.Validate(); err != nil { return err @@ -60,7 +60,7 @@ func NewCommand() *cobra.Command { func prepareClusterBootstrapGroup(cmd *cobra.Command, opts *cmd.Options) { group := &cobra.Group{ ID: "cluster-bootstrap", - Title: "Autonomous Shoot Cluster Bootstrap Commands:", + Title: "Self-Hosted Shoot Cluster Bootstrap Commands:", } cmd.AddGroup(group) diff --git a/cmd/gardener-extension-provider-local/app/app.go b/cmd/gardener-extension-provider-local/app/app.go index 5e6a4617e3d..685e2dd1fd9 100644 --- a/cmd/gardener-extension-provider-local/app/app.go +++ b/cmd/gardener-extension-provider-local/app/app.go @@ -280,7 +280,7 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { serviceCtrlOpts.Completed().Apply(&localservice.DefaultAddOptions) workerCtrlOpts.Completed().Apply(&localworker.DefaultAddOptions.Controller) localworker.DefaultAddOptions.GardenCluster = gardenCluster - localworker.DefaultAddOptions.AutonomousShootCluster = generalOpts.Completed().AutonomousShootCluster + localworker.DefaultAddOptions.SelfHostedShootCluster = generalOpts.Completed().SelfHostedShootCluster localBackupBucketOptions.Completed().Apply(&localbackupbucket.DefaultAddOptions) localBackupBucketOptions.Completed().Apply(&localbackupentry.DefaultAddOptions) heartbeatCtrlOptions.Completed().Apply(&heartbeat.DefaultAddOptions) @@ -309,7 +309,7 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { return fmt.Errorf("could not add readycheck of webhook to manager: %w", err) } - atomicShootWebhookConfig, err := webhookOptions.Completed().AddToManager(ctx, mgr, nil, generalOpts.Completed().AutonomousShootCluster) + atomicShootWebhookConfig, err := webhookOptions.Completed().AddToManager(ctx, mgr, nil, generalOpts.Completed().SelfHostedShootCluster) if err != nil { return fmt.Errorf("could not add webhooks to manager: %w", err) } diff --git a/cmd/gardenlet/app/app.go b/cmd/gardenlet/app/app.go index d448f4747ec..a9cde085cc4 100644 --- a/cmd/gardenlet/app/app.go +++ b/cmd/gardenlet/app/app.go @@ -174,23 +174,23 @@ func run(ctx context.Context, cancel context.CancelFunc, log logr.Logger, cfg *g return err } - if !gardenlet.IsResponsibleForAutonomousShoot() { - // TODO(rfranzke): This healthz check is currently not enabled for autonomous shoots because it depends on the + if !gardenlet.IsResponsibleForSelfHostedShoot() { + // TODO(rfranzke): This healthz check is currently not enabled for self-hosted shoots because it depends on the // seed-lease controller. This controller is currently not enabled, but it will be in the future. Once it is - // enabled, we can also enable this healthz check for autonomous shoots. + // enabled, we can also enable this healthz check for self-hosted shoots. if err := mgr.AddHealthzCheck("periodic-health", gardenerhealthz.CheckerFunc(healthManager)); err != nil { return err } } - var autonomousShootMeta *types.NamespacedName - if gardenlet.IsResponsibleForAutonomousShoot() { + var selfHostedShootMeta *types.NamespacedName + if gardenlet.IsResponsibleForSelfHostedShoot() { configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: v1beta1constants.ConfigMapNameShootInfo, Namespace: metav1.NamespaceSystem}} if err := mgr.GetAPIReader().Get(ctx, client.ObjectKeyFromObject(configMap), configMap); err != nil { return fmt.Errorf("failed reading ConfigMap %s: %w", client.ObjectKeyFromObject(configMap), err) } - autonomousShootMeta = &types.NamespacedName{Namespace: configMap.Data["shootNamespace"], Name: configMap.Data["shootName"]} - log.Info("Fetched information about autonomous shoot", "shootMeta", autonomousShootMeta) + selfHostedShootMeta = &types.NamespacedName{Namespace: configMap.Data["shootNamespace"], Name: configMap.Data["shootName"]} + log.Info("Fetched information about self-hosted shoot", "shootMeta", selfHostedShootMeta) } log.Info("Adding runnables to manager for bootstrapping") @@ -203,7 +203,7 @@ func run(ctx context.Context, cancel context.CancelFunc, log logr.Logger, cfg *g RuntimeClient: mgr.GetClient(), Log: mgr.GetLogger().WithName("bootstrap"), Config: cfg, - AutonomousShootMeta: autonomousShootMeta, + SelfHostedShootMeta: selfHostedShootMeta, Result: kubeconfigBootstrapResult, }, }, @@ -212,7 +212,7 @@ func run(ctx context.Context, cancel context.CancelFunc, log logr.Logger, cfg *g cancel: cancel, mgr: mgr, config: cfg, - autonomousShootMeta: autonomousShootMeta, + selfHostedShootMeta: selfHostedShootMeta, healthManager: healthManager, kubeconfigBootstrapResult: kubeconfigBootstrapResult, }, @@ -220,7 +220,7 @@ func run(ctx context.Context, cancel context.CancelFunc, log logr.Logger, cfg *g } ) - if !gardenlet.IsResponsibleForAutonomousShoot() { + if !gardenlet.IsResponsibleForSelfHostedShoot() { runner.BootstrapRunnables = append([]manager.Runnable{ &bootstrappers.SeedConfigChecker{ SeedClient: mgr.GetClient(), @@ -241,7 +241,7 @@ type garden struct { cancel context.CancelFunc mgr manager.Manager config *gardenletconfigv1alpha1.GardenletConfiguration - autonomousShootMeta *types.NamespacedName + selfHostedShootMeta *types.NamespacedName healthManager gardenerhealthz.Manager kubeconfigBootstrapResult *bootstrappers.KubeconfigBootstrapResult } @@ -271,7 +271,7 @@ func (g *garden) Start(ctx context.Context) error { }, } - if !gardenlet.IsResponsibleForAutonomousShoot() { + if !gardenlet.IsResponsibleForSelfHostedShoot() { seedNamespace := gardenerutils.ComputeGardenNamespace(g.config.SeedConfig.Name) opts.NewCache = func(config *rest.Config, opts cache.Options) (cache.Cache, error) { @@ -361,12 +361,12 @@ func (g *garden) Start(ctx context.Context) error { // gardenlet should watch only objects which are related to the shoot it is responsible for. opts.ByObject = map[client.Object]cache.ByObject{ &gardencorev1beta1.Shoot{}: { - Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: g.autonomousShootMeta.Name}), - Namespaces: map[string]cache.Config{g.autonomousShootMeta.Namespace: {}}, + Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: g.selfHostedShootMeta.Name}), + Namespaces: map[string]cache.Config{g.selfHostedShootMeta.Namespace: {}}, }, &seedmanagementv1alpha1.Gardenlet{}: { - Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: gardenlet.ResourcePrefixAutonomousShoot + g.autonomousShootMeta.Name}), - Namespaces: map[string]cache.Config{g.autonomousShootMeta.Namespace: {}}, + Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: gardenlet.ResourcePrefixSelfHostedShoot + g.selfHostedShootMeta.Name}), + Namespaces: map[string]cache.Config{g.selfHostedShootMeta.Namespace: {}}, }, } @@ -419,7 +419,7 @@ func (g *garden) Start(ctx context.Context) error { return fmt.Errorf("failed waiting for cache to be synced") } - if !gardenlet.IsResponsibleForAutonomousShoot() { + if !gardenlet.IsResponsibleForSelfHostedShoot() { log.Info("Registering Seed object in garden cluster") if err := g.registerSeed(ctx, gardenCluster.GetClient()); err != nil { return err @@ -459,7 +459,7 @@ func (g *garden) Start(ctx context.Context) error { } if g.config.GardenClientConnection.KubeconfigSecret != nil { - certificateManager, err := certificate.NewCertificateManager(log, gardenCluster, g.mgr.GetClient(), g.config, g.autonomousShootMeta) + certificateManager, err := certificate.NewCertificateManager(log, gardenCluster, g.mgr.GetClient(), g.config, g.selfHostedShootMeta) if err != nil { return fmt.Errorf("failed to create a new certificate manager: %w", err) } @@ -473,21 +473,21 @@ func (g *garden) Start(ctx context.Context) error { return err } - if gardenlet.IsResponsibleForAutonomousShoot() { + if gardenlet.IsResponsibleForSelfHostedShoot() { if err := retry.Until(ctx, 5*time.Second, func(ctx context.Context) (done bool, err error) { - shoot := &gardencorev1beta1.Shoot{ObjectMeta: metav1.ObjectMeta{Name: g.autonomousShootMeta.Name, Namespace: g.autonomousShootMeta.Namespace}} + shoot := &gardencorev1beta1.Shoot{ObjectMeta: metav1.ObjectMeta{Name: g.selfHostedShootMeta.Name, Namespace: g.selfHostedShootMeta.Namespace}} if err := gardenCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(shoot), shoot); err != nil { if !apierrors.IsNotFound(err) { return retry.SevereError(err) } - log.Info("Shoot resource for autonomous shoot not found yet, waiting for it to be created in the Gardener API (usually, the 'gardenadm connect' command invocation creates it)", "shoot", g.autonomousShootMeta) - return retry.MinorError(fmt.Errorf("the Shoot resource %s for autonomous shoot was not found yet", g.autonomousShootMeta)) + log.Info("Shoot resource for self-hosted shoot not found yet, waiting for it to be created in the Gardener API (usually, the 'gardenadm connect' command invocation creates it)", "shoot", g.selfHostedShootMeta) + return retry.MinorError(fmt.Errorf("the Shoot resource %s for self-hosted shoot was not found yet", g.selfHostedShootMeta)) } return retry.Ok() }); err != nil { - return fmt.Errorf("failed waiting for Shoot %s: %w", g.autonomousShootMeta, err) + return fmt.Errorf("failed waiting for Shoot %s: %w", g.selfHostedShootMeta, err) } - log.Info("Successfully fetched Shoot resource for autonomous shoot", "shoot", g.autonomousShootMeta) + log.Info("Successfully fetched Shoot resource for self-hosted shoot", "shoot", g.selfHostedShootMeta) } log.Info("Adding controllers to manager") @@ -761,7 +761,7 @@ func (g *garden) overwriteGardenHostWhenDeployedInRuntimeCluster(ctx context.Con func addAllFieldIndexes(ctx context.Context, i client.FieldIndexer) error { // TODO(rfranzke): Revisit this once `gardenadm connect` progresses (currently, this causes informers to be started // against the API server, but gardenlet does not have the needed permissions). - if gardenlet.IsResponsibleForAutonomousShoot() { + if gardenlet.IsResponsibleForSelfHostedShoot() { return nil } diff --git a/dev-setup/gardenadm.sh b/dev-setup/gardenadm.sh index e960501ede1..4f85fa26ca4 100755 --- a/dev-setup/gardenadm.sh +++ b/dev-setup/gardenadm.sh @@ -9,8 +9,8 @@ set -o pipefail COMMAND="${1:-up}" VALID_COMMANDS=("up" "down") -SCENARIO="${SCENARIO:-high-touch}" -VALID_SCENARIOS=("high-touch" "medium-touch" "connect") +SCENARIO="${SCENARIO:-unmanaged-infra}" +VALID_SCENARIOS=("unmanaged-infra" "managed-infra" "connect") valid_scenario=false for scenario in "${VALID_SCENARIOS[@]}"; do @@ -28,7 +28,7 @@ case "$COMMAND" in up) if [[ "$SCENARIO" != "connect" ]]; then # Prepare resources and generate manifests. - # The manifests are copied to the high-touch machine pods or can be passed to the `--config-dir` flag of `gardenadm bootstrap`. + # The manifests are copied to the unmanaged-infra machine pods or can be passed to the `--config-dir` flag of `gardenadm bootstrap`. skaffold build \ -p "$SCENARIO" \ -m gardenadm,provider-local-node,provider-local \ @@ -42,9 +42,9 @@ case "$COMMAND" in --build-artifacts \ - - if [[ "$SCENARIO" == "high-touch" ]]; then + if [[ "$SCENARIO" == "unmanaged-infra" ]]; then skaffold run \ - -n gardenadm-high-touch \ + -n gardenadm-unmanaged-infra \ -m provider-local-node,machine fi diff --git a/dev-setup/gardenadm/machines/ippool.yaml b/dev-setup/gardenadm/machines/ippool.yaml index dcbcfa668fa..6077a5d328f 100644 --- a/dev-setup/gardenadm/machines/ippool.yaml +++ b/dev-setup/gardenadm/machines/ippool.yaml @@ -1,7 +1,7 @@ apiVersion: crd.projectcalico.org/v1 kind: IPPool metadata: - name: gardenadm-high-touch + name: gardenadm-unmanaged-infra spec: cidr: 10.0.0.0/16 ipipMode: Always diff --git a/dev-setup/gardenadm/machines/machine.yaml b/dev-setup/gardenadm/machines/machine.yaml index fad0c98acb9..5c29819b444 100644 --- a/dev-setup/gardenadm/machines/machine.yaml +++ b/dev-setup/gardenadm/machines/machine.yaml @@ -15,7 +15,7 @@ spec: labels: app: machine annotations: - cni.projectcalico.org/ipv4pools: '["gardenadm-high-touch"]' + cni.projectcalico.org/ipv4pools: '["gardenadm-unmanaged-infra"]' spec: containers: - name: node @@ -42,7 +42,7 @@ spec: - name: bashrc mountPath: /root/.bashrc subPath: bashrc - # we don't want components of the autonomous shoot cluster to communicate with the kind API server + # we don't want components of the self-hosted shoot cluster to communicate with the kind API server automountServiceAccountToken: false hostAliases: - hostnames: @@ -77,7 +77,7 @@ kind: ConfigMap metadata: name: machine-bashrc labels: - app: high-touch-machine + app: unmanaged-infra-machine data: bashrc: | export KUBECONFIG=/etc/kubernetes/admin.conf diff --git a/dev-setup/gardenadm/machines/namespace.yaml b/dev-setup/gardenadm/machines/namespace.yaml index 31ea660a287..bd8ce7b8363 100644 --- a/dev-setup/gardenadm/machines/namespace.yaml +++ b/dev-setup/gardenadm/machines/namespace.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Namespace metadata: - name: gardenadm-high-touch + name: gardenadm-unmanaged-infra labels: # Use label so that machine pods are considered in the export_artifacts() helper function during test runs in Prow. # This makes sure that the usual systemd and pod logs get exported for each machine pod in this namespace. diff --git a/dev-setup/gardenadm/resources/generated/high-touch/.gitkeep b/dev-setup/gardenadm/resources/generated/managed-infra/.gitkeep similarity index 100% rename from dev-setup/gardenadm/resources/generated/high-touch/.gitkeep rename to dev-setup/gardenadm/resources/generated/managed-infra/.gitkeep diff --git a/dev-setup/gardenadm/resources/generated/medium-touch/.gitkeep b/dev-setup/gardenadm/resources/generated/unmanaged-infra/.gitkeep similarity index 100% rename from dev-setup/gardenadm/resources/generated/medium-touch/.gitkeep rename to dev-setup/gardenadm/resources/generated/unmanaged-infra/.gitkeep diff --git a/dev-setup/gardenadm/resources/overlays/medium-touch/kustomization.yaml b/dev-setup/gardenadm/resources/overlays/managed-infra/kustomization.yaml similarity index 91% rename from dev-setup/gardenadm/resources/overlays/medium-touch/kustomization.yaml rename to dev-setup/gardenadm/resources/overlays/managed-infra/kustomization.yaml index f8d0cca8176..111f5bf56bc 100644 --- a/dev-setup/gardenadm/resources/overlays/medium-touch/kustomization.yaml +++ b/dev-setup/gardenadm/resources/overlays/managed-infra/kustomization.yaml @@ -11,7 +11,7 @@ components: - ../../../../gardenconfig/components/credentials/secret-project-garden patches: -- path: patch-shoot-medium-touch.yaml +- path: patch-shoot-managed-infra.yaml - target: kind: Shoot patch: | diff --git a/dev-setup/gardenadm/resources/overlays/medium-touch/patch-shoot-medium-touch.yaml b/dev-setup/gardenadm/resources/overlays/managed-infra/patch-shoot-managed-infra.yaml similarity index 100% rename from dev-setup/gardenadm/resources/overlays/medium-touch/patch-shoot-medium-touch.yaml rename to dev-setup/gardenadm/resources/overlays/managed-infra/patch-shoot-managed-infra.yaml diff --git a/dev-setup/gardenadm/resources/overlays/high-touch/kustomization.yaml b/dev-setup/gardenadm/resources/overlays/unmanaged-infra/kustomization.yaml similarity index 100% rename from dev-setup/gardenadm/resources/overlays/high-touch/kustomization.yaml rename to dev-setup/gardenadm/resources/overlays/unmanaged-infra/kustomization.yaml diff --git a/docs/README.md b/docs/README.md index 8c763807b25..f1e13e09bc3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -154,7 +154,7 @@ * [GEP-25: Namespaced Cloud Profiles](proposals/25-namespaced-cloud-profiles.md) * [GEP-26: Workload Identity - Trust Based Authentication](proposals/26-workload-identity.md) * [GEP-27: Add Optional Bastion Section To CloudProfile](proposals/27-cloudprofile-bastion-section.md) -* [GEP-28: Autonomous Shoot Clusters](proposals/28-autonomous-shoot-clusters.md) +* [GEP-28: Self-Hosted Shoot Clusters](proposals/28-self-hosted-shoot-clusters.md) * [GEP-30: Rework API Server Proxy to Drop Proxy Protocol](proposals/30-apiserver-proxy.md) * [GEP-31: In-Place Node Updates of Shoot Clusters](proposals/31-inplace-node-update.md) * [GEP-32: Cloud Profile Version Classification Lifecycles](proposals/32-version-classification-lifecycles.md) @@ -235,7 +235,7 @@ * [Getting started locally](deployment/getting_started_locally.md) * [Getting started locally with extensions](deployment/getting_started_locally_with_extensions.md) -* [Getting started locally with Autonomous Shoot Clusters](deployment/getting_started_locally_with_gardenadm.md) +* [Getting started locally with Self-Hosted Shoot Clusters](deployment/getting_started_locally_with_gardenadm.md) * [Setup Gardener on a Kubernetes cluster](deployment/setup_gardener.md) * [Version Skew Policy](deployment/version_skew_policy.md) * [Deploying Gardenlets](deployment/deploy_gardenlet.md) diff --git a/docs/api-reference/core.md b/docs/api-reference/core.md index 17c30ec8cfb..370dddc5ad2 100644 --- a/docs/api-reference/core.md +++ b/docs/api-reference/core.md @@ -15038,7 +15038,7 @@ WorkerControlPlane (Optional)

ControlPlane specifies that the shoot cluster control plane components should be running in this worker pool. -This is only relevant for autonomous shoot clusters.

+This is only relevant for self-hosted shoot clusters.

diff --git a/docs/cli-reference/gardenadm/gardenadm.md b/docs/cli-reference/gardenadm/gardenadm.md index 19a1a97065d..793db701a9e 100644 --- a/docs/cli-reference/gardenadm/gardenadm.md +++ b/docs/cli-reference/gardenadm/gardenadm.md @@ -1,6 +1,6 @@ ## gardenadm -gardenadm bootstraps and manages autonomous shoot clusters in the Gardener project. +gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener project. ### Options @@ -12,7 +12,7 @@ gardenadm bootstraps and manages autonomous shoot clusters in the Gardener proje ### SEE ALSO -* [gardenadm bootstrap](gardenadm_bootstrap.md) - Bootstrap the infrastructure for an Autonomous Shoot Cluster +* [gardenadm bootstrap](gardenadm_bootstrap.md) - Bootstrap the infrastructure for a Self-Hosted Shoot Cluster * [gardenadm connect](gardenadm_connect.md) - Deploy a gardenlet for further cluster management * [gardenadm discover](gardenadm_discover.md) - Conveniently download Gardener configuration resources from an existing garden cluster * [gardenadm init](gardenadm_init.md) - Bootstrap the first control plane node diff --git a/docs/cli-reference/gardenadm/gardenadm_bootstrap.md b/docs/cli-reference/gardenadm/gardenadm_bootstrap.md index 02a3e69dd63..a4358e05bae 100644 --- a/docs/cli-reference/gardenadm/gardenadm_bootstrap.md +++ b/docs/cli-reference/gardenadm/gardenadm_bootstrap.md @@ -1,10 +1,10 @@ ## gardenadm bootstrap -Bootstrap the infrastructure for an Autonomous Shoot Cluster +Bootstrap the infrastructure for a Self-Hosted Shoot Cluster ### Synopsis -Bootstrap the infrastructure for an Autonomous Shoot Cluster (networks, machines, etc.) +Bootstrap the infrastructure for a Self-Hosted Shoot Cluster (networks, machines, etc.) ``` gardenadm bootstrap [flags] @@ -36,5 +36,5 @@ gardenadm bootstrap --config-dir /path/to/manifests ### SEE ALSO -* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages autonomous shoot clusters in the Gardener project. +* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener project. diff --git a/docs/cli-reference/gardenadm/gardenadm_connect.md b/docs/cli-reference/gardenadm/gardenadm_connect.md index 978fe8d8fc0..d1d780dda69 100644 --- a/docs/cli-reference/gardenadm/gardenadm_connect.md +++ b/docs/cli-reference/gardenadm/gardenadm_connect.md @@ -20,7 +20,7 @@ gardenadm connect ### Options ``` - --bootstrap-token string Bootstrap token for connecting the autonomous shoot cluster to a garden cluster (create it with 'gardenadm token' in the garden cluster) + --bootstrap-token string Bootstrap token for connecting the self-hosted shoot cluster to a garden cluster (create it with 'gardenadm token' in the garden cluster) --ca-certificate bytesBase64 Base64-encoded certificate authority bundle of the Gardener control plane -d, --config-dir string Path to a directory containing the Gardener configuration files for the init command, i.e., files containing resources like CloudProfile, Shoot, etc. The files must be in YAML/JSON and have .{yaml,yml,json} file extensions to be considered. --force Forces the deployment of gardenlet, even if it already exists @@ -36,5 +36,5 @@ gardenadm connect ### SEE ALSO -* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages autonomous shoot clusters in the Gardener project. +* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener project. diff --git a/docs/cli-reference/gardenadm/gardenadm_discover.md b/docs/cli-reference/gardenadm/gardenadm_discover.md index 43ab2746ee3..add28f7c927 100644 --- a/docs/cli-reference/gardenadm/gardenadm_discover.md +++ b/docs/cli-reference/gardenadm/gardenadm_discover.md @@ -35,5 +35,5 @@ gardenadm discover ### SEE ALSO -* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages autonomous shoot clusters in the Gardener project. +* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener project. diff --git a/docs/cli-reference/gardenadm/gardenadm_init.md b/docs/cli-reference/gardenadm/gardenadm_init.md index 403a330d8df..c0079025a87 100644 --- a/docs/cli-reference/gardenadm/gardenadm_init.md +++ b/docs/cli-reference/gardenadm/gardenadm_init.md @@ -33,5 +33,5 @@ gardenadm init --config-dir /path/to/manifests ### SEE ALSO -* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages autonomous shoot clusters in the Gardener project. +* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener project. diff --git a/docs/cli-reference/gardenadm/gardenadm_join.md b/docs/cli-reference/gardenadm/gardenadm_join.md index f0b789b874f..760bb6fb6a3 100644 --- a/docs/cli-reference/gardenadm/gardenadm_join.md +++ b/docs/cli-reference/gardenadm/gardenadm_join.md @@ -6,7 +6,7 @@ Bootstrap worker nodes and join them to the cluster Bootstrap worker nodes and join them to the cluster. -This command helps to initialize and configure a node to join an existing autonomous shoot cluster. +This command helps to initialize and configure a node to join an existing self-hosted shoot cluster. It ensures that the necessary configurations are applied and the node is properly registered as a worker or control plane node. Note that further control plane nodes cannot be joined currently. @@ -40,5 +40,5 @@ gardenadm join --bootstrap-token --ca-certificate --gardener-n ### SEE ALSO -* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages autonomous shoot clusters in the Gardener project. +* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener project. diff --git a/docs/cli-reference/gardenadm/gardenadm_token.md b/docs/cli-reference/gardenadm/gardenadm_token.md index 66098e6cfed..004db09a173 100644 --- a/docs/cli-reference/gardenadm/gardenadm_token.md +++ b/docs/cli-reference/gardenadm/gardenadm_token.md @@ -21,7 +21,7 @@ Manage bootstrap and discovery tokens for gardenadm join ### SEE ALSO -* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages autonomous shoot clusters in the Gardener project. +* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener project. * [gardenadm token create](gardenadm_token_create.md) - Create a bootstrap token on the cluster for joining a node or connecting a Shoot to Gardener * [gardenadm token delete](gardenadm_token_delete.md) - Delete one or more bootstrap tokens from the cluster * [gardenadm token generate](gardenadm_token_generate.md) - Generate a random bootstrap token for joining a node diff --git a/docs/cli-reference/gardenadm/gardenadm_token_generate.md b/docs/cli-reference/gardenadm/gardenadm_token_generate.md index 1df4ca87cd3..b6a1ff0a3c9 100644 --- a/docs/cli-reference/gardenadm/gardenadm_token_generate.md +++ b/docs/cli-reference/gardenadm/gardenadm_token_generate.md @@ -4,7 +4,7 @@ Generate a random bootstrap token for joining a node ### Synopsis -Generate a random bootstrap token that can be used for joining a node to an autonomous shoot cluster. +Generate a random bootstrap token that can be used for joining a node to a self-hosted shoot cluster. Note that the token is not created on the server (use 'gardenadm token create' for it). The token is securely generated and follows the format "[a-z0-9]{6}.[a-z0-9]{16}". Read more about it here: https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/ diff --git a/docs/cli-reference/gardenadm/gardenadm_version.md b/docs/cli-reference/gardenadm/gardenadm_version.md index 54e63257d4e..b35984da5b1 100644 --- a/docs/cli-reference/gardenadm/gardenadm_version.md +++ b/docs/cli-reference/gardenadm/gardenadm_version.md @@ -25,5 +25,5 @@ gardenadm version [flags] ### SEE ALSO -* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages autonomous shoot clusters in the Gardener project. +* [gardenadm](gardenadm.md) - gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener project. diff --git a/docs/concepts/apiserver-admission-plugins.md b/docs/concepts/apiserver-admission-plugins.md index a181bdd197f..3637aac7fbc 100644 --- a/docs/concepts/apiserver-admission-plugins.md +++ b/docs/concepts/apiserver-admission-plugins.md @@ -65,7 +65,7 @@ Usually, regular project members are not bound to this custom verb, allowing the For `NamespacedCloudProfile`s, the modification of specific fields also require the user to be bound to an RBAC role with custom verbs. Please see [this document](../usage/project/namespaced-cloud-profiles.md#field-modification-restrictions) for more information. -For `Shoot`s, the `mark-autonomous` verb is required to set the `spec.provider.workers[].controlPlane` field (which marks a `Shoot` as "autonomous shoot"). +For `Shoot`s, the `mark-self-hosted` verb is required to set the `spec.provider.workers[].controlPlane` field (which marks a `Shoot` as "self-hosted shoot"). ## `DeletionConfirmation` diff --git a/docs/concepts/controller-manager.md b/docs/concepts/controller-manager.md index 93a6e04342d..d6e3ca1f6c3 100644 --- a/docs/concepts/controller-manager.md +++ b/docs/concepts/controller-manager.md @@ -51,9 +51,9 @@ For regular gardenlets managing seed clusters, the controller auto-approves CSRs 1. The CSR matches the requirements for a seed client certificate (correct organization, common name, and usages) 2. The requesting client has permission to create the `certificatesigningrequests/seedclient` subresource -#### Autonomous Shoot Client Certificates (`shootclient` subresource) +#### Self-Hosted Shoot Client Certificates (`shootclient` subresource) -For gardenlets responsible for autonomous shoots, the controller performs additional security validation: +For gardenlets responsible for self-hosted shoots, the controller performs additional security validation: 1. The CSR must match the requirements for a shoot client certificate (correct organization, common name, and usages) 2. The requesting client must have permission to create the `certificatesigningrequests/shootclient` subresource 3. **Bootstrap Token Validation**: The CSR must be requested via a bootstrap token with a specific description format containing the shoot's namespace and name @@ -66,7 +66,7 @@ Both types of CSRs rely on the same RBAC setup: - This ClusterRole grants permission to create both `certificatesigningrequests/seedclient` and `certificatesigningrequests/shootclient` subresources - The gardenlet's bootstrap kubeconfig contains a bootstrap token that authenticates it as part of the `system:bootstrappers` group -This dual approval mechanism ensures that both regular seed gardenlets and autonomous shoot gardenlets can obtain client certificates while maintaining appropriate security controls for each deployment model. +This dual approval mechanism ensures that both regular seed gardenlets and self-hosted shoot gardenlets can obtain client certificates while maintaining appropriate security controls for each deployment model. ### [`CloudProfile` Controller](../../pkg/controllermanager/controller/cloudprofile) diff --git a/docs/concepts/gardenadm.md b/docs/concepts/gardenadm.md index 227d6ae406e..f46d8fe6e5d 100644 --- a/docs/concepts/gardenadm.md +++ b/docs/concepts/gardenadm.md @@ -1,47 +1,47 @@ --- title: gardenadm -description: Bootstrapping and management of autonomous shoot clusters. +description: Bootstrapping and management of self-hosted shoot clusters. --- > [!CAUTION] > This tool is currently under development and considered highly experimental. > Do not use it in production environments. -> Read more about it in [GEP-28](../proposals/28-autonomous-shoot-clusters.md). +> Read more about it in [GEP-28](../proposals/28-self-hosted-shoot-clusters.md). gardenadm ## Overview -`gardenadm` is a command line tool for bootstrapping Kubernetes clusters called "Autonomous Shoot Clusters". +`gardenadm` is a command line tool for bootstrapping Kubernetes clusters called "Self-Hosted Shoot Clusters". In contrast to usual Gardener-managed clusters (called Shoot Clusters), the Kubernetes control plane components run as static pods on a dedicated control plane worker pool in the cluster itself (instead of running them as pods on another Kubernetes cluster (called Seed Cluster)). -Autonomous shoot clusters can be bootstrapped without an existing Gardener installation. +Self-hosted shoot clusters can be bootstrapped without an existing Gardener installation. Hence, they can host a Gardener installation itself and/or serve as the initial seed cluster of a Gardener installation. -Furthermore, autonomous shoot clusters can only be created by the `gardenadm` tool and not via an API of an existing Gardener system. +Furthermore, self-hosted shoot clusters can only be created by the `gardenadm` tool and not via an API of an existing Gardener system. ![Architectural overview](../proposals/assets/28-overview.png) -Such autonomous shoot clusters are meant to operate autonomously, but not to exist completely independently of Gardener. +Such self-hosted shoot clusters are meant to operate autonomously, but not to exist completely independently of Gardener. Hence, after their initial creation, they are connected to an existing Gardener system such that the established cluster management functionality via the `Shoot` API can be applied. -I.e., day-2 operations for autonomous shoot clusters are only supported after connecting them to a Gardener system. -This Gardener system could also run in an autonomous shoot cluster itself (in this case, you would first need to deploy it before being able to connect the autonomous shoot cluster to it). +I.e., day-2 operations for self-hosted shoot clusters are only supported after connecting them to a Gardener system. +This Gardener system could also run in a self-hosted shoot cluster itself (in this case, you would first need to deploy it before being able to connect the self-hosted shoot cluster to it). -Furthermore, autonomous shoot clusters are not considered a replacement or alternative for regular shoot clusters. +Furthermore, self-hosted shoot clusters are not considered a replacement or alternative for regular shoot clusters. They should be only used for special use-cases or requirements as creating them is more complex and as their costs will most likely be higher (since control plane nodes are typically not fully utilized in such architecture). In this light, a high cluster creation/deletion churn rate is neither expected nor in scope. ## Getting Started Locally -[This document](../deployment/getting_started_locally_with_gardenadm.md) walks you through deploying Autonomous Shoot Clusters using `gardenadm` on your local machine. +[This document](../deployment/getting_started_locally_with_gardenadm.md) walks you through deploying Self-Hosted Shoot Clusters using `gardenadm` on your local machine. This setup can be used for trying out and developing `gardenadm` locally without additional infrastructure. The setup is also used for running e2e tests for `gardenadm` in CI. ## Scenarios -We distinguish between two different scenarios for bootstrapping autonomous shoot clusters: +We distinguish between two different scenarios for bootstrapping self-hosted shoot clusters: -- High Touch, meaning that there is no programmable infrastructure available. +- Unmanaged Infrastructure, meaning that there is no programmable infrastructure available. We consider this the "bare metal" or "edge" use-case, where at first machines must be (often manually) prepared by human operators. In this case, network setup (e.g., VPCs, subnets, route tables, etc.) and machine management are out of scope. -- Medium Touch, meaning that there is programmable infrastructure available where we can leverage [provider extensions](../../extensions/README.md#infrastructure-provider) and [`machine-controller-manager`](https://github.com/gardener/machine-controller-manager) in order to manage the network setup and the machines. +- Managed Infrastructure, meaning that there is programmable infrastructure available where we can leverage [provider extensions](../../extensions/README.md#infrastructure-provider) and [`machine-controller-manager`](https://github.com/gardener/machine-controller-manager) in order to manage the network setup and the machines. -The general procedure of bootstrapping an autonomous shoot cluster is similar in both scenarios. +The general procedure of bootstrapping a self-hosted shoot cluster is similar in both scenarios. diff --git a/docs/deployment/getting_started_locally_with_gardenadm.md b/docs/deployment/getting_started_locally_with_gardenadm.md index d56e956cf10..924190bcb95 100644 --- a/docs/deployment/getting_started_locally_with_gardenadm.md +++ b/docs/deployment/getting_started_locally_with_gardenadm.md @@ -1,11 +1,11 @@ -# Deploying Autonomous Shoot Clusters Locally +# Deploying Self-Hosted Shoot Clusters Locally > [!CAUTION] > The `gardenadm` tool is currently under development and considered highly experimental. > Do not use it in production environments. -> Read more about it in [GEP-28](../proposals/28-autonomous-shoot-clusters.md). +> Read more about it in [GEP-28](../proposals/28-self-hosted-shoot-clusters.md). -This document walks you through deploying Autonomous Shoot Clusters using `gardenadm` on your local machine. +This document walks you through deploying Self-Hosted Shoot Clusters using `gardenadm` on your local machine. This setup can be used for trying out and developing `gardenadm` locally without additional infrastructure. The setup is also used for running e2e tests for `gardenadm` in CI ([Prow](https://prow.gardener.cloud)). @@ -13,10 +13,10 @@ If you encounter difficulties, please open an issue so that we can make this pro ## Overview -`gardenadm` is a command line tool for bootstrapping Kubernetes clusters called "Autonomous Shoot Clusters". Read the [`gardenadm` documentation](../concepts/gardenadm.md) for more details on its concepts. +`gardenadm` is a command line tool for bootstrapping Kubernetes clusters called "Self-Hosted Shoot Clusters". Read the [`gardenadm` documentation](../concepts/gardenadm.md) for more details on its concepts. -In this guide, we will start a [KinD](https://kind.sigs.k8s.io/) cluster which hosts pods serving as machines for the autonomous shoot cluster – just as for shoot clusters of [provider-local](../extensions/provider-local.md). -The setup supports both the high-touch and medium-touch scenario of `gardenadm`. +In this guide, we will start a [KinD](https://kind.sigs.k8s.io/) cluster which hosts pods serving as machines for the self-hosted shoot cluster – just as for shoot clusters of [provider-local](../extensions/provider-local.md). +The setup supports both the "unmanaged infrastructure" and the "managed infrastructure" scenario of `gardenadm`. Based on [Skaffold](https://skaffold.dev/), the container images for all required components will be built and deployed into the cluster. This also includes the `gardenadm` CLI, which is installed on the machine pods by pulling the container image and extracting the binary. @@ -44,9 +44,9 @@ All following steps assume that you are using the kubeconfig for this KinD clust export KUBECONFIG=$PWD/example/gardener-local/kind/multi-zone/kubeconfig ``` -## High-Touch Scenario +## "Unmanaged Infrastructure" Scenario -Use the following command to prepare the `gardenadm` high-touch scenario: +Use the following command to prepare the `gardenadm` unmanaged infrastructure scenario: ```shell make gardenadm-up @@ -59,9 +59,9 @@ Afterward, you can use `kubectl exec` to execute `gardenadm` commands on the mac Let's start with exec'ing into the `machine-0` pod: ```shell -$ kubectl -n gardenadm-high-touch exec -it machine-0 -- bash +$ kubectl -n gardenadm-unmanaged-infra exec -it machine-0 -- bash root@machine-0:/# gardenadm -h -gardenadm bootstraps and manages autonomous shoot clusters in the Gardener project. +gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener project. ... root@machine-0:/# cat /gardenadm/resources/manifests.yaml @@ -83,13 +83,13 @@ Your Shoot cluster control-plane has initialized successfully! ... ``` -### Connecting to the Autonomous Shoot Cluster +### Connecting to the Self-Hosted Shoot Cluster -The machine pod's shell environment is configured for easily connecting to the autonomous shoot cluster. +The machine pod's shell environment is configured for easily connecting to the self-hosted shoot cluster. Just execute `kubectl` within a `bash` shell in the machine pod: ```shell -$ kubectl -n gardenadm-high-touch exec -it machine-0 -- bash +$ kubectl -n gardenadm-unmanaged-infra exec -it machine-0 -- bash root@machine-0:/# kubectl get node NAME STATUS ROLES AGE VERSION machine-0 Ready 4m11s v1.32.0 @@ -98,8 +98,8 @@ machine-0 Ready 4m11s v1.32.0 You can also copy the kubeconfig to your local machine and use a port-forward to connect to the cluster's API server: ```shell -$ kubectl -n gardenadm-high-touch exec -it machine-0 -- cat /etc/kubernetes/admin.conf | sed 's/api.root.garden.local.gardener.cloud/localhost:6443/' > /tmp/shoot--garden--root.conf -$ kubectl -n gardenadm-high-touch port-forward pod/machine-0 6443:443 +$ kubectl -n gardenadm-unmanaged-infra exec -it machine-0 -- cat /etc/kubernetes/admin.conf | sed 's/api.root.garden.local.gardener.cloud/localhost:6443/' > /tmp/shoot--garden--root.conf +$ kubectl -n gardenadm-unmanaged-infra port-forward pod/machine-0 6443:443 # in a new terminal $ export KUBECONFIG=/tmp/shoot--garden--root.conf @@ -117,7 +117,7 @@ Then exec into the `machine-1` pod to run the command: root@machine-0:/# gardenadm token create --print-join-command # now copy the output, terminate the exec session and start a new one for machine-1 -$ kubectl -n gardenadm-high-touch exec -it machine-1 -- bash +$ kubectl -n gardenadm-unmanaged-infra exec -it machine-1 -- bash # paste the copied 'gardenadm join' command here and execute it root@machine-1:/# gardenadm join ... ... @@ -125,7 +125,7 @@ Your node has successfully been instructed to join the cluster as a worker! ... ``` -Using the kubeconfig as described in [this section](#connecting-to-the-autonomous-shoot-cluster), you should now be able to see the new node in the cluster: +Using the kubeconfig as described in [this section](#connecting-to-the-self-hosted-shoot-cluster), you should now be able to see the new node in the cluster: ```shell $ kubectl get no @@ -134,32 +134,32 @@ machine-0 Ready 10m v1.32.0 machine-1 Ready 37s v1.32.0 ``` -## Medium-Touch Scenario +## "Managed Infrastructure" Scenario -Use the following command to prepare the `gardenadm` medium-touch scenario: +Use the following command to prepare the `gardenadm` managed infrastructure scenario: ```shell -make gardenadm-up SCENARIO=medium-touch +make gardenadm-up SCENARIO=managed-infra ``` -This will first build the needed images and then render the needed manifests for `gardenadm bootstrap` to the [`./dev-setup/gardenadm/resources/generated/medium-touch`](../../dev-setup/gardenadm/resources/generated/medium-touch) directory. +This will first build the needed images and then render the needed manifests for `gardenadm bootstrap` to the [`./dev-setup/gardenadm/resources/generated/managed-infra`](../../dev-setup/gardenadm/resources/generated/managed-infra) directory. -### Bootstrapping the Autonomous Shoot Cluster +### Bootstrapping the Self-Hosted Shoot Cluster Use `go run` to execute `gardenadm` commands on your machine: ```shell $ export IMAGEVECTOR_OVERWRITE=$PWD/dev-setup/gardenadm/resources/generated/.imagevector-overwrite.yaml -$ go run ./cmd/gardenadm bootstrap -d ./dev-setup/gardenadm/resources/generated/medium-touch +$ go run ./cmd/gardenadm bootstrap -d ./dev-setup/gardenadm/resources/generated/managed-infra ... [shoot--garden--root-control-plane-58ffc-2l6s7] Your Shoot cluster control-plane has initialized successfully! ... ``` -### Connecting to the Autonomous Shoot Cluster +### Connecting to the Self-Hosted Shoot Cluster -`gardenadm init` stores the kubeconfig of the autonomous shoot cluster in the `/etc/kubernetes/admin.conf` file on the control plane machine. -To connect to the autonomous shoot cluster, set the `KUBECONFIG` environment variable and execute `kubectl` within a `bash` shell in the machine pod: +`gardenadm init` stores the kubeconfig of the self-hosted shoot cluster in the `/etc/kubernetes/admin.conf` file on the control plane machine. +To connect to the self-hosted shoot cluster, set the `KUBECONFIG` environment variable and execute `kubectl` within a `bash` shell in the machine pod: ```shell $ machine="$(kubectl -n shoot--garden--root get po -l app=machine -oname | head -1 | cut -d/ -f2)" @@ -185,9 +185,9 @@ NAME STATUS ROLES AGE machine-shoot--garden--root-control-plane-58ffc-2l6s7 Ready 4m11s v1.33.0 ``` -## Connecting the Autonomous Shoot Cluster to Gardener +## Connecting the Self-Hosted Shoot Cluster to Gardener -After you have successfully bootstrapped an autonomous shoot cluster (either via the [high touch](#high-touch-scenario) or the [medium touch](#medium-touch-scenario) scenario), you can connect it to an existing Gardener system. +After you have successfully bootstrapped a self-hosted shoot cluster (either via the [unmanaged infrastructure](#unmanaged-infrastructure-scenario) or the [managed infrastructure](#managed-infrastructure-scenario) scenario), you can connect it to an existing Gardener system. For this, you need to have a Gardener running locally in your KinD cluster. In order to deploy it, you can run @@ -199,10 +199,10 @@ This will deploy [`gardener-operator`](../concepts/operator.md) and create a `Ga Find all information about it [here](getting_started_locally.md#alternative-way-to-set-up-garden-and-seed-leveraging-gardener-operator). Note, that in this setup, no `Seed` will be registered in the Gardener - it's just a plain garden cluster without the ability to create regular shoot clusters. -Once above command is finished, you can connect the autonomous shoot cluster to this Gardener instance: +Once above command is finished, you can connect the self-hosted shoot cluster to this Gardener instance: ```shell -$ kubectl -n gardenadm-high-touch exec -it machine-0 -- bash +$ kubectl -n gardenadm-unmanaged-infra exec -it machine-0 -- bash root@machine-0:/# gardenadm connect 2025-07-03T12:21:49.586Z INFO Command is work in progress ``` @@ -212,13 +212,13 @@ root@machine-0:/# gardenadm connect Based on the described setup, you can execute the e2e test suite for `gardenadm`: ```shell -make gardenadm-up SCENARIO=high-touch +make gardenadm-up SCENARIO=unmanaged-infra make gardenadm-up SCENARIO=connect -make test-e2e-local-gardenadm-high-touch +make test-e2e-local-gardenadm-unmanaged-infra # or -make gardenadm-up SCENARIO=medium-touch -make test-e2e-local-gardenadm-medium-touch +make gardenadm-up SCENARIO=managed-infra +make test-e2e-local-gardenadm-managed-infra ``` ## Tear Down the KinD Cluster diff --git a/docs/extensions/provider-local.md b/docs/extensions/provider-local.md index 4f89c353500..ea959155e1f 100644 --- a/docs/extensions/provider-local.md +++ b/docs/extensions/provider-local.md @@ -85,7 +85,7 @@ data: rewrite stop name regex api.local.local.internal.local.gardener.cloud istio-ingressgateway.istio-ingress.svc.cluster.local answer auto ``` -For autonomous shoots, the controller configures the CoreDNS `template` plugin instead of using a `rewrite` rule for each `DNSRecord`: +For self-hosted shoots, the controller configures the CoreDNS `template` plugin instead of using a `rewrite` rule for each `DNSRecord`: ```yaml data: diff --git a/docs/extensions/registration.md b/docs/extensions/registration.md index 625a7c12ec9..92580a67360 100644 --- a/docs/extensions/registration.md +++ b/docs/extensions/registration.md @@ -275,7 +275,7 @@ In order to allow extension controller deployments to get information about the gardenlet: featureGates: ``` -- If the extension is deployed in an [autonomous shoot cluster](../proposals/28-autonomous-shoot-clusters.md), then the `.gardener.autonomousShootCluster` field is additionally propagated and set to `true`. +- If the extension is deployed in an [self-hosted shoot cluster](../proposals/28-self-hosted-shoot-clusters.md), then the `.gardener.selfHostedShootCluster` field is additionally propagated and set to `true`. Extension controller deployments can use this information in their Helm chart in case they require knowledge about the garden and the seed environment. The list might be extended in the future. diff --git a/docs/extensions/shoot-webhooks.md b/docs/extensions/shoot-webhooks.md index 053e27f931f..bfa6ac5a171 100644 --- a/docs/extensions/shoot-webhooks.md +++ b/docs/extensions/shoot-webhooks.md @@ -22,8 +22,8 @@ The shoot's kube-apiserver must be allowed to talk to the provider extension. To achieve this, you need to make sure that the relevant `NetworkPolicy` get created for allowing the network traffic. Please refer to [this guide](../operations/network_policies.md#webhook-servers) for more information. -## Autonomous Shoot Clusters +## Self-Hosted Shoot Clusters -If running in an autonomous shoot cluster, the shoot webhooks should be merged into the seed webhooks. +If running in a self-hosted shoot cluster, the shoot webhooks should be merged into the seed webhooks. You can do so by setting the `mergeShootWebhooksIntoSeedWebhooks` to `true` in the `extensions/pkg/webhook/cmd.AddToManager` function. -Take a look at [this document](registration.md#helm-values) in order to determine whether the extension runs in an autonomous shoot cluster. +Take a look at [this document](registration.md#helm-values) in order to determine whether the extension runs in a self-hosted shoot cluster. diff --git a/docs/proposals/28-autonomous-shoot-clusters.md b/docs/proposals/28-self-hosted-shoot-clusters.md similarity index 84% rename from docs/proposals/28-autonomous-shoot-clusters.md rename to docs/proposals/28-self-hosted-shoot-clusters.md index 5159f6c91eb..cf5e255b8e3 100644 --- a/docs/proposals/28-autonomous-shoot-clusters.md +++ b/docs/proposals/28-self-hosted-shoot-clusters.md @@ -1,5 +1,5 @@ --- -title: Autonomous Shoot Clusters +title: Self-Hosted Shoot Clusters gep-number: 28 creation-date: 2024-09-17 status: implementable @@ -9,11 +9,11 @@ authors: - "@vlerenc" --- -# GEP-28: Autonomous Shoot Clusters +# GEP-28: Self-Hosted Shoot Clusters ## Table of Contents -- [GEP-28: Autonomous Shoot Clusters](#gep-28-autonomous-shoot-clusters) +- [GEP-28: Self-Hosted Shoot Clusters](#gep-28-self-hosted-shoot-clusters) - [Table of Contents](#table-of-contents) - [Summary](#summary) - [Motivation](#motivation) @@ -35,19 +35,19 @@ authors: - [`gardenadm version`](#gardenadm-version) - [`Shoot` API](#shoot-api) - [Scenarios](#scenarios) - - [High Touch](#high-touch) - - [Medium Touch](#medium-touch) + - [Unmanaged Infrastructure](#unmanaged-infrastructure) + - [Managed Infrastructure](#managed-infrastructure) - [`gardenadm bootstrap`](#gardenadm-bootstrap) - [Disaster Recovery](#disaster-recovery) - [Single-Node Failure](#single-node-failure) - [Multi-Node Failure](#multi-node-failure) - [Data Center Failure](#data-center-failure) - [Future Work](#future-work) - - [Autonomous Shoot Clusters for End-Users](#autonomous-shoot-clusters-for-end-users) + - [Self-Hosted Shoot Clusters for End-Users](#self-hosted-shoot-clusters-for-end-users) - [Network Connectivity to Garden Cluster](#network-connectivity-to-garden-cluster) - [Changes to Shoot Components](#changes-to-shoot-components) - [Air-Gapped Gardener Landscapes](#air-gapped-gardener-landscapes) - - [Scaling the Control Plane of Autonomous Shoot Clusters](#scaling-the-control-plane-of-autonomous-shoot-clusters) + - [Scaling the Control Plane of Self-Hosted Shoot Clusters](#scaling-the-control-plane-of-self-hosted-shoot-clusters) - [Alternatives](#alternatives) - [Using Existing Tools](#using-existing-tools) - [Using Existing Services](#using-existing-services) @@ -55,34 +55,34 @@ authors: ## Summary -Gardener ships with functionality that would allow the creation of "autonomous" shoot clusters. -Such "autonomous" clusters would be regular shoot clusters with the main difference that their control plane runs on dedicated nodes within the cluster itself (instead of a seed cluster which typically resides in another infrastructure account and network domain). +Gardener ships with functionality that would allow the creation of "self-hosted" shoot clusters. +Such "self-hosted" clusters would be regular shoot clusters with the main difference that their control plane runs on dedicated nodes within the cluster itself (instead of a seed cluster which typically resides in another infrastructure account and network domain). This deployment model is known as the typical/standard Kubernetes deployment model, while Gardener's regular shoot clusters today follow the "Kubeception" deployment model. ![Overview](assets/28-overview.png) > [!IMPORTANT] > This shall not be a drop-in replacement for [`k3s`](https://k3s.io) or [`kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm) (that would be too far away from Gardener's mission statement, TCO goals, and current implementation), but we want to offer this new type of shoot clusters as a separate flavor to enable on-prem use-cases that cannot be served with Gardener today. -> By allowing the remote management through Gardener, "autonomous" clusters will meet our mission statement (e.g., in regards to low TCO) while operating without the need of a seed cluster. +> By allowing the remote management through Gardener, "self-hosted" clusters will meet our mission statement (e.g., in regard to low TCO) while operating without the need of a seed cluster. -Such autonomous shoot clusters are meant to operate autonomously, but not to exist completely independently of Gardener. +Such self-hosted shoot clusters are meant to operate autonomously, but not to exist completely independently of Gardener. Hence, after initial creation, they shall be connected to an existing Gardener system such that the established cluster management functionality via the `Shoot` API can be applied. -Furthermore, autonomous shoot clusters are not considered a replacement or alternative for regular shoot clusters. +Furthermore, self-hosted shoot clusters are not considered a replacement or alternative for regular shoot clusters. They should be only used for special use-cases or requirements as their creation time will be longer and their costs will be higher. In this light, a high cluster creation/deletion churn rate is neither expected nor in scope. Gardener's APIs and cluster topology are defined in a way such that seed clusters are a separate "security domain" and not accessible to end-users. -With autonomous shoot clusters, seed clusters do not exist anymore, hence, running a `gardenlet` and certain seed system components (like extensions) within the end-user's infrastructure requires rework and adaptation of some of today's concepts. -As operating autonomous shoot clusters is a challenge big enough on its own, we'll start with focusing only on how Gardener operators can create and connect autonomous shoot clusters. -Allowing "untrusted" end-users to create such clusters is out of scope of this GEP and considered to be future work ([see below](#autonomous-shoot-clusters-for-end-users)). +With self-hosted shoot clusters, seed clusters do not exist anymore, hence, running a `gardenlet` and certain seed system components (like extensions) within the end-user's infrastructure requires rework and adaptation of some of today's concepts. +As operating self-hosted shoot clusters is a challenge big enough on its own, we'll start with focusing only on how Gardener operators can create and connect self-hosted shoot clusters. +Allowing "untrusted" end-users to create such clusters is out of scope of this GEP and considered to be future work ([see below](#self-hosted-shoot-clusters-for-end-users)). ## Motivation Gardener is a great tool for creating and managing clusters with very low TCO. Part of this success is its architecture, i.e., running control planes in seed clusters under joint supervision by Gardener and Kubernetes (a.k.a. Kubeception deployment model). -However, there is sufficient pull to also find a way to create autonomous shoot clusters using Gardener (e.g., for the edge or for temporarily air-gapped scenarios) where the control plane must run side-by-side and cannot run in seed clusters. +However, there is sufficient pull to also find a way to create self-hosted shoot clusters using Gardener (e.g., for the edge or for temporarily air-gapped scenarios) where the control plane must run side-by-side and cannot run in seed clusters. We want to establish Gardener in (new) environments where clusters cannot run their control plane "somewhere else", but where it needs to be co-located with the worker nodes, e.g., to avoid costly or exposed network traffic, firewall the entire cluster off, rely on the control plane while the cluster is air-gapped/has no internet access, generally avoid runtime dependencies, fulfill compliance obligations, etc. In addition, Gardener itself requires to run in a conformant Kubernetes cluster satisfying the [minimum Kubernetes version requirements](../usage/shoot-operations/supported_k8s_versions.md#garden-clusters), but so far there was no way of setting up or managing this initial cluster via Gardener itself. @@ -91,20 +91,20 @@ Augmenting Gardener's capability to also manage the initial cluster will help to ### Goals -- Creation of autonomous shoot clusters running dedicated nodes for control plane components +- Creation of self-hosted shoot clusters running dedicated nodes for control plane components - Maximize re-use of existing code/functionality to prevent re-inventing the wheel - Enable cluster management and upgrades via the `Shoot` API of an existing Gardener ### Non-Goals -- Creation/deletion of autonomous shoot clusters with high churn rates and convenience compared to regular shoot clusters -- Allow end-users to create autonomous shoot clusters and connect them to a Gardener system - this aspect will be covered in a future GEP after this one has been implemented -- Provide functionality to create autonomous shoot clusters via the API of an existing Gardener system +- Creation/deletion of self-hosted shoot clusters with high churn rates and convenience compared to regular shoot clusters +- Allow end-users to create self-hosted shoot clusters and connect them to a Gardener system - this aspect will be covered in a future GEP after this one has been implemented +- Provide functionality to create self-hosted shoot clusters via the API of an existing Gardener system - Support day-2 operations without connecting the cluster to an existing Gardener system ## Proposal -In short, the basic idea is to develop a new command line tool that is going to be shipped with every Gardener release, `gardenadm`, which can be used to bootstrap an autonomous shoot cluster. +In short, the basic idea is to develop a new command line tool that is going to be shipped with every Gardener release, `gardenadm`, which can be used to bootstrap a self-hosted shoot cluster. gardenadm @@ -129,17 +129,17 @@ Similar to [`kubeadm`'s prerequisites](https://kubernetes.io/docs/setup/producti `gardenadm` will have the following commands: -- [`gardenadm discover`](#gardenadm-discover) to discover the configuration of the Gardener system the autonomous shoot cluster shall be connected to later. +- [`gardenadm discover`](#gardenadm-discover) to discover the configuration of the Gardener system the self-hosted shoot cluster shall be connected to later. - [`gardenadm init`](#gardenadm-init) to bootstrap a first control plane node. - [`gardenadm token`](#gardenadm-token) to manage bootstrap and discovery tokens for `gardenadm join`. - [`gardenadm join`](#gardenadm-join) to bootstrap further control plane nodes, or to bootstrap worker nodes and join them to the cluster. - [`gardenadm connect`](#gardenadm-connect) to deploy a `gardenlet` for further cluster management. - [`gardenadm version`](#gardenadm-version) to print the `gardenadm` version. -The general flow for bootstrapping an autonomous shoot cluster will be as follows: +The general flow for bootstrapping a self-hosted shoot cluster will be as follows: 1. The user installs `gardenadm` on the first control plane node and runs `gardenadm discover`. - This will download relevant configuration from the Gardener system the autonomous shoot cluster shall be later connected to (e.g., `Controller{Registration,Deployment}`s, `CloudProfile`s, etc.). + This will download relevant configuration from the Gardener system the self-hosted shoot cluster shall be later connected to (e.g., `Controller{Registration,Deployment}`s, `CloudProfile`s, etc.). If there is no existing Gardener system yet, the needed configuration must be provided manually. 2. The downloaded configuration and a `Shoot` manifest are provided to the `gardenadm init` command. This will bootstrap the first control plane node. @@ -156,11 +156,11 @@ A more detailed description of the commands is following. This command takes a kubeconfig to an existing Gardener system, and a `Shoot` manifest. It downloads the required `CloudProfile`, `ControllerRegistration` and `ControllerDeployment` resources and stores them locally for usage of other `gardenadm` commands. -This simplifies the crafting of these documents, and it also ensures that the configuration and versions used for the bootstrapping of the autonomous shoot cluster fit to the configuration later found in the connected Gardener system. +This simplifies the crafting of these documents, and it also ensures that the configuration and versions used for the bootstrapping of the self-hosted shoot cluster fit to the configuration later found in the connected Gardener system. #### `gardenadm init` -This command bootstraps the first control plane node of the autonomous shoot cluster. +This command bootstraps the first control plane node of the self-hosted shoot cluster. The configuration is specified via separate files. It needs a `Shoot` manifest and some additional configuration files, which usually reside in the garden cluster, e.g., a `CloudProfile` (best downloaded with `gardenadm discover`). For the initial cluster Gardener bootstrapping use-case however, these won't come from a pre-existing garden cluster as it doesn't exist yet. @@ -244,7 +244,7 @@ The static pod manifest can then be exchanged on the host and replace the "boots > [!NOTE] > At the time of writing this GEP, it is not yet fully clear how/whether this approach fits into the design plans for `etcd-druid`. -> An alternative idea is to make `etcd-druid` aware of the concept of autonomous shoot clusters and give it further control into the management of static `etcd` pods. +> An alternative idea is to make `etcd-druid` aware of the concept of self-hosted shoot clusters and give it further control into the management of static `etcd` pods. > The details are still subject to discussion and will become clearer in the future. > Since we need a "bootstrap `etcd`" anyway to get the cluster up and running, we can proceed with this GEP and its implementation in parallel while the discussion around how `etcd-druid` can take over the management is ongoing. @@ -290,16 +290,16 @@ Apart from this, `etcd` also needs to be re-configured to have the new node join #### `gardenadm connect` -This command deploys a `gardenlet` into the autonomous shoot cluster and connects it to an existing garden cluster. +This command deploys a `gardenlet` into the self-hosted shoot cluster and connects it to an existing garden cluster. As part of this operation, the corresponding `Shoot` resource will be created in the garden cluster. -Afterwards, the `gardenlet` will reconcile the autonomous shoot cluster similarly to regular shoot clusters. +Afterwards, the `gardenlet` will reconcile the self-hosted shoot cluster similarly to regular shoot clusters. > [!NOTE] -> When there is no existing Gardener system yet (for example, because the created autonomous shoot cluster is the initial cluster), then [`gardener-operator`](../concepts/operator.md) must be deployed first, and the [`Garden`](../../example/operator/20-garden.yaml) must be created, before the `gardenlet` can be deployed. +> When there is no existing Gardener system yet (for example, because the created self-hosted shoot cluster is the initial cluster), then [`gardener-operator`](../concepts/operator.md) must be deployed first, and the [`Garden`](../../example/operator/20-garden.yaml) must be created, before the `gardenlet` can be deployed. > -> Also, it is not yet clear whether the existing `gardenlet` binary will be re-used, or whether we better create a dedicated, new binary with slimmed down functionality catered for only managing autonomous shoot clusters. +> Also, it is not yet clear whether the existing `gardenlet` binary will be re-used, or whether we better create a dedicated, new binary with slimmed down functionality catered for only managing self-hosted shoot clusters. -Depending on the scenario, an autonomous shoot cluster may be air-gapped for periods of time. +Depending on the scenario, a self-hosted shoot cluster may be air-gapped for periods of time. Therefore, please note that the cluster should function also when air-gapped/when it has limited or no network connectivity. Updates can not be performed during that time, but ordinary operations with the cluster itself (e.g., creating pods or services) should still be possible. It is important that the [version skew](../deployment/version_skew_policy.md) is always respected. @@ -310,7 +310,7 @@ This command prints the version of `gardenadm`. ### `Shoot` API -We need some mechanism to distinguish regular shoot clusters from autonomous shoot clusters. +We need some mechanism to distinguish regular shoot clusters from self-hosted shoot clusters. As the most prominent difference is that there are dedicated machines for the control plane components, we propose to notice the difference based on the existence of a worker pool for the control plane: ```yaml @@ -327,14 +327,14 @@ An alternative is to augment `.spec.controlPlane` with this information, however We distinguish between two different scenarios (for now): -- [High Touch](#high-touch), meaning that there is no programmable infrastructure available. +- [Unmanaged Infrastructure](#unmanaged-infrastructure), meaning that there is no programmable infrastructure available. We consider this the "bare metal" or "edge" use-case, where at first machines must be (often manually) prepared by human operators. In this case, network setup (e.g., VPCs, subnets, route tables, etc.) and machine management are out of scope. -- [Medium Touch](#medium-touch), meaning that there is programmable infrastructure available where we can leverage [provider extensions](../../extensions/README.md#infrastructure-provider) and [`machine-controller-manager`](https://github.com/gardener/machine-controller-manager) in order to manage the network setup and the machines. +- [Managed Infrastructure](#managed-infrastructure), meaning that there is programmable infrastructure available where we can leverage [provider extensions](../../extensions/README.md#infrastructure-provider) and [`machine-controller-manager`](https://github.com/gardener/machine-controller-manager) in order to manage the network setup and the machines. -The general procedure of bootstrapping an autonomous shoot cluster is similar in both scenarios. +The general procedure of bootstrapping a self-hosted shoot cluster is similar in both scenarios. -#### High Touch +#### Unmanaged Infrastructure In this scenario, the starting point is right on the first machine that shall be transformed into a control plane node. It is assumed that the network was set up properly (see [Prerequisites](#prerequisites)). @@ -342,15 +342,15 @@ Since machine management is out of scope, upgrades requiring to roll out the nod We will leverage the functionality that is to be developed in the scope of [#10219](https://github.com/gardener/gardener/issues/10219). This will also include operating system upgrades [planned for Garden Linux](https://github.com/gardener/gardener/issues/10219#issuecomment-2262299606). -#### Medium Touch +#### Managed Infrastructure In this scenario, we can leverage the existing [`extensions.gardener.cloud/v1alpha1.Infrastructure` API](../extensions/resources/infrastructure.md) in order to let the respective provider extension generate the necessary network resources (VPCs, subnets, etc., whatever is needed for the respective infrastructure). -`machine-controller-manager` can be used to create the control plane nodes (and later also the worker nodes) of the autonomous shoot cluster. +`machine-controller-manager` can be used to create the control plane nodes (and later also the worker nodes) of the self-hosted shoot cluster. The challenge is that both Gardener provider extensions and `machine-controller-manager` depend on a pre-existing Kubernetes cluster or in general an API server. Hence, we propose to use a [KinD cluster](https://kind.sigs.k8s.io), deploy the needed components into it, and create the necessary `Infrastructure`, `MachineClass`, and `MachineDeployment` resources. This way, these components can be reused as-is, without big changes or re-wiring of these complex components to run in-process with a fake client (as we do during the `gardenadm init` flow for the simpler components). -Once the controllers have reconciled successfully, we can export the created infrastructure state and the `Machine` resources, bootstrap the autonomous shoot cluster similar to the [general flow](#commands) also needed for the [High Touch scenario](#high-touch), and import the state again into the created cluster. +Once the controllers have reconciled successfully, we can export the created infrastructure state and the `Machine` resources, bootstrap the self-hosted shoot cluster similar to the [general flow](#commands) also needed for the [Unmanaged Infrastructure scenario](#unmanaged-infrastructure), and import the state again into the created cluster. There is no special assumption about the location of the KinD cluster. It should run somewhere where it can access the API of the target infrastructure. @@ -368,8 +368,8 @@ Since we assume that this scenario will be the prominent one, we plan to add ano ##### `gardenadm bootstrap` -This command creates the virtual network and machines to be used for the control plane of the autonomous shoot cluster. -It uses an existing KinD cluster to temporarily deploy existing Gardener components (unaware of the autonomous shoot cluster use-case). +This command creates the virtual network and machines to be used for the control plane of the self-hosted shoot cluster. +It uses an existing KinD cluster to temporarily deploy existing Gardener components (unaware of the self-hosted shoot cluster use-case). These components, such as provider extensions and `machine-controller-manager`, are used to create the necessary infrastructure and machines. The high-level steps are as follows: @@ -384,7 +384,7 @@ The high-level steps are as follows: 8. Create a `Bastion` resource to be able to access the machines and execute `gardenadm init` (can be later optimized via `cloud-init` through a cloud config that invokes `gardenadm init` directly when creating the machines). The states of the `Infrastructure` and `Machine` resources need to be exported from the KinD cluster. -Once the autonomous shoot cluster control plane is running, the state should be imported so that it can be managed by the provider extension and `machine-controller-manager` respectively. +Once the self-hosted shoot cluster control plane is running, the state should be imported so that it can be managed by the provider extension and `machine-controller-manager` respectively. The KinD cluster can be deleted afterwards. @@ -392,7 +392,7 @@ The KinD cluster can be deleted afterwards. Disasters can strike in many different ways. This section discusses disasters in ascending order of severity. -Some scenarios can be handled by autonomous shoot clusters automatically, others require manual intervention. +Some scenarios can be handled by self-hosted shoot clusters automatically, others require manual intervention. #### Single-Node Failure @@ -433,59 +433,59 @@ The resolution/handling of data center failures is out of scope of this GEP. ## Future Work -### Autonomous Shoot Clusters for End-Users +### Self-Hosted Shoot Clusters for End-Users -As implicitly mentioned in the [Non-Goals](#non-goals) section, the focus of this GEP is to allow Gardener operators to create autonomous shoot clusters. +As implicitly mentioned in the [Non-Goals](#non-goals) section, the focus of this GEP is to allow Gardener operators to create self-hosted shoot clusters. However, the creation of such clusters by end-users is not covered by this GEP. This could become a future topic if we consider it relevant enough. -Handling creation of autonomous shoot clusters by end-users requires refined access controls for the `gardenlet`. +Handling creation of self-hosted shoot clusters by end-users requires refined access controls for the `gardenlet`. Currently, the `gardenlet` runs as part of a seed cluster and has access to some resources, which are not accessible to end-users but only to operators. An example resource is the `ControllerDeployment`, which may contain credentials, e.g., for certificate management or domain name services. In case this scenario is desired and the security boundary between Gardener operators and end-users should stay as is, the extension deployment needs to be adapted. -While it may be appealing to have support for autonomous shoot cluster for end-users, it is important to understand that this feature may complicate the world for both operators and end-users. -The benefits of autonomous shoot clusters for the garden runtime cluster and/or soil clusters, i.e., unmanaged seed clusters, is fairly obvious: +While it may be appealing to have support for self-hosted shoot cluster for end-users, it is important to understand that this feature may complicate the world for both operators and end-users. +The benefits of self-hosted shoot clusters for the garden runtime cluster and/or soil clusters, i.e., unmanaged seed clusters, is fairly obvious: It allows to create a Gardener landscape without external dependency to a Kubernetes service provider. However, the benefits for end-users are less clear (except for the mentioned edge/on-prem/air-gapped use cases). -It needs to be carefully considered and documented for what scenarios autonomous shoot clusters are the right choice, and when ordinary shoot clusters with their control plane running on a seed cluster are the better way to go. +It needs to be carefully considered and documented for what scenarios self-hosted shoot clusters are the right choice, and when ordinary shoot clusters with their control plane running on a seed cluster are the better way to go. Without this consideration, the feature may increase the operational complexity for Gardener operators, and the cognitive load for end-users. ### Network Connectivity to Garden Cluster -Another aspect that was only barely touched in this GEP is the reliability of the network connectivity between the autonomous shoot cluster and Gardener. +Another aspect that was only barely touched in this GEP is the reliability of the network connectivity between the self-hosted shoot cluster and Gardener. In the [`gardenadm connect` section](#gardenadm-connect), it was mentioned that the cluster should function also when air-gapped/when it has limited or no network connectivity. However, we are deliberately vague what "limited network connectivity" means. Detailing this aspect out can be considered future work. -As mentioned in the corresponding section, the [version skew](../deployment/version_skew_policy.md) plays an important role in this context, determining the maximum time the autonomous shoot cluster can be disconnected from Gardener. +As mentioned in the corresponding section, the [version skew](../deployment/version_skew_policy.md) plays an important role in this context, determining the maximum time the self-hosted shoot cluster can be disconnected from Gardener. Long term support of Gardener and associated components may also be considered future work. ### Changes to Shoot Components -Due to the changed network layout of autonomous shoot clusters, the usage of some shoot components needs to be reconsidered. +Due to the changed network layout of self-hosted shoot clusters, the usage of some shoot components needs to be reconsidered. Examples are the VPN connection between control plane and data plane, the `apiserver-proxy` and the way metrics are scraped. This GEP does not cover these changes in detail, but they are important to be considered when implementing it. The following examples are just used to illustrate the need for changes, but are not meant to be exhaustive: - The VPN connection between control plane and data plane is not needed anymore as the control plane runs on the same network as the data plane. Previously, the VPN connection was used to secure the communication between the control plane and the data plane. This was important as the control plane and the data plane were running in different networks with a potentially untrusted network in between. -In the autonomous shoot cluster scenario, this is no longer the case. +In the self-hosted shoot cluster scenario, this is no longer the case. The control plane can directly communicate with the data plane without the need for a VPN connection. - The `apiserver-proxy` (see [GEP-11](11-apiserver-network-proxy.md) for details) is used to forward requests from the data plane directed at `kubernetes.default.svc.cluster.local` to the control plane. - In the autonomous shoot cluster scenario, this can be simplified by directly pointing `kubernetes.default.svc.cluster.local` to the actual endpoints of the `kube-apiserver`. + In the self-hosted shoot cluster scenario, this can be simplified by directly pointing `kubernetes.default.svc.cluster.local` to the actual endpoints of the `kube-apiserver`. This way, the `apiserver-proxy` is no longer needed. - Another aspect is the [mutating webhook for injecting the `KUBERNETES_SERVICE_HOST` environment variable](../concepts/resource-manager.md#kubernetes-service-host-injection), which can be disabled in the autonomous shoot cluster scenario as well. + Another aspect is the [mutating webhook for injecting the `KUBERNETES_SERVICE_HOST` environment variable](../concepts/resource-manager.md#kubernetes-service-host-injection), which can be disabled in the self-hosted shoot cluster scenario as well. - In ordinary shoot clusters, the metrics of components running in the data plane are scraped indirectly via the `kube-apiserver`. This allows the observability stack to use the VPN connection without explicitly configuring it. - As stated above, the VPN connection is no longer needed in the autonomous shoot cluster scenario. + As stated above, the VPN connection is no longer needed in the self-hosted shoot cluster scenario. Therefore, the observability stack could also directly scrape metrics from the data plane components instead of going through the `kube-apiserver`. There may be other components, which are affected by the changed network layout and need to be adapted. ### Air-Gapped Gardener Landscapes -While it is already possible today to run a Gardener landscape in an air-gapped environment, the autonomous shoot cluster feature will make it easier to set up such a landscape. +While it is already possible today to run a Gardener landscape in an air-gapped environment, the self-hosted shoot cluster feature will make it easier to set up such a landscape. This is due to the fact that it removes the dependency on an external Kubernetes cluster to run the Garden and act as the initial seed cluster. Therefore, it may be worthwhile to take a closer look at the requirements for air-gapped Gardener landscapes and document them properly. @@ -498,12 +498,12 @@ Topics to consider (without any claim of completeness) are: The goal would be to provide a comprehensive guide on how to set up an air-gapped Gardener landscape, which can be applied on various infrastructures. It may also be useful for scenarios that are not fully air-gapped, but only desire selected parts of the sovereignty this provides. -### Scaling the Control Plane of Autonomous Shoot Clusters +### Scaling the Control Plane of Self-Hosted Shoot Clusters The control plane components of ordinary Gardener shoot clusters are scaled horizontally and vertically in various ways. Autoscaling is not covered in this GEP, but it is important in the long run to ensure efficient resource usage. -As far as this GEP is concerned, the control plane of autonomous shoot clusters is scaled horizontally by adding more control plane nodes (see [`gardenadm join`](#gardenadm-join)). +As far as this GEP is concerned, the control plane of self-hosted shoot clusters is scaled horizontally by adding more control plane nodes (see [`gardenadm join`](#gardenadm-join)). This approach can be problematic in various ways. As this GEP proposes to always deploy the full set of default control plane components on each control plane node, there may be more instances running for `kube-scheduler` and `kube-controller-manager` than practically necessary. @@ -511,7 +511,7 @@ While this may be a good approach in scaling `kube-apiserver` horizontally, it m Furthermore, from `etcd`'s perspective the amount of control plane nodes should ideally be odd and no more than seven (see [etcd faq](https://etcd.io/docs/v3.4/faq/)). This further limits the scalability of the control plane. -Therefore, it may be a good idea to investigate how node roles can help improve efficiency in a scale-out autonomous shoot cluster scenario. +Therefore, it may be a good idea to investigate how node roles can help improve efficiency in a scale-out self-hosted shoot cluster scenario. Some control plane nodes could include the full set of default control plane components while others may use a reduced set of components. This could be achieved by having roles for either individual components or different set of components. @@ -522,14 +522,14 @@ Yet another option in the contrary direction would be to allow non-control plane This would allow to use the control plane nodes to their full capacity, but it would also blur the separation between control and data plane nodes. While it may improve the resource utilisation of control plane nodes, it may also introduce security risks. -In any case, revisiting the scaling problem is required to ensure efficient resource usage with growing control plane usage in the context of autonomous shoot clusters. +In any case, revisiting the scaling problem is required to ensure efficient resource usage with growing control plane usage in the context of self-hosted shoot clusters. ## Alternatives ### Using Existing Tools Existing tools, e.g., [`k3s`](https://k3s.io) or [`kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm), could be used to create the initial Kubernetes cluster. -This cluster could then be "adopted" by Gardener and managed by the `gardenlet` as described for the day-2 operations in the autonomous shoot cluster case. +This cluster could then be "adopted" by Gardener and managed by the `gardenlet` as described for the day-2 operations in the self-hosted shoot cluster case. It was deemed much more complex to align a cluster created by a third-party tool with Gardener's expectations than to create the cluster directly with Gardener. The fact that these tools evolve independently from Gardener and may introduce breaking changes was another reason to prefer the direct creation with Gardener. @@ -542,7 +542,7 @@ For example, it may be undesirable/impossible for an air-gapped environment to h ### Using Existing Gardener -Using an existing Gardener to create an autonomous cluster is an option, yet it requires a more complex satisfaction of prerequisites to make this approach work. +Using an existing Gardener to create a self-hosred cluster is an option, yet it requires a more complex satisfaction of prerequisites to make this approach work. For example, network connectivity from a seed cluster (where extensions and a `gardenlet` runs) to the target infrastructure must exist. When the initial cluster shall be created, pivoting from a local KinD cluster can be an option, but it also requires to solve the challenge of migrating the Gardener control plane. In summary, we considered this approach more complex to develop and use. diff --git a/example/90-shoot.yaml b/example/90-shoot.yaml index 76fe6d25028..db868ebb9bb 100644 --- a/example/90-shoot.yaml +++ b/example/90-shoot.yaml @@ -150,7 +150,7 @@ spec: # sysctls: # optional, allows to specify kernel settings to override defaults # net.ipv4.tcp_wmem: "4096 131072 16777216" # net.ipv4.tcp_rmem: "4096 131072 16777216" - # controlPlane: # mark the shoot as "autonomous shoot cluster", see GEP-28 + # controlPlane: # mark the shoot as "self-hosted shoot cluster", see GEP-28 # backup: # provider: # e.g., aws, azure, gcp, ... # providerConfig: diff --git a/extensions/pkg/controller/cmd/options.go b/extensions/pkg/controller/cmd/options.go index 58e282aeb17..c5703d13202 100644 --- a/extensions/pkg/controller/cmd/options.go +++ b/extensions/pkg/controller/cmd/options.go @@ -66,9 +66,9 @@ const ( // GardenerVersionFlag is the name of the command line flag containing the Gardener version. GardenerVersionFlag = "gardener-version" - // AutonomousShootClusterFlag is the name of the command line flag indicating that the extension runs in an - // autonomous shoot cluster. - AutonomousShootClusterFlag = "autonomous-shoot-cluster" + // SelfHostedShootClusterFlag is the name of the command line flag indicating that the extension runs in a + // self-hosted shoot cluster. + SelfHostedShootClusterFlag = "self-hosted-shoot-cluster" // LogLevelFlag is the name of the command line flag containing the log level. LogLevelFlag = "log-level" @@ -479,8 +479,8 @@ type SwitchConfig struct { type GeneralOptions struct { // GardenerVersion is the version of the Gardener. GardenerVersion string - // AutonomousShootCluster indicates whether the extension runs in an autonomous shoot cluster. - AutonomousShootCluster bool + // SelfHostedShootCluster indicates whether the extension runs in a self-hosted shoot cluster. + SelfHostedShootCluster bool config *GeneralConfig } @@ -489,13 +489,13 @@ type GeneralOptions struct { type GeneralConfig struct { // GardenerVersion is the version of the Gardener. GardenerVersion string - // AutonomousShootCluster indicates whether the extension runs in an autonomous shoot cluster. - AutonomousShootCluster bool + // SelfHostedShootCluster indicates whether the extension runs in a self-hosted shoot cluster. + SelfHostedShootCluster bool } // Complete implements Complete. func (r *GeneralOptions) Complete() error { - r.config = &GeneralConfig{r.GardenerVersion, r.AutonomousShootCluster} + r.config = &GeneralConfig{r.GardenerVersion, r.SelfHostedShootCluster} return nil } @@ -507,5 +507,5 @@ func (r *GeneralOptions) Completed() *GeneralConfig { // AddFlags implements Flagger.AddFlags. func (r *GeneralOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&r.GardenerVersion, GardenerVersionFlag, "", "Version of the gardenlet.") - fs.BoolVar(&r.AutonomousShootCluster, AutonomousShootClusterFlag, false, "Does the extension run in an autonomous shoot cluster?") + fs.BoolVar(&r.SelfHostedShootCluster, SelfHostedShootClusterFlag, false, "Does the extension run in a self-hosted shoot cluster?") } diff --git a/extensions/pkg/controller/controlplane/genericactuator/actuator.go b/extensions/pkg/controller/controlplane/genericactuator/actuator.go index 9465bc76b54..502442e3889 100644 --- a/extensions/pkg/controller/controlplane/genericactuator/actuator.go +++ b/extensions/pkg/controller/controlplane/genericactuator/actuator.go @@ -419,7 +419,7 @@ func (a *actuator) delete(ctx context.Context, log logr.Logger, cp *extensionsv1 } func (a *actuator) hasShootWebhooks(shoot *gardencorev1beta1.Shoot) bool { - return a.atomicShootWebhookConfig != nil && !v1beta1helper.IsShootAutonomous(shoot.Spec.Provider.Workers) + return a.atomicShootWebhookConfig != nil && !v1beta1helper.IsShootSelfHosted(shoot.Spec.Provider.Workers) } // computeChecksums computes and returns all needed checksums. This includes the checksums for the given deployed secrets, diff --git a/extensions/pkg/controller/worker/controller.go b/extensions/pkg/controller/worker/controller.go index f8b2e126c1b..b4c850562c7 100644 --- a/extensions/pkg/controller/worker/controller.go +++ b/extensions/pkg/controller/worker/controller.go @@ -53,8 +53,8 @@ type AddArgs struct { IgnoreOperationAnnotation bool // ExtensionClass defines the extension class this extension is responsible for. ExtensionClass extensionsv1alpha1.ExtensionClass - // AutonomousShootCluster indicates whether the extension runs in an autonomous shoot cluster. - AutonomousShootCluster bool + // SelfHostedShootCluster indicates whether the extension runs in a self-hosted shoot cluster. + SelfHostedShootCluster bool } // DefaultPredicates returns the default predicates for a Worker reconciler. @@ -81,7 +81,7 @@ func Add(ctx context.Context, mgr manager.Manager, args AddArgs) error { return err } - if mustWatchMachines, err := wantMachineWatch(args.AutonomousShootCluster, mgr.GetRESTMapper()); err != nil { + if mustWatchMachines, err := wantMachineWatch(args.SelfHostedShootCluster, mgr.GetRESTMapper()); err != nil { return fmt.Errorf("failed to determine if machine API exists: %w", err) } else if mustWatchMachines { if err := c.Watch(source.Kind[client.Object]( @@ -109,8 +109,8 @@ func Add(ctx context.Context, mgr manager.Manager, args AddArgs) error { return nil } -func wantMachineWatch(isAutonomousShootCluster bool, restMapper meta.RESTMapper) (bool, error) { - if !isAutonomousShootCluster { +func wantMachineWatch(isSelfHostedShootCluster bool, restMapper meta.RESTMapper) (bool, error) { + if !isSelfHostedShootCluster { return true, nil } return machineAPIPresent(restMapper) diff --git a/hack/ci-e2e-kind-gardenadm.sh b/hack/ci-e2e-kind-gardenadm.sh index ed83fa81bcb..03981316cf3 100755 --- a/hack/ci-e2e-kind-gardenadm.sh +++ b/hack/ci-e2e-kind-gardenadm.sh @@ -21,26 +21,26 @@ trap " ( make kind-single-node-down ) " EXIT -# medium-touch tests cannot run when there is a gardener-operator deployment (i.e., high-touch/connect tests must run -# separately). Hence, let's run the medium-touch tests first, then clean them up, and then run the high-touch/connect +# managed-infra tests cannot run when there is a gardener-operator deployment (i.e., unmanaged-infra/connect tests must run +# separately). Hence, let's run the managed-infra tests first, then clean them up, and then run the unmanaged-infra/connect # tests. -# medium-touch +# managed infrastructure make kind-single-node-up -make gardenadm-up SCENARIO=medium-touch +make gardenadm-up SCENARIO=managed-infra -make test-e2e-local-gardenadm-medium-touch +make test-e2e-local-gardenadm-managed-infra -make gardenadm-down SCENARIO=medium-touch +make gardenadm-down SCENARIO=managed-infra make kind-single-node-down -# high touch +# unmanaged infrastructure make kind-single-node-up -make gardenadm-up SCENARIO=high-touch +make gardenadm-up SCENARIO=unmanaged-infra make gardenadm-up SCENARIO=connect -make test-e2e-local-gardenadm-high-touch +make test-e2e-local-gardenadm-unmanaged-infra make gardenadm-down SCENARIO=connect -make gardenadm-down SCENARIO=high-touch +make gardenadm-down SCENARIO=unmanaged-infra make kind-single-node-down diff --git a/pkg/admissioncontroller/gardenletidentity/shoot/identity.go b/pkg/admissioncontroller/gardenletidentity/shoot/identity.go index b02d8795cac..749c9aee921 100644 --- a/pkg/admissioncontroller/gardenletidentity/shoot/identity.go +++ b/pkg/admissioncontroller/gardenletidentity/shoot/identity.go @@ -18,8 +18,8 @@ import ( ) // FromUserInfoInterface returns the shoot namespace and name, a boolean indicating whether the provided user is an -// autonomous shoot client, and the client's UserType. -func FromUserInfoInterface(u user.Info) (namespace string, name string, isAutonomousShoot bool, userType gardenletidentity.UserType) { +// self-hosted shoot client, and the client's UserType. +func FromUserInfoInterface(u user.Info) (namespace string, name string, isSelfHostedShoot bool, userType gardenletidentity.UserType) { if u == nil { return "", "", false, "" } @@ -37,7 +37,7 @@ func FromUserInfoInterface(u user.Info) (namespace string, name string, isAutono // FromAuthenticationV1UserInfo converts an authenticationv1.UserInfo structure to the user.Info interface and calls // FromUserInfoInterface to return the shoot namespace and name. -func FromAuthenticationV1UserInfo(userInfo authenticationv1.UserInfo) (namespace string, name string, isAutonomousShoot bool, userType gardenletidentity.UserType) { +func FromAuthenticationV1UserInfo(userInfo authenticationv1.UserInfo) (namespace string, name string, isSelfHostedShoot bool, userType gardenletidentity.UserType) { return FromUserInfoInterface(&user.DefaultInfo{ Name: userInfo.Username, UID: userInfo.UID, @@ -48,7 +48,7 @@ func FromAuthenticationV1UserInfo(userInfo authenticationv1.UserInfo) (namespace // FromCertificateSigningRequest converts a *x509.CertificateRequest structure to the user.Info interface and calls // FromUserInfoInterface to return the shoot namespace and name. -func FromCertificateSigningRequest(csr *x509.CertificateRequest) (namespace string, name string, isAutonomousShoot bool, userType gardenletidentity.UserType) { +func FromCertificateSigningRequest(csr *x509.CertificateRequest) (namespace string, name string, isSelfHostedShoot bool, userType gardenletidentity.UserType) { return FromUserInfoInterface(&user.DefaultInfo{ Name: csr.Subject.CommonName, Groups: csr.Subject.Organization, @@ -68,7 +68,7 @@ func convertAuthenticationV1ExtraValueToUserInfoExtra(extra map[string]authentic return ret } -func getIdentityForShootsGroup(u user.Info) (namespace string, name string, isAutonomousShoot bool, userType gardenletidentity.UserType) { +func getIdentityForShootsGroup(u user.Info) (namespace string, name string, isSelfHostedShoot bool, userType gardenletidentity.UserType) { userName := u.GetName() var prefix string @@ -103,8 +103,8 @@ func userTypeFromPrefix(prefix string) gardenletidentity.UserType { return "" } -func getIdentityForServiceAccountsGroup(_ user.Info) (namespace string, name string, isAutonomousShoot bool, userType gardenletidentity.UserType) { - // TODO(rfranzke): Implement this function once the concept of how extensions running in autonomous shoots +func getIdentityForServiceAccountsGroup(_ user.Info) (namespace string, name string, isSelfHostedShoot bool, userType gardenletidentity.UserType) { + // TODO(rfranzke): Implement this function once the concept of how extensions running in self-hosted shoots // authenticate with the garden cluster gets clear. return "", "", false, gardenletidentity.UserTypeExtension } diff --git a/pkg/admissioncontroller/gardenletidentity/shoot/identity_test.go b/pkg/admissioncontroller/gardenletidentity/shoot/identity_test.go index 80138609e17..be23cfcdf4b 100644 --- a/pkg/admissioncontroller/gardenletidentity/shoot/identity_test.go +++ b/pkg/admissioncontroller/gardenletidentity/shoot/identity_test.go @@ -19,12 +19,12 @@ import ( var _ = Describe("identity", func() { DescribeTable("#FromUserInfoInterface", - func(u user.Info, expectedShootNamespace, expectedShootName string, expectedIsAutonomousShootValue bool, expectedUserType gardenletidentity.UserType) { - shootNamespace, shootName, isAutonomousShoot, userType := FromUserInfoInterface(u) + func(u user.Info, expectedShootNamespace, expectedShootName string, expectedIsSelfHostedShootValue bool, expectedUserType gardenletidentity.UserType) { + shootNamespace, shootName, isSelfHostedShoot, userType := FromUserInfoInterface(u) Expect(shootNamespace).To(Equal(expectedShootNamespace)) Expect(shootName).To(Equal(expectedShootName)) - Expect(isAutonomousShoot).To(Equal(expectedIsAutonomousShootValue)) + Expect(isSelfHostedShoot).To(Equal(expectedIsSelfHostedShootValue)) Expect(userType).To(Equal(expectedUserType)) }, @@ -38,12 +38,12 @@ var _ = Describe("identity", func() { ) DescribeTable("#FromAuthenticationV1UserInfo", - func(u authenticationv1.UserInfo, expectedShootNamespace, expectedShootName string, expectedIsAutonomousShootValue bool, expectedUserType gardenletidentity.UserType) { - shootNamespace, shootName, isAutonomousShoot, userType := FromAuthenticationV1UserInfo(u) + func(u authenticationv1.UserInfo, expectedShootNamespace, expectedShootName string, expectedIsSelfHostedShootValue bool, expectedUserType gardenletidentity.UserType) { + shootNamespace, shootName, isSelfHostedShoot, userType := FromAuthenticationV1UserInfo(u) Expect(shootNamespace).To(Equal(expectedShootNamespace)) Expect(shootName).To(Equal(expectedShootName)) - Expect(isAutonomousShoot).To(Equal(expectedIsAutonomousShootValue)) + Expect(isSelfHostedShoot).To(Equal(expectedIsSelfHostedShootValue)) Expect(userType).To(Equal(expectedUserType)) }, @@ -56,12 +56,12 @@ var _ = Describe("identity", func() { ) DescribeTable("#FromCertificateSigningRequest", - func(csr *x509.CertificateRequest, expectedShootNamespace, expectedShootName string, expectedIsAutonomousShootValue bool, expectedUserType gardenletidentity.UserType) { - shootNamespace, shootName, isAutonomousShoot, userType := FromCertificateSigningRequest(csr) + func(csr *x509.CertificateRequest, expectedShootNamespace, expectedShootName string, expectedIsSelfHostedShootValue bool, expectedUserType gardenletidentity.UserType) { + shootNamespace, shootName, isSelfHostedShoot, userType := FromCertificateSigningRequest(csr) Expect(shootNamespace).To(Equal(expectedShootNamespace)) Expect(shootName).To(Equal(expectedShootName)) - Expect(isAutonomousShoot).To(Equal(expectedIsAutonomousShootValue)) + Expect(isSelfHostedShoot).To(Equal(expectedIsSelfHostedShootValue)) Expect(userType).To(Equal(expectedUserType)) }, diff --git a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go index 2807ebbd102..a51d28963d9 100644 --- a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go +++ b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go @@ -47,8 +47,8 @@ type Handler struct { // Handle restricts requests made by gardenlets. func (h *Handler) Handle(_ context.Context, request admission.Request) admission.Response { - shootNamespace, shootName, isAutonomousShoot, userType := shootidentity.FromAuthenticationV1UserInfo(request.UserInfo) - if !isAutonomousShoot { + shootNamespace, shootName, isSelfHostedShoot, userType := shootidentity.FromAuthenticationV1UserInfo(request.UserInfo) + if !isSelfHostedShoot { return admissionwebhook.Allowed("") } @@ -113,11 +113,11 @@ func (h *Handler) admitGardenlet(gardenletShootInfo types.NamespacedName, reques return admission.Errored(http.StatusBadRequest, fmt.Errorf("unexpected operation: %q", request.Operation)) } - if !strings.HasPrefix(request.Name, gardenletutils.ResourcePrefixAutonomousShoot) { - return admission.Errored(http.StatusBadRequest, fmt.Errorf("the Gardenlet resources for autonomous shoots must be prefixed with %q", gardenletutils.ResourcePrefixAutonomousShoot)) + if !strings.HasPrefix(request.Name, gardenletutils.ResourcePrefixSelfHostedShoot) { + return admission.Errored(http.StatusBadRequest, fmt.Errorf("the Gardenlet resources for self-hosted shoots must be prefixed with %q", gardenletutils.ResourcePrefixSelfHostedShoot)) } - return h.admit(gardenletShootInfo, types.NamespacedName{Name: strings.TrimPrefix(request.Name, gardenletutils.ResourcePrefixAutonomousShoot), Namespace: request.Namespace}) + return h.admit(gardenletShootInfo, types.NamespacedName{Name: strings.TrimPrefix(request.Name, gardenletutils.ResourcePrefixSelfHostedShoot), Namespace: request.Namespace}) } func (h *Handler) admit(gardenletShootInfo, objectShootInfo types.NamespacedName) admission.Response { diff --git a/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go b/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go index e9a915b6876..feb9b1eddc7 100644 --- a/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go +++ b/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go @@ -31,7 +31,7 @@ import ( authorizerwebhook "github.com/gardener/gardener/pkg/webhook/authorizer" ) -// NewAuthorizer returns a new authorizer for requests from gardenlets running in autonomous shoots. +// NewAuthorizer returns a new authorizer for requests from gardenlets running in self-hosted shoots. func NewAuthorizer(logger logr.Logger, c client.Client, graph graph.Interface, authorizeWithSelectors authorizerwebhook.WithSelectorsChecker) *authorizer { return &authorizer{ logger: logger, @@ -69,8 +69,8 @@ var ( ) func (a *authorizer) Authorize(ctx context.Context, attrs auth.Attributes) (auth.Decision, string, error) { - shootNamespace, shootName, isAutonomousShoot, userType := shootidentity.FromUserInfoInterface(attrs.GetUser()) - if !isAutonomousShoot { + shootNamespace, shootName, isSelfHostedShoot, userType := shootidentity.FromUserInfoInterface(attrs.GetUser()) + if !isSelfHostedShoot { return auth.DecisionNoOpinion, "", nil } @@ -116,7 +116,7 @@ func (a *authorizer) Authorize(ctx context.Context, attrs auth.Attributes) (auth authwebhook.WithAlwaysAllowedVerbs("create"), authwebhook.WithAllowedSubresources("status"), authwebhook.WithAllowedNamespaces(requestAuthorizer.ToNamespace), - authwebhook.WithFieldSelectors(map[string]string{metav1.ObjectNameField: gardenletutils.ResourcePrefixAutonomousShoot + requestAuthorizer.ToName}), + authwebhook.WithFieldSelectors(map[string]string{metav1.ObjectNameField: gardenletutils.ResourcePrefixSelfHostedShoot + requestAuthorizer.ToName}), ) case secretResource: diff --git a/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go b/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go index 8a68d76efdd..8e912903d01 100644 --- a/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go +++ b/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go @@ -485,7 +485,7 @@ var _ = Describe("Shoot", func() { attrs.Verb = verb if withSelector { - selector, err := fields.ParseSelector("metadata.name=autonomous-shoot-" + shootName) + selector, err := fields.ParseSelector("metadata.name=self-hosted-shoot-" + shootName) Expect(err).NotTo(HaveOccurred()) attrs.FieldSelectorRequirements = selector.Requirements() } @@ -569,7 +569,7 @@ var _ = Describe("Shoot", func() { }) It("should allow if shoot meta matches", func() { - bootstrapTokenSecret.Data = map[string][]byte{"description": []byte(fmt.Sprintf("Used for connecting the autonomous Shoot %s/%s to the Garden cluster", shootNamespace, shootName))} + bootstrapTokenSecret.Data = map[string][]byte{"description": []byte(fmt.Sprintf("Used for connecting the self-hosted Shoot %s/%s to the Garden cluster", shootNamespace, shootName))} Expect(fakeClient.Create(ctx, bootstrapTokenSecret)).To(Succeed()) decision, reason, err := authorizer.Authorize(ctx, attrs) @@ -579,7 +579,7 @@ var _ = Describe("Shoot", func() { }) It("should not have an opinion if shoot meta does not match", func() { - bootstrapTokenSecret.Data = map[string][]byte{"description": []byte("Used for connecting the autonomous Shoot not-the-namespace/not-the-name to the Garden cluster")} + bootstrapTokenSecret.Data = map[string][]byte{"description": []byte("Used for connecting the self-hosted Shoot not-the-namespace/not-the-name to the Garden cluster")} Expect(fakeClient.Create(ctx, bootstrapTokenSecret)).To(Succeed()) decision, reason, err := authorizer.Authorize(ctx, attrs) @@ -592,7 +592,7 @@ var _ = Describe("Shoot", func() { Expect(fakeClient.Create(ctx, bootstrapTokenSecret)).To(Succeed()) decision, reason, err := authorizer.Authorize(ctx, attrs) - Expect(err).To(MatchError(ContainSubstring(`failed fetching shoot meta from bootstrap token description: bootstrap token description does not start with "Used for connecting the autonomous Shoot "`))) + Expect(err).To(MatchError(ContainSubstring(`failed fetching shoot meta from bootstrap token description: bootstrap token description does not start with "Used for connecting the self-hosted Shoot "`))) Expect(decision).To(Equal(auth.DecisionNoOpinion)) Expect(reason).To(BeEmpty()) }) @@ -943,7 +943,7 @@ var _ = Describe("Shoot", func() { }, Entry("create", "create"), - Entry("mark-autonomous", "mark-autonomous"), + Entry("mark-self-hosted", "mark-self-hosted"), ) It("should have no opinion because verb is not allowed", func() { diff --git a/pkg/admissioncontroller/webhook/auth/shoot/gardenadm.go b/pkg/admissioncontroller/webhook/auth/shoot/gardenadm.go index 02a97b23010..0dd0f9827fc 100644 --- a/pkg/admissioncontroller/webhook/auth/shoot/gardenadm.go +++ b/pkg/admissioncontroller/webhook/auth/shoot/gardenadm.go @@ -32,7 +32,7 @@ func (a *authorizer) authorizeGardenadmRequests(requestLog logr.Logger, shootNam } case shootResource: - if isGardenadmRequestAllowed(attrs, &shootNamespace, "create", "mark-autonomous") { + if isGardenadmRequestAllowed(attrs, &shootNamespace, "create", "mark-self-hosted") { return auth.DecisionAllow, "", nil } diff --git a/pkg/apis/core/helper/shoot.go b/pkg/apis/core/helper/shoot.go index f3535ff67ac..f93e7fdf048 100644 --- a/pkg/apis/core/helper/shoot.go +++ b/pkg/apis/core/helper/shoot.go @@ -310,14 +310,14 @@ func IsKubeProxyIPVSMode(kubeProxyConfig *core.KubeProxyConfig) bool { kubeProxyConfig.Mode != nil && *kubeProxyConfig.Mode == core.ProxyModeIPVS } -// IsShootAutonomous returns true if the shoot has a worker pool dedicated for running the control plane components. -func IsShootAutonomous(workers []core.Worker) bool { +// IsShootSelfHosted returns true if the shoot has a worker pool dedicated for running the control plane components. +func IsShootSelfHosted(workers []core.Worker) bool { return slices.ContainsFunc(workers, func(worker core.Worker) bool { return worker.ControlPlane != nil }) } -// ControlPlaneWorkerPoolForShoot returns the worker pool running the control plane in case the shoot is autonomous. +// ControlPlaneWorkerPoolForShoot returns the worker pool running the control plane in case the shoot is self-hosted. func ControlPlaneWorkerPoolForShoot(workers []core.Worker) *core.Worker { idx := slices.IndexFunc(workers, func(worker core.Worker) bool { return worker.ControlPlane != nil diff --git a/pkg/apis/core/helper/shoot_test.go b/pkg/apis/core/helper/shoot_test.go index bd2515c4dd9..be75fb72446 100644 --- a/pkg/apis/core/helper/shoot_test.go +++ b/pkg/apis/core/helper/shoot_test.go @@ -631,12 +631,12 @@ var _ = Describe("Helper", func() { Entry("with KubeProxy in IPTables mode", &core.KubeProxyConfig{Enabled: ptr.To(true), Mode: ptr.To(core.ProxyModeIPTables)}, false), ) - Describe("#IsShootAutonomous", func() { + Describe("#IsShootSelfHosted", func() { It("should return true (single worker pool with control plane configuration)", func() { shoot := &core.Shoot{Spec: core.ShootSpec{Provider: core.Provider{Workers: []core.Worker{ {ControlPlane: &core.WorkerControlPlane{}}, }}}} - Expect(IsShootAutonomous(shoot.Spec.Provider.Workers)).To(BeTrue()) + Expect(IsShootSelfHosted(shoot.Spec.Provider.Workers)).To(BeTrue()) }) It("should return true (multiple worker pools, one with control plane configuration)", func() { @@ -645,12 +645,12 @@ var _ = Describe("Helper", func() { {ControlPlane: &core.WorkerControlPlane{}}, {}, }}}} - Expect(IsShootAutonomous(shoot.Spec.Provider.Workers)).To(BeTrue()) + Expect(IsShootSelfHosted(shoot.Spec.Provider.Workers)).To(BeTrue()) }) It("should return false (no worker pools)", func() { shoot := &core.Shoot{} - Expect(IsShootAutonomous(shoot.Spec.Provider.Workers)).To(BeFalse()) + Expect(IsShootSelfHosted(shoot.Spec.Provider.Workers)).To(BeFalse()) }) It("should return false (worker pools, but none with control plane configuration)", func() { @@ -659,7 +659,7 @@ var _ = Describe("Helper", func() { {}, {}, }}}} - Expect(IsShootAutonomous(shoot.Spec.Provider.Workers)).To(BeFalse()) + Expect(IsShootSelfHosted(shoot.Spec.Provider.Workers)).To(BeFalse()) }) }) diff --git a/pkg/apis/core/types_shoot.go b/pkg/apis/core/types_shoot.go index 0d209e07abf..308284982fd 100644 --- a/pkg/apis/core/types_shoot.go +++ b/pkg/apis/core/types_shoot.go @@ -1319,7 +1319,7 @@ type Worker struct { // UpdateStrategy specifies the machine update strategy for the worker pool. UpdateStrategy *MachineUpdateStrategy // ControlPlane specifies that the shoot cluster control plane components should be running in this worker pool. - // This is only relevant for autonomous shoot clusters. + // This is only relevant for self-hosted shoot clusters. ControlPlane *WorkerControlPlane } diff --git a/pkg/apis/core/v1beta1/constants/types_constants.go b/pkg/apis/core/v1beta1/constants/types_constants.go index b6423835267..4cc93d54be7 100644 --- a/pkg/apis/core/v1beta1/constants/types_constants.go +++ b/pkg/apis/core/v1beta1/constants/types_constants.go @@ -487,8 +487,8 @@ const ( // LabelKeyAggregateToProjectMember is a constant for a label on ClusterRoles that are aggregated to the project // member ClusterRole. LabelKeyAggregateToProjectMember = "rbac.gardener.cloud/aggregate-to-project-member" - // LabelAutonomousShootCluster is a constant for a label on a Seed indicating that it is an autonomous shoot cluster. - LabelAutonomousShootCluster = "seed.gardener.cloud/autonomous-shoot-cluster" + // LabelSelfHostedShootCluster is a constant for a label on a Seed indicating that it is a self-hosted shoot cluster. + LabelSelfHostedShootCluster = "seed.gardener.cloud/self-hosted-shoot-cluster" // LabelSecretBindingReference is used to identify secrets which are referred by a SecretBinding (not necessarily in the same namespace). LabelSecretBindingReference = "reference.gardener.cloud/secretbinding" // LabelCredentialsBindingReference is used to identify credentials which are referred by a CredentialsBinding (not necessarily in the same namespace). @@ -813,10 +813,10 @@ const ( SeedsGroup = "gardener.cloud:system:seeds" // SeedUserNamePrefix is the identity username prefix for gardenlets when authenticating to the API server. SeedUserNamePrefix = "gardener.cloud:system:seed:" - // ShootsGroup is the identity group for gardenlets running in autonomous shoot clusters when authenticating to the + // ShootsGroup is the identity group for gardenlets running in self-hosted shoot clusters when authenticating to the // API server. ShootsGroup = "gardener.cloud:system:shoots" - // ShootUserNamePrefix is the identity username prefix for gardenlets running in autonomous shoot clusters when + // ShootUserNamePrefix is the identity username prefix for gardenlets running in self-hosted shoot clusters when // authenticating to the API server. ShootUserNamePrefix = "gardener.cloud:system:shoot:" // GardenadmUserNamePrefix is the identity username prefix for `gardenadm connect` when it bootstraps the diff --git a/pkg/apis/core/v1beta1/generated.proto b/pkg/apis/core/v1beta1/generated.proto index 0aa02682d38..404abf251ab 100644 --- a/pkg/apis/core/v1beta1/generated.proto +++ b/pkg/apis/core/v1beta1/generated.proto @@ -3796,7 +3796,7 @@ message Worker { optional string updateStrategy = 23; // ControlPlane specifies that the shoot cluster control plane components should be running in this worker pool. - // This is only relevant for autonomous shoot clusters. + // This is only relevant for self-hosted shoot clusters. // +optional optional WorkerControlPlane controlPlane = 24; } diff --git a/pkg/apis/core/v1beta1/helper/shoot.go b/pkg/apis/core/v1beta1/helper/shoot.go index e5db7a80ee6..86e7f047ba8 100644 --- a/pkg/apis/core/v1beta1/helper/shoot.go +++ b/pkg/apis/core/v1beta1/helper/shoot.go @@ -636,20 +636,19 @@ func LastInitiationTimeForWorkerPool(name string, pendingWorkersRollout []garden return globalLastInitiationTime } -// IsShootAutonomous returns true if the shoot has a worker pool dedicated for running the control plane components. -func IsShootAutonomous(workers []gardencorev1beta1.Worker) bool { +// IsShootSelfHosted returns true if the shoot has a worker pool dedicated for running the control plane components. +func IsShootSelfHosted(workers []gardencorev1beta1.Worker) bool { return slices.ContainsFunc(workers, func(worker gardencorev1beta1.Worker) bool { return worker.ControlPlane != nil }) } // HasManagedInfrastructure returns true if the shoot's infrastructure (network, machines, etc.) is managed by Gardener. -// I.e., it returns false for high-touch autonomous shoots, where the infrastructure is managed by the user. func HasManagedInfrastructure(shoot *gardencorev1beta1.Shoot) bool { return shoot.Spec.CredentialsBindingName != nil || shoot.Spec.SecretBindingName != nil } -// ControlPlaneWorkerPoolForShoot returns the worker pool running the control plane in case the shoot is autonomous. +// ControlPlaneWorkerPoolForShoot returns the worker pool running the control plane in case the shoot is self-hosted. func ControlPlaneWorkerPoolForShoot(workers []gardencorev1beta1.Worker) *gardencorev1beta1.Worker { idx := slices.IndexFunc(workers, func(worker gardencorev1beta1.Worker) bool { return worker.ControlPlane != nil @@ -661,10 +660,10 @@ func ControlPlaneWorkerPoolForShoot(workers []gardencorev1beta1.Worker) *gardenc return &workers[idx] } -// ControlPlaneNamespaceForShoot returns the control plane namespace for the shoot. If it is an autonomous shoot, +// ControlPlaneNamespaceForShoot returns the control plane namespace for the shoot. If it is a self-hosted shoot, // kube-system is returned. Otherwise, it is the technical ID of the shoot. func ControlPlaneNamespaceForShoot(shoot *gardencorev1beta1.Shoot) string { - if IsShootAutonomous(shoot.Spec.Provider.Workers) { + if IsShootSelfHosted(shoot.Spec.Provider.Workers) { return metav1.NamespaceSystem } return shoot.Status.TechnicalID @@ -696,9 +695,9 @@ func IsShootIstioTLSTerminationEnabled(shoot *gardencorev1beta1.Shoot) bool { } // GetBackupConfigForShoot returns the backup config from the Seed resource in case the shoot is a regular shoot. -// For autonomous shoots, it is returned from the Shoot resource. +// For self-hosted shoots, it is returned from the Shoot resource. func GetBackupConfigForShoot(shoot *gardencorev1beta1.Shoot, seed *gardencorev1beta1.Seed) *gardencorev1beta1.Backup { - if !IsShootAutonomous(shoot.Spec.Provider.Workers) { + if !IsShootSelfHosted(shoot.Spec.Provider.Workers) { if seed == nil { return nil } diff --git a/pkg/apis/core/v1beta1/helper/shoot_test.go b/pkg/apis/core/v1beta1/helper/shoot_test.go index f8e62009db0..515fd0eeaeb 100644 --- a/pkg/apis/core/v1beta1/helper/shoot_test.go +++ b/pkg/apis/core/v1beta1/helper/shoot_test.go @@ -1428,12 +1428,12 @@ var _ = Describe("Helper", func() { }) }) - Describe("#IsShootAutonomous", func() { + Describe("#IsShootSelfHosted", func() { It("should return true (single worker pool with control plane configuration)", func() { shoot := &gardencorev1beta1.Shoot{Spec: gardencorev1beta1.ShootSpec{Provider: gardencorev1beta1.Provider{Workers: []gardencorev1beta1.Worker{ {ControlPlane: &gardencorev1beta1.WorkerControlPlane{}}, }}}} - Expect(IsShootAutonomous(shoot.Spec.Provider.Workers)).To(BeTrue()) + Expect(IsShootSelfHosted(shoot.Spec.Provider.Workers)).To(BeTrue()) }) It("should return true (multiple worker pools, one with control plane configuration)", func() { @@ -1442,12 +1442,12 @@ var _ = Describe("Helper", func() { {ControlPlane: &gardencorev1beta1.WorkerControlPlane{}}, {}, }}}} - Expect(IsShootAutonomous(shoot.Spec.Provider.Workers)).To(BeTrue()) + Expect(IsShootSelfHosted(shoot.Spec.Provider.Workers)).To(BeTrue()) }) It("should return false (no worker pools)", func() { shoot := &gardencorev1beta1.Shoot{} - Expect(IsShootAutonomous(shoot.Spec.Provider.Workers)).To(BeFalse()) + Expect(IsShootSelfHosted(shoot.Spec.Provider.Workers)).To(BeFalse()) }) It("should return false (worker pools, but none with control plane configuration)", func() { @@ -1456,7 +1456,7 @@ var _ = Describe("Helper", func() { {}, {}, }}}} - Expect(IsShootAutonomous(shoot.Spec.Provider.Workers)).To(BeFalse()) + Expect(IsShootSelfHosted(shoot.Spec.Provider.Workers)).To(BeFalse()) }) }) @@ -1499,7 +1499,7 @@ var _ = Describe("Helper", func() { }) Describe("#ControlPlaneNamespaceForShoot", func() { - It("should return kube-system for autonomous shoots", func() { + It("should return kube-system for self-hosted shoots", func() { shoot := &gardencorev1beta1.Shoot{ Spec: gardencorev1beta1.ShootSpec{Provider: gardencorev1beta1.Provider{Workers: []gardencorev1beta1.Worker{{ControlPlane: &gardencorev1beta1.WorkerControlPlane{}}}}}, Status: gardencorev1beta1.ShootStatus{TechnicalID: "shoot--foo--bar"}, @@ -1567,11 +1567,11 @@ var _ = Describe("Helper", func() { shoot = &gardencorev1beta1.Shoot{} }) - It("should return the seed backup config because shoot is not autonomous", func() { + It("should return the seed backup config because shoot is not self-hosted", func() { Expect(GetBackupConfigForShoot(shoot, seed)).To(Equal(seedBackup)) }) - It("should return the shoot backup config because shoot is autonomous", func() { + It("should return the shoot backup config because shoot is self-hosted", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, gardencorev1beta1.Worker{ ControlPlane: &gardencorev1beta1.WorkerControlPlane{Backup: shootBackup}, }) diff --git a/pkg/apis/core/v1beta1/types_shoot.go b/pkg/apis/core/v1beta1/types_shoot.go index 011c62a1aa4..ae546316215 100644 --- a/pkg/apis/core/v1beta1/types_shoot.go +++ b/pkg/apis/core/v1beta1/types_shoot.go @@ -1725,7 +1725,7 @@ type Worker struct { // +optional UpdateStrategy *MachineUpdateStrategy `json:"updateStrategy,omitempty" protobuf:"bytes,23,opt,name=updateStrategy,casttype=MachineUpdateStrategy"` // ControlPlane specifies that the shoot cluster control plane components should be running in this worker pool. - // This is only relevant for autonomous shoot clusters. + // This is only relevant for self-hosted shoot clusters. // +optional ControlPlane *WorkerControlPlane `json:"controlPlane,omitempty" protobuf:"bytes,24,opt,name=controlPlane"` } diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index ac7c42cc739..80be1449b40 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -268,7 +268,7 @@ func ValidateShootSpec(meta metav1.ObjectMeta, spec *core.ShootSpec, fldPath *fi if spec.CredentialsBindingName != nil { allErrs = append(allErrs, field.Forbidden(fldPath.Child("credentialsBindingName"), workerlessErrorMsg)) } - } else if !helper.IsShootAutonomous(spec.Provider.Workers) { + } else if !helper.IsShootSelfHosted(spec.Provider.Workers) { if len(ptr.Deref(spec.SecretBindingName, "")) == 0 && len(ptr.Deref(spec.CredentialsBindingName, "")) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("secretBindingName"), "must be set when credentialsBindingName is not")) } else if len(ptr.Deref(spec.SecretBindingName, "")) != 0 && len(ptr.Deref(spec.CredentialsBindingName, "")) != 0 { @@ -308,8 +308,8 @@ func ValidateShootSpec(meta metav1.ObjectMeta, spec *core.ShootSpec, fldPath *fi } } - if helper.IsShootAutonomous(spec.Provider.Workers) && spec.SeedName != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Child("seedName"), *spec.SeedName, "cannot set seedName for autonomous shoots")) + if helper.IsShootSelfHosted(spec.Provider.Workers) && spec.SeedName != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("seedName"), *spec.SeedName, "cannot set seedName for self-hosted shoots")) } if spec.SchedulerName != nil { @@ -479,9 +479,9 @@ func ValidateProviderUpdate(newProvider, oldProvider *core.Provider, fldPath *fi } } - if helper.IsShootAutonomous(oldProvider.Workers) != helper.IsShootAutonomous(newProvider.Workers) { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("workers"), "cannot switch a Shoot between autonomous and non-autonomous mode")) - } else if helper.IsShootAutonomous(newProvider.Workers) { + if helper.IsShootSelfHosted(oldProvider.Workers) != helper.IsShootSelfHosted(newProvider.Workers) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("workers"), "cannot switch a Shoot between self-hosted and hosted mode")) + } else if helper.IsShootSelfHosted(newProvider.Workers) { oldControlPlaneWorkerPool, newControlPlaneWorkerPool := helper.ControlPlaneWorkerPoolForShoot(oldProvider.Workers), helper.ControlPlaneWorkerPoolForShoot(newProvider.Workers) if oldControlPlaneWorkerPool.Name != newControlPlaneWorkerPool.Name { allErrs = append(allErrs, field.Forbidden(fldPath.Child("workers"), "cannot change control plane worker pool")) @@ -2122,7 +2122,7 @@ func ValidateWorker(worker core.Worker, kubernetes core.Kubernetes, shootNamespa if worker.ControlPlane != nil { if worker.Minimum != worker.Maximum || worker.Minimum != 1 { - allErrs = append(allErrs, field.Invalid(fldPath.Child("minimum"), worker.Minimum, "autonomous shoots only support minimum=maximum=1 for the control plane worker pool (might change in the future)")) + allErrs = append(allErrs, field.Invalid(fldPath.Child("minimum"), worker.Minimum, "self-hosted shoots only support minimum=maximum=1 for the control plane worker pool (might change in the future)")) } if backup := worker.ControlPlane.Backup; backup != nil { diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index db3a2b72527..55742d0b816 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -1584,7 +1584,7 @@ var _ = Describe("Shoot Validation Tests", func() { }) }) - Context("autonomous shoots", func() { + Context("self-hosted shoots", func() { It("should allow 'ControlPlane' field for worker pool", func() { shoot.Spec.Provider.Workers[0].ControlPlane = &core.WorkerControlPlane{} @@ -1609,7 +1609,7 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.seedName"), - "Detail": ContainSubstring("cannot set seedName for autonomous shoots"), + "Detail": ContainSubstring("cannot set seedName for self-hosted shoots"), })))) }) @@ -1621,18 +1621,18 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.provider.workers[0].minimum"), - "Detail": ContainSubstring("autonomous shoots only support minimum=maximum=1 for the control plane worker pool (might change in the future)"), + "Detail": ContainSubstring("self-hosted shoots only support minimum=maximum=1 for the control plane worker pool (might change in the future)"), })))) }) - It("should prevent marking a shoot as 'autonomous' after creation", func() { + It("should prevent marking a shoot as 'self-hosted' after creation", func() { newShoot := prepareShootForUpdate(shoot) shoot.Spec.Provider.Workers[0].ControlPlane = &core.WorkerControlPlane{} Expect(ValidateShootUpdate(newShoot, shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.provider.workers"), - "Detail": ContainSubstring("cannot switch a Shoot between autonomous and non-autonomous mode"), + "Detail": ContainSubstring("cannot switch a Shoot between self-hosted and hosted mode"), })))) }) @@ -1644,7 +1644,7 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShootUpdate(newShoot, shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.provider.workers"), - "Detail": ContainSubstring("cannot switch a Shoot between autonomous and non-autonomous mode"), + "Detail": ContainSubstring("cannot switch a Shoot between self-hosted and hosted mode"), })))) }) diff --git a/pkg/apiserver/openapi/openapi_generated.go b/pkg/apiserver/openapi/openapi_generated.go index a8e2442480a..05d2c502b8e 100644 --- a/pkg/apiserver/openapi/openapi_generated.go +++ b/pkg/apiserver/openapi/openapi_generated.go @@ -10636,7 +10636,7 @@ func schema_pkg_apis_core_v1beta1_Worker(ref common.ReferenceCallback) common.Op }, "controlPlane": { SchemaProps: spec.SchemaProps{ - Description: "ControlPlane specifies that the shoot cluster control plane components should be running in this worker pool. This is only relevant for autonomous shoot clusters.", + Description: "ControlPlane specifies that the shoot cluster control plane components should be running in this worker pool. This is only relevant for self-hosted shoot clusters.", Ref: ref("github.com/gardener/gardener/pkg/apis/core/v1beta1.WorkerControlPlane"), }, }, diff --git a/pkg/component/extensions/operatingsystemconfig/operatingsystemconfig.go b/pkg/component/extensions/operatingsystemconfig/operatingsystemconfig.go index e26577fc224..c598c5573c1 100644 --- a/pkg/component/extensions/operatingsystemconfig/operatingsystemconfig.go +++ b/pkg/component/extensions/operatingsystemconfig/operatingsystemconfig.go @@ -1075,7 +1075,7 @@ func KeyV2( ) if worker.Machine.Image != nil { - // worker.Machine.Image.Version is unset for autonomous shoots with unmanaged infrastructure + // worker.Machine.Image.Version is unset for self-hosted shoots with unmanaged infrastructure data = append(data, worker.Machine.Image.Name+ptr.Deref(worker.Machine.Image.Version, "")) } diff --git a/pkg/component/garden/system/virtual/virtual.go b/pkg/component/garden/system/virtual/virtual.go index ebceb164345..24c0af956a2 100644 --- a/pkg/component/garden/system/virtual/virtual.go +++ b/pkg/component/garden/system/virtual/virtual.go @@ -148,7 +148,7 @@ func (g *gardenSystem) computeResourcesData() (map[string][]byte, error) { { APIGroups: []string{gardencorev1beta1.GroupName}, Resources: []string{"*"}, - Verbs: []string{"create", "delete", "deletecollection", "get", "list", "watch", "patch", "update", "manage-members", "modify-spec-tolerations-whitelist", "modify-spec-kubernetes", "modify-spec-machineimages", "modify-spec-providerconfig", "mark-autonomous"}, + Verbs: []string{"create", "delete", "deletecollection", "get", "list", "watch", "patch", "update", "manage-members", "modify-spec-tolerations-whitelist", "modify-spec-kubernetes", "modify-spec-machineimages", "modify-spec-providerconfig", "mark-self-hosted"}, }, { APIGroups: []string{ diff --git a/pkg/component/garden/system/virtual/virtual_test.go b/pkg/component/garden/system/virtual/virtual_test.go index 53b9773b840..dce52016a4e 100644 --- a/pkg/component/garden/system/virtual/virtual_test.go +++ b/pkg/component/garden/system/virtual/virtual_test.go @@ -170,7 +170,7 @@ var _ = Describe("Virtual", func() { { APIGroups: []string{"core.gardener.cloud"}, Resources: []string{"*"}, - Verbs: []string{"create", "delete", "deletecollection", "get", "list", "watch", "patch", "update", "manage-members", "modify-spec-tolerations-whitelist", "modify-spec-kubernetes", "modify-spec-machineimages", "modify-spec-providerconfig", "mark-autonomous"}, + Verbs: []string{"create", "delete", "deletecollection", "get", "list", "watch", "patch", "update", "manage-members", "modify-spec-tolerations-whitelist", "modify-spec-kubernetes", "modify-spec-machineimages", "modify-spec-providerconfig", "mark-self-hosted"}, }, { APIGroups: []string{ diff --git a/pkg/component/gardener/resourcemanager/resource_manager.go b/pkg/component/gardener/resourcemanager/resource_manager.go index 42a585e4b20..b288b84b835 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager.go +++ b/pkg/component/gardener/resourcemanager/resource_manager.go @@ -360,7 +360,7 @@ const ( // the virtual garden cluster, or GRM in a seed cluster, being responsible for a shoot cluster). ForTarget ResponsibilityMode = "target" // ForSourceAndTarget is a deployment mode for a gardener-resource-manager deployed in a source cluster, - // taking over responsibilities for both the source and a target cluster (e.g., GRM in an autonomous shoot cluster + // taking over responsibilities for both the source and a target cluster (e.g., GRM in a self-hosted shoot cluster // where the control plane is running in the cluster itself). ForSourceAndTarget ResponsibilityMode = "both" ) diff --git a/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager.go b/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager.go index 002cfaa6268..eb694fc445e 100644 --- a/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager.go +++ b/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager.go @@ -89,15 +89,15 @@ type Values struct { Image string // Replicas is the number of replicas for the deployment. Replicas int32 - // AutonomousShoot is true if the machine-controller-manager is deployed for an autonomous shoot cluster. - AutonomousShoot bool + // SelfHostedShoot is true if the machine-controller-manager is deployed for a self-hosted shoot cluster. + SelfHostedShoot bool } func (m *machineControllerManager) Deploy(ctx context.Context) error { var ( // In `gardenadm bootstrap`, machine-controller-manager runs without a target cluster. We don't need the shoot // resources (e.g., RBAC) in this case. - hasTargetCluster = !m.values.AutonomousShoot || m.namespace == metav1.NamespaceSystem + hasTargetCluster = !m.values.SelfHostedShoot || m.namespace == metav1.NamespaceSystem shootAccessSecret = m.newShootAccessSecret() serviceAccount = m.emptyServiceAccount() @@ -207,7 +207,7 @@ func (m *machineControllerManager) Deploy(ctx context.Context) error { return err } - if !m.values.AutonomousShoot { + if !m.values.SelfHostedShoot { if err := shootAccessSecret.Reconcile(ctx, m.client); err != nil { return err } @@ -247,7 +247,7 @@ func (m *machineControllerManager) Deploy(ctx context.Context) error { fmt.Sprintf("--port=%d", portMetrics), "--safety-up=2", "--safety-down=1", - "--target-kubeconfig=" + targetKubeconfig(m.values.AutonomousShoot, m.namespace), + "--target-kubeconfig=" + targetKubeconfig(m.values.SelfHostedShoot, m.namespace), "--concurrent-syncs=30", "--kube-api-qps=150", "--kube-api-burst=200", @@ -289,7 +289,7 @@ func (m *machineControllerManager) Deploy(ctx context.Context) error { }, } - if !m.values.AutonomousShoot { + if !m.values.SelfHostedShoot { genericTokenKubeconfigSecret, found := m.secretsManager.Get(v1beta1constants.SecretNameGenericTokenKubeconfig) if !found { return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameGenericTokenKubeconfig) @@ -484,8 +484,8 @@ func (m *machineControllerManager) Deploy(ctx context.Context) error { } // targetKubeconfig returns the path to the target kubeconfig file depending on the shoot configuration. -func targetKubeconfig(autonomousShoot bool, controlPlaneNamespace string) string { - if !autonomousShoot { +func targetKubeconfig(selfHostedShoot bool, controlPlaneNamespace string) string { + if !selfHostedShoot { return gardenerutils.PathGenericKubeconfig } @@ -494,7 +494,7 @@ func targetKubeconfig(autonomousShoot bool, controlPlaneNamespace string) string return "" } - // There is no control plane for the autonomous shoot cluster yet, i.e., we're creating machines for the control plane + // There is no control plane for the self-hosted shoot cluster yet, i.e., we're creating machines for the control plane // nodes with `gardenadm bootstrap`. machine-controller-manager should not interact with a target cluster. return "none" } diff --git a/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager_test.go b/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager_test.go index 54eb7789f81..c7e160a674b 100644 --- a/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager_test.go +++ b/pkg/component/nodemanagement/machinecontrollermanager/machine_controller_manager_test.go @@ -597,7 +597,7 @@ subjects: Expect(actualServiceMonitor).To(DeepEqual(serviceMonitor)) actualManagedResource := &resourcesv1alpha1.ManagedResource{} - if values.AutonomousShoot && namespace != "kube-system" { + if values.SelfHostedShoot && namespace != "kube-system" { Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResource), actualManagedResource)).To(BeNotFoundError()) } else { Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResource), actualManagedResource)).To(Succeed()) @@ -624,7 +624,7 @@ subjects: } } - When("the shoot is not autonomous", func() { + When("the shoot is not self-hosted", func() { JustBeforeEach(func() { Expect(gardenerutils.InjectGenericKubeconfig(deployment, "generic-token-kubeconfig", shootAccessSecret.Name)).To(Succeed()) }) @@ -639,9 +639,9 @@ subjects: }) }) - When("the shoot is autonomous", func() { + When("the shoot is self-hosted", func() { BeforeEach(func() { - values.AutonomousShoot = true + values.SelfHostedShoot = true }) JustBeforeEach(func() { diff --git a/pkg/component/nodemanagement/machinecontrollermanager/provider.go b/pkg/component/nodemanagement/machinecontrollermanager/provider.go index d9648019580..0d6390f3494 100644 --- a/pkg/component/nodemanagement/machinecontrollermanager/provider.go +++ b/pkg/component/nodemanagement/machinecontrollermanager/provider.go @@ -28,7 +28,7 @@ const ( // implementations when the standard sidecar container is required. // The shoot object can be read from the `Cluster` object, e.g., using the GardenContext.GetCluster method in webhooks. func ProviderSidecarContainer(shoot *gardencorev1beta1.Shoot, controlPlaneNamespace, providerName, image string) corev1.Container { - autonomousShoot := v1beta1helper.IsShootAutonomous(shoot.Spec.Provider.Workers) + selfHostedShoot := v1beta1helper.IsShootSelfHosted(shoot.Spec.Provider.Workers) c := corev1.Container{ Name: providerSidecarContainerName(providerName), @@ -46,7 +46,7 @@ func ProviderSidecarContainer(shoot *gardencorev1beta1.Shoot, controlPlaneNamesp "--machine-safety-orphan-vms-period=30m", "--namespace=" + controlPlaneNamespace, "--port=" + strconv.Itoa(portProviderMetrics), - "--target-kubeconfig=" + targetKubeconfig(autonomousShoot, controlPlaneNamespace), + "--target-kubeconfig=" + targetKubeconfig(selfHostedShoot, controlPlaneNamespace), "--v=3", }, LivenessProbe: &corev1.Probe{ @@ -79,7 +79,7 @@ func ProviderSidecarContainer(shoot *gardencorev1beta1.Shoot, controlPlaneNamesp }, } - if !autonomousShoot { + if !selfHostedShoot { c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ Name: "kubeconfig", MountPath: gardenerutils.VolumeMountPathGenericKubeconfig, diff --git a/pkg/component/nodemanagement/machinecontrollermanager/provider_test.go b/pkg/component/nodemanagement/machinecontrollermanager/provider_test.go index adde1d032b9..ab279e12b1d 100644 --- a/pkg/component/nodemanagement/machinecontrollermanager/provider_test.go +++ b/pkg/component/nodemanagement/machinecontrollermanager/provider_test.go @@ -89,7 +89,7 @@ var _ = Describe("Provider", func() { } }) - When("the shoot is not autonomous", func() { + When("the shoot is not self-hosted", func() { JustBeforeEach(func() { container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ Name: "kubeconfig", @@ -103,7 +103,7 @@ var _ = Describe("Provider", func() { }) }) - When("the shoot is autonomous", func() { + When("the shoot is self-hosted", func() { BeforeEach(func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, gardencorev1beta1.Worker{ ControlPlane: &gardencorev1beta1.WorkerControlPlane{}, diff --git a/pkg/component/shared/resourcemanager.go b/pkg/component/shared/resourcemanager.go index c01a0b57f84..bb7f7aa523b 100644 --- a/pkg/component/shared/resourcemanager.go +++ b/pkg/component/shared/resourcemanager.go @@ -133,7 +133,7 @@ func combinedGardenerResourceManagerDefaultValues() resourcemanager.Values { } // NewCombinedGardenerResourceManager instantiates a new `gardener-resource-manager` component configured to reconcile -// objects in an autonomous shoot cluster. +// objects in a self-hosted shoot cluster. func NewCombinedGardenerResourceManager(c client.Client, namespaceName string, secretsManager secretsmanager.Interface, diff --git a/pkg/controller/gardenletdeployer/actuator.go b/pkg/controller/gardenletdeployer/actuator.go index 4a701f5b6ef..52dc4c8d5b2 100644 --- a/pkg/controller/gardenletdeployer/actuator.go +++ b/pkg/controller/gardenletdeployer/actuator.go @@ -685,7 +685,7 @@ func PrepareGardenletChartValues( } } - // Ensure seed name is set if we are not deploying gardenlet into an autonomous shoot cluster + // Ensure seed name is set if we are not deploying gardenlet into a self-hosted shoot cluster if gardenletConfig.SeedConfig != nil { gardenletConfig.SeedConfig.Name = obj.GetName() } @@ -725,7 +725,7 @@ func PrepareGardenletChartValues( return values, nil } - // enable self-upgrades for autonomous shoots + // enable self-upgrades for self-hosted shoots gardenletChartImage, err := imagevector.Charts().FindImage(imagevector.ChartImageNameGardenlet) if err != nil { return nil, fmt.Errorf("failed fetching gardenlet chart image: %w", err) @@ -733,7 +733,7 @@ func PrepareGardenletChartValues( gardenletChartImage.WithOptionalTag(version.Get().GitVersion) return utils.SetToValuesMap(values, map[string]any{ - "name": gardenletutils.ResourcePrefixAutonomousShoot + shoot.Name, + "name": gardenletutils.ResourcePrefixSelfHostedShoot + shoot.Name, "namespace": shoot.Namespace, "deployment": map[string]any{"helm": map[string]any{"ociRepository": map[string]any{"ref": gardenletChartImage.String()}}}, }, "selfUpgrade") diff --git a/pkg/controller/gardenletdeployer/actuator_test.go b/pkg/controller/gardenletdeployer/actuator_test.go index 57205b11c9d..2a5848a8b93 100644 --- a/pkg/controller/gardenletdeployer/actuator_test.go +++ b/pkg/controller/gardenletdeployer/actuator_test.go @@ -438,7 +438,7 @@ var _ = Describe("Interface", func() { ) } - expectGetGardenletChartValues = func(withBootstrap, seedIsGarden, autonomousShoot bool) { + expectGetGardenletChartValues = func(withBootstrap, seedIsGarden, selfHostedShoot bool) { gardenletChartValues = map[string]any{"foo": "bar"} vh.EXPECT().GetGardenletChartValues(gomock.AssignableToTypeOf(&seedmanagementv1alpha1.GardenletDeployment{}), gomock.AssignableToTypeOf(&gardenletconfigv1alpha1.GardenletConfiguration{}), gomock.AssignableToTypeOf("")).DoAndReturn( @@ -465,7 +465,7 @@ var _ = Describe("Interface", func() { Expect(deployment.PodLabels).To(BeEmpty()) } - if !autonomousShoot { + if !selfHostedShoot { Expect(gc.SeedConfig.SeedTemplate).To(Equal(gardencorev1beta1.SeedTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -737,7 +737,7 @@ var _ = Describe("Interface", func() { }) }) - Context("autonomous shoot", func() { + Context("self-hosted shoot", func() { var ( deployment *seedmanagementv1alpha1.GardenletDeployment config *gardenletconfigv1alpha1.GardenletConfiguration diff --git a/pkg/controllermanager/controller/certificatesigningrequest/reconciler.go b/pkg/controllermanager/controller/certificatesigningrequest/reconciler.go index 686dc0ddead..7c23cf40753 100644 --- a/pkg/controllermanager/controller/certificatesigningrequest/reconciler.go +++ b/pkg/controllermanager/controller/certificatesigningrequest/reconciler.go @@ -116,7 +116,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( } default: - log.Info("Ignoring CSR, as it does not match the requirements for a seed client or an autonomous shoot client", "reasonSeedCheck", reasonSeed, "reasonShootCheck", reasonShoot, "reasonGardenadmCheck", reasonGardenadm) + log.Info("Ignoring CSR, as it does not match the requirements for a seed client or a self-hosted shoot client", "reasonSeedCheck", reasonSeed, "reasonShootCheck", reasonShoot, "reasonGardenadmCheck", reasonGardenadm) return reconcile.Result{}, nil } diff --git a/pkg/controllermanager/controller/certificatesigningrequest/reconciler_test.go b/pkg/controllermanager/controller/certificatesigningrequest/reconciler_test.go index 21140317e9c..f21dc16ec9a 100644 --- a/pkg/controllermanager/controller/certificatesigningrequest/reconciler_test.go +++ b/pkg/controllermanager/controller/certificatesigningrequest/reconciler_test.go @@ -316,7 +316,7 @@ var _ = Describe("Reconciler", func() { }) When("bootstrap token secret has invalid description", func() { - It("should lead to denial of the CSR when description does not have autonomous shoot prefix", func() { + It("should lead to denial of the CSR when description does not have self-hosted shoot prefix", func() { bootstrapTokenSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: bootstrapTokenName, @@ -362,7 +362,7 @@ var _ = Describe("Reconciler", func() { Namespace: metav1.NamespaceSystem, }, Data: map[string][]byte{ - bootstraptokenapi.BootstrapTokenDescriptionKey: []byte(bootstraptoken.AutonomousShootBootstrapTokenSecretDescriptionPrefix + "invalid-format"), + bootstraptokenapi.BootstrapTokenDescriptionKey: []byte(bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix + "invalid-format"), }, } @@ -414,7 +414,7 @@ var _ = Describe("Reconciler", func() { Namespace: metav1.NamespaceSystem, }, Data: map[string][]byte{ - bootstraptokenapi.BootstrapTokenDescriptionKey: []byte(bootstraptoken.AutonomousShootBootstrapTokenSecretDescriptionPrefix + shootNamespace + "/" + shootName), + bootstraptokenapi.BootstrapTokenDescriptionKey: []byte(bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix + shootNamespace + "/" + shootName), }, } @@ -453,7 +453,7 @@ var _ = Describe("Reconciler", func() { Namespace: metav1.NamespaceSystem, }, Data: map[string][]byte{ - bootstraptokenapi.BootstrapTokenDescriptionKey: []byte(bootstraptoken.AutonomousShootBootstrapTokenSecretDescriptionPrefix + shootNamespace + "/" + shootName), + bootstraptokenapi.BootstrapTokenDescriptionKey: []byte(bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix + shootNamespace + "/" + shootName), }, } @@ -494,7 +494,7 @@ var _ = Describe("Reconciler", func() { Namespace: metav1.NamespaceSystem, }, Data: map[string][]byte{ - bootstraptokenapi.BootstrapTokenDescriptionKey: []byte(bootstraptoken.AutonomousShootBootstrapTokenSecretDescriptionPrefix + shootNamespace + "/" + shootName), + bootstraptokenapi.BootstrapTokenDescriptionKey: []byte(bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix + shootNamespace + "/" + shootName), }, } diff --git a/pkg/gardenadm/botanist/bastion.go b/pkg/gardenadm/botanist/bastion.go index 6742a6e7ce6..83ca2ffbfb4 100644 --- a/pkg/gardenadm/botanist/bastion.go +++ b/pkg/gardenadm/botanist/bastion.go @@ -9,7 +9,7 @@ import ( ) // DefaultBastion creates the Bastion component for accessing the control plane machines in `gardenadm bootstrap`. -func (b *AutonomousBotanist) DefaultBastion() *bastion.Bastion { +func (b *GardenadmBotanist) DefaultBastion() *bastion.Bastion { return bastion.New(b.Logger, b.SeedClientSet.Client(), b.SecretsManager, &bastion.Values{ Name: "gardenadm-bootstrap", Namespace: b.Shoot.ControlPlaneNamespace, diff --git a/pkg/gardenadm/botanist/botanist.go b/pkg/gardenadm/botanist/botanist.go index e0b221d4302..94a896ee69b 100644 --- a/pkg/gardenadm/botanist/botanist.go +++ b/pkg/gardenadm/botanist/botanist.go @@ -48,8 +48,8 @@ import ( // even when running `gardenadm bootstrap` on Windows. const GardenadmBaseDir = "/var/lib/gardenadm" -// AutonomousBotanist is a struct which has methods that perform operations for an autonomous shoot cluster. -type AutonomousBotanist struct { +// GardenadmBotanist is a struct which has methods that perform operations for a self-hosted shoot cluster. +type GardenadmBotanist struct { *botanistpkg.Botanist HostName string @@ -89,15 +89,15 @@ var ( NewFs = afero.NewOsFs ) -// NewAutonomousBotanistFromManifests reads the manifests from dir and initializes a new AutonomousBotanist with them. -func NewAutonomousBotanistFromManifests( +// NewGardenadmBotanistFromManifests reads the manifests from dir and initializes a new GardenadmBotanist with them. +func NewGardenadmBotanistFromManifests( ctx context.Context, log logr.Logger, clientSet kubernetes.Interface, dir string, runsControlPlane bool, ) ( - *AutonomousBotanist, + *GardenadmBotanist, error, ) { resources, err := gardenadm.ReadManifests(log, DirFS(dir)) @@ -110,7 +110,7 @@ func NewAutonomousBotanistFromManifests( return nil, fmt.Errorf("failed computing extensions: %w", err) } - b, err := NewAutonomousBotanist(ctx, log, clientSet, resources, extensions, runsControlPlane) + b, err := NewGardenadmBotanist(ctx, log, clientSet, resources, extensions, runsControlPlane) if err != nil { return nil, fmt.Errorf("failed constructing botanist: %w", err) } @@ -118,8 +118,8 @@ func NewAutonomousBotanistFromManifests( return b, nil } -// NewAutonomousBotanist creates a new botanist.AutonomousBotanist instance for the gardenadm command execution. -func NewAutonomousBotanist( +// NewGardenadmBotanist creates a new botanist.GardenadmBotanist instance for the gardenadm command execution. +func NewGardenadmBotanist( ctx context.Context, log logr.Logger, clientSet kubernetes.Interface, @@ -127,15 +127,15 @@ func NewAutonomousBotanist( extensions []Extension, runsControlPlane bool, ) ( - *AutonomousBotanist, + *GardenadmBotanist, error, ) { - autonomousBotanist, err := NewAutonomousBotanistWithoutResources(log) + gardenadmBotanist, err := NewGardenadmBotanistWithoutResources(log) if err != nil { - return nil, fmt.Errorf("failed creating autonomous botanist: %w", err) + return nil, fmt.Errorf("failed creating gardenadm botanist: %w", err) } - if err := initializeShootResource(resources, autonomousBotanist.FS, runsControlPlane); err != nil { + if err := initializeShootResource(resources, gardenadmBotanist.FS, runsControlPlane); err != nil { return nil, fmt.Errorf("failed initializing shoot resource: %w", err) } @@ -146,36 +146,36 @@ func NewAutonomousBotanist( return nil, fmt.Errorf("failed initializing resources in fake garden client: %w", err) } - autonomousBotanist.Botanist, err = newBotanist(ctx, log, clientSet, gardenClient, resources, runsControlPlane) + gardenadmBotanist.Botanist, err = newBotanist(ctx, log, clientSet, gardenClient, resources, runsControlPlane) if err != nil { return nil, fmt.Errorf("failed creating botanist: %w", err) } - if !autonomousBotanist.Shoot.RunsControlPlane() { - autonomousBotanist.Bastion = autonomousBotanist.DefaultBastion() + if !gardenadmBotanist.Shoot.RunsControlPlane() { + gardenadmBotanist.Bastion = gardenadmBotanist.DefaultBastion() // For `gardenadm bootstrap`, we don't initialize the control plane machines with a "full OSC". // Instead, we provide a small alternative OSC, that only fetches the `gardenadm` binary from the registry. - autonomousBotanist.Shoot.Components.Extensions.OperatingSystemConfig, err = autonomousBotanist.ControlPlaneBootstrapOperatingSystemConfig() + gardenadmBotanist.Shoot.Components.Extensions.OperatingSystemConfig, err = gardenadmBotanist.ControlPlaneBootstrapOperatingSystemConfig() if err != nil { return nil, err } } - autonomousBotanist.Resources = resources - autonomousBotanist.Extensions = extensions + gardenadmBotanist.Resources = resources + gardenadmBotanist.Extensions = extensions - return autonomousBotanist, nil + return gardenadmBotanist, nil } -// NewAutonomousBotanistWithoutResources creates a new AutonomousBotanist without instantiating a Botanist struct. -func NewAutonomousBotanistWithoutResources(log logr.Logger) (*AutonomousBotanist, error) { +// NewGardenadmBotanistWithoutResources creates a new GardenadmBotanist without instantiating a Botanist struct. +func NewGardenadmBotanistWithoutResources(log logr.Logger) (*GardenadmBotanist, error) { hostName, err := nodeagent.GetHostName() if err != nil { return nil, fmt.Errorf("failed fetching hostname: %w", err) } - return &AutonomousBotanist{ + return &GardenadmBotanist{ Botanist: &botanistpkg.Botanist{Operation: newOperation(log, newFakeGardenClient(), newFakeSeedClientSet(""))}, HostName: hostName, @@ -223,9 +223,9 @@ func newBotanist( keysAndValues := []any{"cloudProfile", resources.CloudProfile, "project", resources.Project, "shoot", resources.Shoot} if clientSet == nil { clientSet = newFakeSeedClientSet(seedObj.KubernetesVersion.String()) - log.Info("Initializing autonomous botanist with fake client set", keysAndValues...) //nolint:logcheck + log.Info("Initializing gardenadm botanist with fake client set", keysAndValues...) //nolint:logcheck } else { - log.Info("Initializing autonomous botanist with control plane client set", keysAndValues...) //nolint:logcheck + log.Info("Initializing gardenadm botanist with control plane client set", keysAndValues...) //nolint:logcheck } o := newOperation(log, gardenClient, clientSet) @@ -330,11 +330,11 @@ func newShootObject( return nil, fmt.Errorf("failed computing shoot networks: %w", err) } - // In autonomous shoot clusters, kube-system is used as the control plane namespace. - // However, when bootstrapping an autonomous shoot cluster with `gardenadm bootstrap` using a temporary local cluster, + // In self-hosted shoot clusters, kube-system is used as the control plane namespace. + // However, when bootstrapping a self-hosted shoot cluster with `gardenadm bootstrap` using a temporary local cluster, // we want to avoid conflicts with kube-system components of the bootstrap cluster by placing all shoot-related // components in another namespace. In this case, we use the technical ID as the control plane namespace, as usual. - // TODO(timebertt): double-check if this causes problems when importing the state into the autonomous shoot cluster + // TODO(timebertt): double-check if this causes problems when importing the state into the self-hosted shoot cluster if !runsControlPlane { obj.ControlPlaneNamespace = resources.Shoot.Status.TechnicalID } @@ -388,7 +388,7 @@ func initializeShootResource(resources gardenadm.Resources, fs afero.Afero, runs if resources.ShootState == nil { return fmt.Errorf("shoot has managed infrastructure, but ShootState is missing " + "(the ShootState is usually exported by `gardenadm bootstrap` and read by `gardenadm init`): " + - "you should either use `gardenadm bootstrap` to create the autonomous shoot cluster with managed infrastructure or " + + "you should either use `gardenadm bootstrap` to create the self-hosted shoot cluster with managed infrastructure or " + "remove the `Shoot.spec.{secret,credentials}BindingName` field to mark the shoot as having unmanaged infrastructure") } @@ -412,11 +412,11 @@ func initializeSeedResource(resources gardenadm.Resources, runsControlPlane bool seed.Status = gardencorev1beta1.SeedStatus{ClusterIdentity: ptr.To(resources.Shoot.Name)} if runsControlPlane { - // When running the control plane (`gardenadm init`), mark the seed as an autonomous shoot cluster. + // When running the control plane (`gardenadm init`), mark the seed as a self-hosted shoot cluster. // Otherwise (`gardenadm bootstrap`), the bootstrap cluster should behave like a standard seed cluster. - // If the seed is marked as an autonomous shoot cluster, extensions are configured differently, e.g., they merge the + // If the seed is marked as a self-hosted shoot cluster, extensions are configured differently, e.g., they merge the // shoot webhooks into the seed webhooks. - metav1.SetMetaDataLabel(&seed.ObjectMeta, v1beta1constants.LabelAutonomousShootCluster, "true") + metav1.SetMetaDataLabel(&seed.ObjectMeta, v1beta1constants.LabelSelfHostedShootCluster, "true") } kubernetes.GardenScheme.Default(seed) diff --git a/pkg/gardenadm/botanist/botanist_test.go b/pkg/gardenadm/botanist/botanist_test.go index edff501def8..cfc370d3681 100644 --- a/pkg/gardenadm/botanist/botanist_test.go +++ b/pkg/gardenadm/botanist/botanist_test.go @@ -24,8 +24,8 @@ import ( "github.com/gardener/gardener/pkg/utils/test" ) -var _ = Describe("AutonomousBotanist", func() { - Describe("#NewAutonomousBotanistFromManifests", func() { +var _ = Describe("GardenadmBotanist", func() { + Describe("#NewGardenadmBotanistFromManifests", func() { const configDir = "manifests" var ( @@ -50,13 +50,13 @@ var _ = Describe("AutonomousBotanist", func() { }) It("should fail if the directory does not exist", func() { - Expect(NewAutonomousBotanistFromManifests(ctx, log, nil, "does/not/exist", false)).Error().To(MatchError(fs.ErrNotExist)) + Expect(NewGardenadmBotanistFromManifests(ctx, log, nil, "does/not/exist", false)).Error().To(MatchError(fs.ErrNotExist)) }) - When("running the control plane (acting on the autonomous shoot cluster)", func() { - Context("with unmanaged infrastructure (high-touch scenario)", func() { - It("should create a new Autonomous Botanist", func() { - b, err := NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, true) + When("running the control plane (acting on the self-hosted shoot cluster)", func() { + Context("with unmanaged infrastructure", func() { + It("should create a new Self-Hosted Botanist", func() { + b, err := NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, true) Expect(err).NotTo(HaveOccurred()) Expect(b.Shoot.CloudProfile.Name).To(Equal("stackit")) @@ -66,11 +66,11 @@ var _ = Describe("AutonomousBotanist", func() { HaveField("ControllerRegistration.Name", "provider-stackit"), HaveField("ControllerRegistration.Name", "networking-cilium"), )) - Expect(b.Seed.GetInfo()).To(HaveField("ObjectMeta.Labels", HaveKeyWithValue("seed.gardener.cloud/autonomous-shoot-cluster", "true"))) + Expect(b.Seed.GetInfo()).To(HaveField("ObjectMeta.Labels", HaveKeyWithValue("seed.gardener.cloud/self-hosted-shoot-cluster", "true"))) }) }) - Context("with managed infrastructure (medium-touch scenario)", func() { + Context("with managed infrastructure", func() { BeforeEach(func() { shootFile := fsys[configDir+"/shoot.yaml"] shootFile.Data = append(shootFile.Data, []byte("\n credentialsBindingName: provider-account\n")...) @@ -84,11 +84,11 @@ metadata: It("should fail if the ShootState is missing", func() { delete(fsys, configDir+"/shootstate.yaml") - Expect(NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, true)).Error().To(MatchError(ContainSubstring("ShootState is missing"))) + Expect(NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, true)).Error().To(MatchError(ContainSubstring("ShootState is missing"))) }) It("should set the LastOperation to Restore and fetch the ShootState", func() { - b, err := NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, true) + b, err := NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, true) Expect(err).NotTo(HaveOccurred()) Expect(b.Shoot.GetInfo().Status.LastOperation.Type).To(Equal(gardencorev1beta1.LastOperationTypeRestore)) @@ -98,7 +98,7 @@ metadata: }) It("should use kube-system as the control plane namespace", func() { - b, err := NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, true) + b, err := NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, true) Expect(err).NotTo(HaveOccurred()) Expect(b.Shoot.ControlPlaneNamespace).To(Equal("kube-system")) }) @@ -108,7 +108,7 @@ metadata: DeferCleanup(test.WithVar(&NewFs, func() afero.Fs { return fs })) By("Generate new shoot UID and write it to the host") - b, err := NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, true) + b, err := NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, true) Expect(err).NotTo(HaveOccurred()) uid := b.Shoot.GetInfo().Status.UID @@ -121,7 +121,7 @@ metadata: Expect(string(content)).To(Equal(string(uid))) By("Do not regenerate shoot UID when file is present on host") - b, err = NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, true) + b, err = NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, true) Expect(err).NotTo(HaveOccurred()) Expect(b.Shoot.GetInfo().Status.UID).To(Equal(uid)) @@ -132,8 +132,8 @@ metadata: }) When("not running the control plane (acting on the bootstrap cluster)", func() { - It("should create a new Autonomous Botanist", func() { - b, err := NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, false) + It("should create a new Self-Hosted Botanist", func() { + b, err := NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, false) Expect(err).NotTo(HaveOccurred()) Expect(b.Shoot.CloudProfile.Name).To(Equal("stackit")) @@ -143,11 +143,11 @@ metadata: HaveField("ControllerRegistration.Name", "provider-stackit"), HaveField("ControllerRegistration.Name", "dns-local"), )) - Expect(b.Seed.GetInfo()).To(HaveField("ObjectMeta.Labels", Not(HaveKeyWithValue("seed.gardener.cloud/autonomous-shoot-cluster", "true")))) + Expect(b.Seed.GetInfo()).To(HaveField("ObjectMeta.Labels", Not(HaveKeyWithValue("seed.gardener.cloud/self-hosted-shoot-cluster", "true")))) }) It("should use the technical ID as the control plane namespace", func() { - b, err := NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, false) + b, err := NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, false) Expect(err).NotTo(HaveOccurred()) Expect(b.Shoot.ControlPlaneNamespace).To(Equal("shoot--gardenadm--gardenadm")) }) @@ -156,7 +156,7 @@ metadata: fs := afero.NewMemMapFs() DeferCleanup(test.WithVar(&NewFs, func() afero.Fs { return fs })) - b, err := NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, false) + b, err := NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, false) Expect(err).NotTo(HaveOccurred()) uid := b.Shoot.GetInfo().Status.UID @@ -168,7 +168,7 @@ metadata: }) It("should create the secrets with the fake garden client", func() { - b, err := NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, false) + b, err := NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, false) Expect(err).NotTo(HaveOccurred()) Expect(b.GardenClient.Get(ctx, client.ObjectKey{Name: "secret1"}, &corev1.Secret{})).To(Succeed()) @@ -177,7 +177,7 @@ metadata: }) It("should create the secret binding and credentials binding", func() { - b, err := NewAutonomousBotanistFromManifests(ctx, log, nil, configDir, true) + b, err := NewGardenadmBotanistFromManifests(ctx, log, nil, configDir, true) Expect(err).NotTo(HaveOccurred()) Expect(b.GardenClient.Get(ctx, client.ObjectKey{Name: "provider-account"}, &corev1.Secret{})).To(Succeed()) diff --git a/pkg/gardenadm/botanist/controlplane.go b/pkg/gardenadm/botanist/controlplane.go index 5e612fff149..97e58c0b639 100644 --- a/pkg/gardenadm/botanist/controlplane.go +++ b/pkg/gardenadm/botanist/controlplane.go @@ -49,10 +49,10 @@ import ( var PathKubeconfig = filepath.Join(string(filepath.Separator), "etc", "kubernetes", "admin.conf") // KubeconfigSecretName is the name of the secret in the shoot namespace of the bootstrap cluster containing the -// kubeconfig of the autonomous shoot. +// kubeconfig of the self-hosted shoot. const KubeconfigSecretName = "kubeconfig" -func (b *AutonomousBotanist) deployETCD(role string) func(context.Context) error { +func (b *GardenadmBotanist) deployETCD(role string) func(context.Context) error { var portClient, portPeer, portMetrics int32 = 2379, 2380, 2381 if role == v1beta1constants.ETCDRoleEvents { portClient, portPeer, portMetrics = etcdconstants.StaticPodPortEtcdEventsClient, 2383, 2384 @@ -74,7 +74,7 @@ func (b *AutonomousBotanist) deployETCD(role string) func(context.Context) error } } -func (b *AutonomousBotanist) deployKubeAPIServer(ctx context.Context) error { +func (b *GardenadmBotanist) deployKubeAPIServer(ctx context.Context) error { b.Shoot.Components.ControlPlane.KubeAPIServer.EnableStaticTokenKubeconfig() b.Shoot.Components.ControlPlane.KubeAPIServer.SetAutoscalingReplicas(ptr.To[int32](0)) @@ -102,7 +102,7 @@ type staticControlPlaneComponent struct { mutate func(*corev1.Pod) } -func (b *AutonomousBotanist) staticControlPlaneComponents() []staticControlPlaneComponent { +func (b *GardenadmBotanist) staticControlPlaneComponents() []staticControlPlaneComponent { var ( components []staticControlPlaneComponent @@ -147,7 +147,7 @@ func (b *AutonomousBotanist) staticControlPlaneComponents() []staticControlPlane // DeployControlPlaneDeployments deploys the deployments for the static control plane components. It also updates the // OperatingSystemConfig, waits for it to be reconciled by the OS extension, and deploys the ManagedResource containing // the Secret with OperatingSystemConfig for gardener-node-agent. -func (b *AutonomousBotanist) DeployControlPlaneDeployments(ctx context.Context) error { +func (b *GardenadmBotanist) DeployControlPlaneDeployments(ctx context.Context) error { if err := b.deployControlPlaneDeployments(ctx); err != nil { return fmt.Errorf("failed deploying control plane deployments: %w", err) } @@ -171,7 +171,7 @@ func (b *AutonomousBotanist) DeployControlPlaneDeployments(ctx context.Context) // WaitUntilControlPlaneDeploymentsReady waits until the control plane deployments are ready. It checks that the // operating system config has been updated for all worker pools. In addition, it verifies that the static pod hashes // match the desired hashes. -func (b *AutonomousBotanist) WaitUntilControlPlaneDeploymentsReady(ctx context.Context) error { +func (b *GardenadmBotanist) WaitUntilControlPlaneDeploymentsReady(ctx context.Context) error { timeoutCtx, cancel := context.WithTimeout(ctx, botanist.GetTimeoutWaitOperatingSystemConfigUpdated(b.Shoot)) defer cancel() @@ -201,7 +201,7 @@ func (b *AutonomousBotanist) WaitUntilControlPlaneDeploymentsReady(ctx context.C return b.WaitUntilOperatingSystemConfigUpdatedForAllWorkerPools(ctx) } -func (b *AutonomousBotanist) deployControlPlaneDeployments(ctx context.Context) error { +func (b *GardenadmBotanist) deployControlPlaneDeployments(ctx context.Context) error { for _, component := range b.staticControlPlaneComponents() { if err := b.deployControlPlaneComponent(ctx, component.deploy, component.targetObject, component.name); err != nil { return fmt.Errorf("failed deploying %q: %w", component.name, err) @@ -211,7 +211,7 @@ func (b *AutonomousBotanist) deployControlPlaneDeployments(ctx context.Context) return nil } -func (b *AutonomousBotanist) deployControlPlaneComponent(ctx context.Context, deploy func(context.Context) error, targetObject client.Object, componentName string) error { +func (b *GardenadmBotanist) deployControlPlaneComponent(ctx context.Context, deploy func(context.Context) error, targetObject client.Object, componentName string) error { if err := deploy(ctx); err != nil { return fmt.Errorf("failed deploying component %q: %w", componentName, err) } @@ -225,7 +225,7 @@ func (b *AutonomousBotanist) deployControlPlaneComponent(ctx context.Context, de return b.SeedClientSet.Client().Get(ctx, client.ObjectKeyFromObject(targetObject), targetObject) } -func (b *AutonomousBotanist) populateStaticAdminTokenToAccessTokenSecret(ctx context.Context, componentName string) error { +func (b *GardenadmBotanist) populateStaticAdminTokenToAccessTokenSecret(ctx context.Context, componentName string) error { secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: gardenerutils.SecretNamePrefixShootAccess + componentName, Namespace: b.Shoot.ControlPlaneNamespace}} if err := b.SeedClientSet.Client().Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { if apierrors.IsNotFound(err) { @@ -281,7 +281,7 @@ func (s staticPods) nameToHashMap() map[string]string { return m } -func (b *AutonomousBotanist) staticControlPlanePods(ctx context.Context) (staticPods, error) { +func (b *GardenadmBotanist) staticControlPlanePods(ctx context.Context) (staticPods, error) { var pods staticPods for _, component := range b.staticControlPlaneComponents() { @@ -314,7 +314,7 @@ func NewClientSetFromFile(kubeconfigPath string, scheme *runtime.Scheme) (kubern } // CreateClientSet creates a client set for the control plane. -func (b *AutonomousBotanist) CreateClientSet(ctx context.Context) (kubernetes.Interface, error) { +func (b *GardenadmBotanist) CreateClientSet(ctx context.Context) (kubernetes.Interface, error) { pathKubeconfig := PathKubeconfig if path := os.Getenv("KUBECONFIG"); path != "" { pathKubeconfig = path @@ -354,7 +354,7 @@ func (b *AutonomousBotanist) CreateClientSet(ctx context.Context) (kubernetes.In var NewWithConfig = kubernetes.NewWithConfig // DiscoverKubernetesVersion discovers the Kubernetes version of the control plane. -func (b *AutonomousBotanist) DiscoverKubernetesVersion(controlPlaneAddress string, caBundle []byte, token string) (*semver.Version, error) { +func (b *GardenadmBotanist) DiscoverKubernetesVersion(controlPlaneAddress string, caBundle []byte, token string) (*semver.Version, error) { clientSet, err := NewWithConfig(kubernetes.WithRESTConfig(&rest.Config{ Host: controlPlaneAddress, TLSClientConfig: rest.TLSClientConfig{CAData: caBundle}, @@ -372,10 +372,10 @@ func (b *AutonomousBotanist) DiscoverKubernetesVersion(controlPlaneAddress strin return version, nil } -// FetchKubeconfig fetches the kubeconfig of the autonomous shoot from the control plane machine via SSH and stores it +// FetchKubeconfig fetches the kubeconfig of the self-hosted shoot from the control plane machine via SSH and stores it // in a secret in the shoot namespace of the bootstrap cluster for later retrieval. It also writes the kubeconfig to the // given output writer, if any. -func (b *AutonomousBotanist) FetchKubeconfig(ctx context.Context, output io.Writer) error { +func (b *GardenadmBotanist) FetchKubeconfig(ctx context.Context, output io.Writer) error { kubeconfigBuffer := &bytes.Buffer{} if err := b.sshConnection.SCP.CopyFromRemotePassThru(ctx, kubeconfigBuffer, PathKubeconfig, nil); err != nil { return fmt.Errorf("error fetching kubeconfig: %w", err) @@ -392,11 +392,11 @@ func (b *AutonomousBotanist) FetchKubeconfig(ctx context.Context, output io.Writ if err != nil { return fmt.Errorf("error writing kubeconfig secret: %w", err) } - b.Logger.Info("Stored kubeconfig of the autonomous shoot in secret", "namespace", kubeconfigSecret.Namespace, "name", kubeconfigSecret.Name) + b.Logger.Info("Stored kubeconfig of the self-hosted shoot in secret", "namespace", kubeconfigSecret.Namespace, "name", kubeconfigSecret.Name) if output != nil { if outputFile, ok := output.(*os.File); ok { - b.Logger.Info("Writing kubeconfig of the autonomous shoot to file", "path", outputFile.Name()) + b.Logger.Info("Writing kubeconfig of the self-hosted shoot to file", "path", outputFile.Name()) } if _, err := output.Write(kubeconfigBytes); err != nil { return fmt.Errorf("error writing kubeconfig to output: %w", err) diff --git a/pkg/gardenadm/botanist/controlplane_test.go b/pkg/gardenadm/botanist/controlplane_test.go index d103bead3fe..524157852cd 100644 --- a/pkg/gardenadm/botanist/controlplane_test.go +++ b/pkg/gardenadm/botanist/controlplane_test.go @@ -28,10 +28,10 @@ import ( ) var _ = Describe("ControlPlane", func() { - var b *AutonomousBotanist + var b *GardenadmBotanist BeforeEach(func() { - b = &AutonomousBotanist{} + b = &GardenadmBotanist{} }) Describe("#DiscoverKubernetesVersion", func() { diff --git a/pkg/gardenadm/botanist/customresourcedefinitions.go b/pkg/gardenadm/botanist/customresourcedefinitions.go index f350eb2d3bf..71fad7d1e5e 100644 --- a/pkg/gardenadm/botanist/customresourcedefinitions.go +++ b/pkg/gardenadm/botanist/customresourcedefinitions.go @@ -21,7 +21,7 @@ import ( ) // ReconcileCustomResourceDefinitions reconciles the custom resource definitions. -func (b *AutonomousBotanist) ReconcileCustomResourceDefinitions(ctx context.Context) error { +func (b *GardenadmBotanist) ReconcileCustomResourceDefinitions(ctx context.Context) error { vpaCRDDeployer, err := vpa.NewCRD(b.SeedClientSet.Client(), nil) if err != nil { return fmt.Errorf("failed creating VPA CRD deployer: %w", err) @@ -72,7 +72,7 @@ func (b *AutonomousBotanist) ReconcileCustomResourceDefinitions(ctx context.Cont } // EnsureCustomResourceDefinitionsReady ensures that the custom resource definitions are ready. -func (b *AutonomousBotanist) EnsureCustomResourceDefinitionsReady(ctx context.Context) error { +func (b *GardenadmBotanist) EnsureCustomResourceDefinitionsReady(ctx context.Context) error { crdList := &apiextensionsv1.CustomResourceDefinitionList{} if err := b.SeedClientSet.Client().List(ctx, crdList); err != nil { return fmt.Errorf("failed to list CustomResourceDefinitions: %w", err) diff --git a/pkg/gardenadm/botanist/customresourcedefinitions_test.go b/pkg/gardenadm/botanist/customresourcedefinitions_test.go index f37373e8359..fa8295df3ce 100644 --- a/pkg/gardenadm/botanist/customresourcedefinitions_test.go +++ b/pkg/gardenadm/botanist/customresourcedefinitions_test.go @@ -34,7 +34,7 @@ var _ = Describe("CustomResourceDefinitions", func() { fakeClient client.Client - b *AutonomousBotanist + b *GardenadmBotanist ) BeforeEach(func() { @@ -46,7 +46,7 @@ var _ = Describe("CustomResourceDefinitions", func() { mapper.Add(apiextensionsv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"), meta.RESTScopeRoot) applier := kubernetes.NewApplier(fakeClient, mapper) - b = &AutonomousBotanist{ + b = &GardenadmBotanist{ Botanist: &botanistpkg.Botanist{ Operation: &operation.Operation{ SeedClientSet: fakekubernetes. @@ -125,7 +125,7 @@ var _ = Describe("CustomResourceDefinitions", func() { }) }) - It("should deploy the additional CRDs for the medium-touch scenario", func() { + It("should deploy the additional CRDs for the managed infrastructure scenario", func() { Expect(b.ReconcileCustomResourceDefinitions(ctx)).To(Succeed()) crdList := &apiextensionsv1.CustomResourceDefinitionList{} diff --git a/pkg/gardenadm/botanist/dnsrecord.go b/pkg/gardenadm/botanist/dnsrecord.go index ba5bffd2030..f06cbc17911 100644 --- a/pkg/gardenadm/botanist/dnsrecord.go +++ b/pkg/gardenadm/botanist/dnsrecord.go @@ -12,10 +12,10 @@ import ( ) // DeployBootstrapDNSRecord deploys the external DNSRecord pointing to the first control plane machine for bootstrapping -// the autonomous shoot cluster. The DNSRecord might be publicly resolvable but not publicly accessible, depending on +// the self-hosted shoot cluster. The DNSRecord might be publicly resolvable but not publicly accessible, depending on // shoot's infrastructure provider and network setup. It should be resolvable and accessible from within the shoot // cluster's network including the machines, so `gardenadm init` can use it to bootstrap the cluster. -func (b *AutonomousBotanist) DeployBootstrapDNSRecord(ctx context.Context) error { +func (b *GardenadmBotanist) DeployBootstrapDNSRecord(ctx context.Context) error { machine, err := b.GetMachineByIndex(0) if err != nil { return err diff --git a/pkg/gardenadm/botanist/etcd.go b/pkg/gardenadm/botanist/etcd.go index c4f1fcdbad8..4b5fd1b4bcb 100644 --- a/pkg/gardenadm/botanist/etcd.go +++ b/pkg/gardenadm/botanist/etcd.go @@ -35,7 +35,7 @@ import ( ) // DeployEtcdDruid deploys the etcd-druid component. -func (b *AutonomousBotanist) DeployEtcdDruid(ctx context.Context) error { +func (b *GardenadmBotanist) DeployEtcdDruid(ctx context.Context) error { var componentImageVectors imagevectorutils.ComponentImageVectors if path := os.Getenv(imagevectorutils.ComponentOverrideEnv); path != "" { var err error @@ -67,7 +67,7 @@ func (b *AutonomousBotanist) DeployEtcdDruid(ctx context.Context) error { } // ReconcileBackupBucket reconciles the core.gardener.cloud/v1beta1.BackupBucket resource for the shoot cluster. -func (b *AutonomousBotanist) ReconcileBackupBucket(ctx context.Context) error { +func (b *GardenadmBotanist) ReconcileBackupBucket(ctx context.Context) error { backupBucket, err := b.reconcileCoreBackupBucketResource(ctx) if err != nil { return fmt.Errorf("failed reconciling core.gardener.cloud/v1beta1.BackupBucket resource: %w", err) @@ -90,7 +90,7 @@ func (b *AutonomousBotanist) ReconcileBackupBucket(ctx context.Context) error { }) } -func (b *AutonomousBotanist) reconcileCoreBackupBucketResource(ctx context.Context) (*gardencorev1beta1.BackupBucket, error) { +func (b *GardenadmBotanist) reconcileCoreBackupBucketResource(ctx context.Context) (*gardencorev1beta1.BackupBucket, error) { component := corebackupbucket.New(b.Logger, b.GardenClient, &corebackupbucket.Values{ Name: string(b.Shoot.GetInfo().Status.UID), Config: v1beta1helper.GetBackupConfigForShoot(b.Shoot.GetInfo(), nil), @@ -106,7 +106,7 @@ func (b *AutonomousBotanist) reconcileCoreBackupBucketResource(ctx context.Conte } // ReconcileBackupEntry reconciles the core.gardener.cloud/v1beta1.BackupEntry resource for the shoot cluster. -func (b *AutonomousBotanist) ReconcileBackupEntry(ctx context.Context) error { +func (b *GardenadmBotanist) ReconcileBackupEntry(ctx context.Context) error { backupEntry, err := b.reconcileCoreBackupEntryResource(ctx) if err != nil { return fmt.Errorf("failed reconciling core.gardener.cloud/v1beta1.BackupEntry resource: %w", err) @@ -129,7 +129,7 @@ func (b *AutonomousBotanist) ReconcileBackupEntry(ctx context.Context) error { }) } -func (b *AutonomousBotanist) reconcileCoreBackupEntryResource(ctx context.Context) (*gardencorev1beta1.BackupEntry, error) { +func (b *GardenadmBotanist) reconcileCoreBackupEntryResource(ctx context.Context) (*gardencorev1beta1.BackupEntry, error) { if err := b.Shoot.Components.BackupEntry.Deploy(ctx); err != nil { return nil, fmt.Errorf("failed reconciling core.gardener.cloud/v1beta1.BackupEntry resource: %w", err) } @@ -161,7 +161,7 @@ func runReconcilerUntilCondition(ctx context.Context, logger logr.Logger, contro // WaitUntilEtcdsReconciled waits until the druid.gardener.cloud/v1alpha1.Etcd resources have been reconciled by // etcd-druid. -func (b *AutonomousBotanist) WaitUntilEtcdsReconciled(ctx context.Context) error { +func (b *GardenadmBotanist) WaitUntilEtcdsReconciled(ctx context.Context) error { if err := b.WaitUntilEtcdsReady(ctx); err != nil { return fmt.Errorf("failed waiting for etcd to become ready: %w", err) } @@ -172,7 +172,7 @@ func (b *AutonomousBotanist) WaitUntilEtcdsReconciled(ctx context.Context) error // FinalizeEtcdBootstrapTransition cleans up no longer needed directories for the bootstrap etcds. Those are not deleted // automatically. -func (b *AutonomousBotanist) FinalizeEtcdBootstrapTransition(_ context.Context) error { +func (b *GardenadmBotanist) FinalizeEtcdBootstrapTransition(_ context.Context) error { for _, dir := range []string{ filepath.Join(string(filepath.Separator), "var", "lib", bootstrapetcd.Name(v1beta1constants.ETCDRoleMain)), filepath.Join(string(filepath.Separator), "var", "lib", bootstrapetcd.Name(v1beta1constants.ETCDRoleEvents)), diff --git a/pkg/gardenadm/botanist/extensions.go b/pkg/gardenadm/botanist/extensions.go index d9069f719e1..e6339663e7d 100644 --- a/pkg/gardenadm/botanist/extensions.go +++ b/pkg/gardenadm/botanist/extensions.go @@ -84,31 +84,29 @@ func ComputeExtensions(resources gardenadm.Resources, runsControlPlane, managedI return extensions, nil } -// wantedExtensionKinds returns the set of extension kinds that are needed and supported for autonomous shoot clusters. +// wantedExtensionKinds returns the set of extension kinds that are needed and supported for self-hosted shoot clusters. // runsControlPlane indicates whether we are bootstrapping the control plane of the cluster (i.e., when executing // `gardenadm init`). -// managedInfrastructure indicates whether the infrastructure of the shoot cluster is managed by Gardener (medium-touch -// scenario) or not (high-touch scenario). func wantedExtensionKinds(runsControlPlane, managedInfrastructure bool) sets.Set[string] { if !runsControlPlane { // When running `gardenadm bootstrap` against the bootstrap cluster, we create Infrastructure, OSC, Worker, and - // DNSRecord for the control plane of the autonomous shoot cluster, so we only need to deploy a subset of the + // DNSRecord for the control plane of the self-hosted shoot cluster, so we only need to deploy a subset of the // extensions required for the shoot. return sets.New[string](extensionsv1alpha1.InfrastructureResource, extensionsv1alpha1.OperatingSystemConfigResource, extensionsv1alpha1.WorkerResource, extensionsv1alpha1.DNSRecordResource) } - // In the high-touch scenario, we don't deploy Infrastructure, Worker, and DNSRecord extensions because they are - // managed outside of Gardener. + // In the "unmanaged infrastructure" scenario, we don't deploy Infrastructure, Worker, and DNSRecord extensions + // because they are managed outside of Gardener. if !managedInfrastructure { return extensionsv1alpha1.AllExtensionKinds.Clone().Delete(extensionsv1alpha1.InfrastructureResource, extensionsv1alpha1.WorkerResource, extensionsv1alpha1.DNSRecordResource) } - // In `gardenadm init`, we deploy all extensions referenced by the shoot in the medium-touch scenario. + // In `gardenadm init`, we deploy all extensions referenced by the shoot in the "managed infrastructure" scenario. return extensionsv1alpha1.AllExtensionKinds.Clone() } -// computeWantedControllerRegistrationNames returns the names of all ControllerRegistrations relevant for the autonomous -// botanist based on the parsed manifests and the wanted extension kinds. +// computeWantedControllerRegistrationNames returns the names of all ControllerRegistrations relevant for the +// gardenadm botanist based on the parsed manifests and the wanted extension kinds. func computeWantedControllerRegistrationNames(resources gardenadm.Resources, wantedExtensionKinds sets.Set[string]) (sets.Set[string], error) { var ( result = sets.New[string]() @@ -164,7 +162,7 @@ func controllerRegistrationSliceToList(controllerRegistrations []*gardencorev1be // ReconcileExtensionControllerInstallations reconciles the ControllerInstallation resources necessary to deploy the // extension controllers. -func (b *AutonomousBotanist) ReconcileExtensionControllerInstallations(ctx context.Context, bootstrapMode bool) error { +func (b *GardenadmBotanist) ReconcileExtensionControllerInstallations(ctx context.Context, bootstrapMode bool) error { reconciler := controllerinstallation.Reconciler{ GardenClient: b.GardenClient, SeedClientSet: b.SeedClientSet, @@ -193,7 +191,7 @@ var TimeoutManagedResourceHealthCheck = 2 * time.Minute // WaitUntilExtensionControllerInstallationsHealthy waits until all ControllerInstallation resources used for // extension controller deployments are healthy. -func (b *AutonomousBotanist) WaitUntilExtensionControllerInstallationsHealthy(ctx context.Context) error { +func (b *GardenadmBotanist) WaitUntilExtensionControllerInstallationsHealthy(ctx context.Context) error { var taskFns []flow.TaskFn for _, extension := range b.Extensions { diff --git a/pkg/gardenadm/botanist/extensions_test.go b/pkg/gardenadm/botanist/extensions_test.go index 50fbe3c707b..d9b415b04c5 100644 --- a/pkg/gardenadm/botanist/extensions_test.go +++ b/pkg/gardenadm/botanist/extensions_test.go @@ -230,7 +230,7 @@ var _ = Describe("Extensions", func() { }) When("running the control plane (gardenadm init)", func() { - When("infrastructure is managed outside of Gardener (high-touch)", func() { + When("infrastructure is not managed by Gardener", func() { It("should return all extensions referenced by shoot (except Infrastructure, Worker, and DNSRecord)", func() { Expect(ComputeExtensions(resources, true, false)).To(ConsistOf( And( @@ -249,7 +249,7 @@ var _ = Describe("Extensions", func() { }) }) - When("infrastructure is managed by Gardener (medium-touch)", func() { + When("infrastructure is managed by Gardener", func() { It("should return all extensions referenced by shoot", func() { Expect(ComputeExtensions(resources, true, true)).To(ConsistOf( And( @@ -305,7 +305,7 @@ var _ = Describe("Extensions", func() { extension2 = "ext2" fakeClient client.Client - b *AutonomousBotanist + b *GardenadmBotanist managedResource1 *resourcesv1alpha1.ManagedResource managedResource2 *resourcesv1alpha1.ManagedResource @@ -313,7 +313,7 @@ var _ = Describe("Extensions", func() { BeforeEach(func() { fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).WithStatusSubresource(&resourcesv1alpha1.ManagedResource{}).Build() - b = &AutonomousBotanist{ + b = &GardenadmBotanist{ Botanist: &botanistpkg.Botanist{ Operation: &operation.Operation{ SeedClientSet: fakekubernetes.NewClientSetBuilder().WithClient(fakeClient).Build(), diff --git a/pkg/gardenadm/botanist/kubelet.go b/pkg/gardenadm/botanist/kubelet.go index 2c9286a4589..09a72e670b0 100644 --- a/pkg/gardenadm/botanist/kubelet.go +++ b/pkg/gardenadm/botanist/kubelet.go @@ -21,7 +21,7 @@ import ( const kubeletTokenFilePermission = 0o600 // WriteKubeletBootstrapKubeconfig writes the kubelet bootstrap kubeconfig to the file system. -func (b *AutonomousBotanist) WriteKubeletBootstrapKubeconfig(ctx context.Context) error { +func (b *GardenadmBotanist) WriteKubeletBootstrapKubeconfig(ctx context.Context) error { if err := b.ensureGardenerNodeAgentDirectories(); err != nil { return fmt.Errorf("failed ensuring gardener-node-agent directories exist: %w", err) } diff --git a/pkg/gardenadm/botanist/kubelet_test.go b/pkg/gardenadm/botanist/kubelet_test.go index 9e331a7bc0e..b9479ee9e32 100644 --- a/pkg/gardenadm/botanist/kubelet_test.go +++ b/pkg/gardenadm/botanist/kubelet_test.go @@ -40,7 +40,7 @@ var _ = Describe("Kubelet", func() { fakeSecretManager secretsmanager.Interface fakeDBus *fakedbus.DBus - b *AutonomousBotanist + b *GardenadmBotanist ) BeforeEach(func() { @@ -52,7 +52,7 @@ var _ = Describe("Kubelet", func() { fakeSecretManager = fakesecretsmanager.New(fakeSeedClient, namespace) fakeDBus = fakedbus.New() - b = &AutonomousBotanist{ + b = &GardenadmBotanist{ Botanist: &botanistpkg.Botanist{ Operation: &operation.Operation{ Logger: logr.Discard(), diff --git a/pkg/gardenadm/botanist/machines.go b/pkg/gardenadm/botanist/machines.go index e5df7b28238..e103dc75851 100644 --- a/pkg/gardenadm/botanist/machines.go +++ b/pkg/gardenadm/botanist/machines.go @@ -16,7 +16,7 @@ import ( // ListControlPlaneMachines stores all control plane machines in controlPlaneMachines for later retrieval. // Listing the machines only once ensures consistent ordering when accessing them by index. -func (b *AutonomousBotanist) ListControlPlaneMachines(ctx context.Context) error { +func (b *GardenadmBotanist) ListControlPlaneMachines(ctx context.Context) error { machineList := &machinev1alpha1.MachineList{} if err := b.SeedClientSet.Client().List(ctx, machineList, client.InNamespace(b.Shoot.ControlPlaneNamespace)); err != nil { return fmt.Errorf("failed to list machines: %w", err) @@ -26,7 +26,7 @@ func (b *AutonomousBotanist) ListControlPlaneMachines(ctx context.Context) error } // GetMachineByIndex returns the control plane machine with the given index or an error if the index is out of bounds. -func (b *AutonomousBotanist) GetMachineByIndex(index int) (*machinev1alpha1.Machine, error) { +func (b *GardenadmBotanist) GetMachineByIndex(index int) (*machinev1alpha1.Machine, error) { if index < 0 { return nil, fmt.Errorf("machine index must be non-negative, got %d", index) } diff --git a/pkg/gardenadm/botanist/network.go b/pkg/gardenadm/botanist/network.go index 0e96ce92e32..caf007a99ef 100644 --- a/pkg/gardenadm/botanist/network.go +++ b/pkg/gardenadm/botanist/network.go @@ -27,7 +27,7 @@ import ( // IsPodNetworkAvailable checks if the ManagedResource for CoreDNS is deployed and ready. If yes, pod network must be // available. Otherwise, CoreDNS which runs in this network wouldn't be available. -func (b *AutonomousBotanist) IsPodNetworkAvailable(ctx context.Context) (bool, error) { +func (b *GardenadmBotanist) IsPodNetworkAvailable(ctx context.Context) (bool, error) { managedResource := &resourcesv1alpha1.ManagedResource{ObjectMeta: metav1.ObjectMeta{Name: coredns.ManagedResourceName, Namespace: b.Shoot.ControlPlaneNamespace}} if err := b.SeedClientSet.Client().Get(ctx, client.ObjectKeyFromObject(managedResource), managedResource); err != nil { if meta.IsNoMatchError(err) || apierrors.IsNotFound(err) { @@ -39,7 +39,7 @@ func (b *AutonomousBotanist) IsPodNetworkAvailable(ctx context.Context) (bool, e } // ApplyNetworkPolicies reconciles all namespaces in the cluster in order to apply the network policies. -func (b *AutonomousBotanist) ApplyNetworkPolicies(ctx context.Context) error { +func (b *GardenadmBotanist) ApplyNetworkPolicies(ctx context.Context) error { reconciler := &networkpolicy.Reconciler{ RuntimeClient: b.SeedClientSet.Client(), Resolver: hostnameresolver.NewNoOpProvider(), diff --git a/pkg/gardenadm/botanist/network_test.go b/pkg/gardenadm/botanist/network_test.go index 4e678f7fcfd..1d6944eb9f5 100644 --- a/pkg/gardenadm/botanist/network_test.go +++ b/pkg/gardenadm/botanist/network_test.go @@ -31,13 +31,13 @@ var _ = Describe("Network", func() { ctx context.Context namespaceName = "kube-system" - b *AutonomousBotanist + b *GardenadmBotanist ) BeforeEach(func() { ctx = context.Background() - b = &AutonomousBotanist{ + b = &GardenadmBotanist{ Botanist: &botanistpkg.Botanist{ Operation: &operation.Operation{ Shoot: &shoot.Shoot{ diff --git a/pkg/gardenadm/botanist/nodeagent.go b/pkg/gardenadm/botanist/nodeagent.go index 65e5564411d..bd8988c46ce 100644 --- a/pkg/gardenadm/botanist/nodeagent.go +++ b/pkg/gardenadm/botanist/nodeagent.go @@ -36,7 +36,7 @@ import ( // WriteBootstrapToken creates a bootstrap token for the gardener-node-agent and kubelet, and writes it to the file // system. -func (b *AutonomousBotanist) WriteBootstrapToken(ctx context.Context) error { +func (b *GardenadmBotanist) WriteBootstrapToken(ctx context.Context) error { bootstrapTokenSecret, err := bootstraptoken.ComputeBootstrapToken( ctx, b.SeedClientSet.Client(), @@ -57,7 +57,7 @@ func emptyTemporaryClusterAdminBinding() *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "zzz-temporary-cluster-admin-access-for-bootstrapping"}} } -func (b *AutonomousBotanist) reconcileTemporaryClusterAdminBindingForBootstrapping(ctx context.Context) error { +func (b *GardenadmBotanist) reconcileTemporaryClusterAdminBindingForBootstrapping(ctx context.Context) error { clusterRoleBinding := emptyTemporaryClusterAdminBinding() _, err := controllerutil.CreateOrUpdate(ctx, b.SeedClientSet.Client(), clusterRoleBinding, func() error { clusterRoleBinding.RoleRef = rbacv1.RoleRef{ @@ -96,7 +96,7 @@ func (b *AutonomousBotanist) reconcileTemporaryClusterAdminBindingForBootstrappi // Finally, as the kubelet has already been started earlier such that it can run the static control plane pods, this // made it create the real kubeconfig file. We have to actively delete it in order to re-trigger its bootstrap process // w/ the bootstrap token (otherwise, it tries to use this real kubeconfig file and fails). -func (b *AutonomousBotanist) ActivateGardenerNodeAgent(ctx context.Context) error { +func (b *GardenadmBotanist) ActivateGardenerNodeAgent(ctx context.Context) error { alreadyBootstrapped, err := b.FS.Exists(nodeagentconfigv1alpha1.KubeconfigFilePath) if err != nil { return fmt.Errorf("failed checking whether gardener-node-agent's kubeconfig %s exists: %w", nodeagentconfigv1alpha1.KubeconfigFilePath, err) @@ -125,7 +125,7 @@ func (b *AutonomousBotanist) ActivateGardenerNodeAgent(ctx context.Context) erro } // ApproveNodeAgentCertificateSigningRequest approves the node agent certificate signing request. -func (b *AutonomousBotanist) ApproveNodeAgentCertificateSigningRequest(ctx context.Context) error { +func (b *GardenadmBotanist) ApproveNodeAgentCertificateSigningRequest(ctx context.Context) error { bootstrapToken, err := b.FS.ReadFile(nodeagentconfigv1alpha1.BootstrapTokenFilePath) if err != nil { if !errors.Is(err, afero.ErrFileNotFound) { @@ -178,7 +178,7 @@ func (b *AutonomousBotanist) ApproveNodeAgentCertificateSigningRequest(ctx conte // FinalizeGardenerNodeAgentBootstrapping deletes the temporary cluster-admin ClusterRoleBinding for // gardener-node-agent. -func (b *AutonomousBotanist) FinalizeGardenerNodeAgentBootstrapping(ctx context.Context) error { +func (b *GardenadmBotanist) FinalizeGardenerNodeAgentBootstrapping(ctx context.Context) error { return kubernetesutils.DeleteObject(ctx, b.SeedClientSet.Client(), emptyTemporaryClusterAdminBinding()) } @@ -194,7 +194,7 @@ var ( // WaitUntilGardenerNodeAgentLeaseIsRenewed waits until the gardener-node-agent lease is renewed, which indicates that // it is ready to be used (and that it still has the needed permissions, even though its cluster-admin binding has been // removed). -func (b *AutonomousBotanist) WaitUntilGardenerNodeAgentLeaseIsRenewed(ctx context.Context) error { +func (b *GardenadmBotanist) WaitUntilGardenerNodeAgentLeaseIsRenewed(ctx context.Context) error { node, err := nodeagent.FetchNodeByHostName(ctx, b.SeedClientSet.Client(), b.HostName) if err != nil { return fmt.Errorf("failed fetching node object by hostname %q: %w", b.HostName, err) diff --git a/pkg/gardenadm/botanist/nodeagent_test.go b/pkg/gardenadm/botanist/nodeagent_test.go index 7a9f37e9699..4f83db7d244 100644 --- a/pkg/gardenadm/botanist/nodeagent_test.go +++ b/pkg/gardenadm/botanist/nodeagent_test.go @@ -56,7 +56,7 @@ var _ = Describe("NodeAgent", func() { fakeFS afero.Afero fakeClock *testclock.FakeClock - b *AutonomousBotanist + b *GardenadmBotanist ) BeforeEach(func() { @@ -75,7 +75,7 @@ var _ = Describe("NodeAgent", func() { fakeFS = afero.Afero{Fs: afero.NewMemMapFs()} fakeClock = testclock.NewFakeClock(time.Now()) - b = &AutonomousBotanist{ + b = &GardenadmBotanist{ Botanist: &botanistpkg.Botanist{ Operation: &operation.Operation{ Logger: logr.Discard(), diff --git a/pkg/gardenadm/botanist/operatingsystemconfig.go b/pkg/gardenadm/botanist/operatingsystemconfig.go index d9d33134ac9..dff9bc4897e 100644 --- a/pkg/gardenadm/botanist/operatingsystemconfig.go +++ b/pkg/gardenadm/botanist/operatingsystemconfig.go @@ -40,7 +40,7 @@ import ( // DeployOperatingSystemConfigSecretForBootstrap deploys the OperatingSystemConfig resource and adds its content into // a Secret so that gardener-node-agent can read it and reconcile its content. -func (b *AutonomousBotanist) DeployOperatingSystemConfigSecretForBootstrap(ctx context.Context) error { +func (b *GardenadmBotanist) DeployOperatingSystemConfigSecretForBootstrap(ctx context.Context) error { if err := b.deployControlPlaneDeployments(ctx); err != nil { return fmt.Errorf("failed deploying control plane deployments: %w", err) } @@ -53,7 +53,7 @@ func (b *AutonomousBotanist) DeployOperatingSystemConfigSecretForBootstrap(ctx c return b.createOperatingSystemConfigSecretForNodeAgent(ctx, oscData.Object, oscData.GardenerNodeAgentSecretName, controlPlaneWorkerPoolName) } -func (b *AutonomousBotanist) createOperatingSystemConfigSecretForNodeAgent(ctx context.Context, osc *extensionsv1alpha1.OperatingSystemConfig, secretName, poolName string) error { +func (b *GardenadmBotanist) createOperatingSystemConfigSecretForNodeAgent(ctx context.Context, osc *extensionsv1alpha1.OperatingSystemConfig, secretName, poolName string) error { var err error b.operatingSystemConfigSecret, err = nodeagentcomponent.OperatingSystemConfigSecret(ctx, b.SeedClientSet.Client(), osc, secretName, poolName) @@ -64,7 +64,7 @@ func (b *AutonomousBotanist) createOperatingSystemConfigSecretForNodeAgent(ctx c return b.SeedClientSet.Client().Create(ctx, b.operatingSystemConfigSecret) } -func (b *AutonomousBotanist) appendAdminKubeconfigToFiles(files []extensionsv1alpha1.File) ([]extensionsv1alpha1.File, error) { +func (b *GardenadmBotanist) appendAdminKubeconfigToFiles(files []extensionsv1alpha1.File) ([]extensionsv1alpha1.File, error) { userKubeconfigSecret, ok := b.SecretsManager.Get(kubeapiserver.SecretNameUserKubeconfig) if !ok { return nil, fmt.Errorf("failed fetching secret %q", kubeapiserver.SecretNameUserKubeconfig) @@ -77,7 +77,7 @@ func (b *AutonomousBotanist) appendAdminKubeconfigToFiles(files []extensionsv1al }), nil } -func (b *AutonomousBotanist) deployOperatingSystemConfig(ctx context.Context) (*operatingsystemconfig.Data, string, error) { +func (b *GardenadmBotanist) deployOperatingSystemConfig(ctx context.Context) (*operatingsystemconfig.Data, string, error) { pods, err := b.staticControlPlanePods(ctx) if err != nil { return nil, "", fmt.Errorf("failed computing files for static control plane pods: %w", err) @@ -115,7 +115,7 @@ func (b *AutonomousBotanist) deployOperatingSystemConfig(ctx context.Context) (* // ApplyOperatingSystemConfig runs gardener-node-agent's reconciliation logic in order to apply the // OperatingSystemConfig. -func (b *AutonomousBotanist) ApplyOperatingSystemConfig(ctx context.Context) error { +func (b *GardenadmBotanist) ApplyOperatingSystemConfig(ctx context.Context) error { if b.operatingSystemConfigSecret == nil { return fmt.Errorf("operating system config secret is nil, make sure to call createOperatingSystemConfigSecretForNodeAgent() first") } @@ -152,7 +152,7 @@ func (b *AutonomousBotanist) ApplyOperatingSystemConfig(ctx context.Context) err return err } -func (b *AutonomousBotanist) ensureGardenerNodeAgentDirectories() error { +func (b *GardenadmBotanist) ensureGardenerNodeAgentDirectories() error { if err := b.FS.MkdirAll(nodeagentconfigv1alpha1.TempDir, os.ModeDir); err != nil { return fmt.Errorf("failed creating temporary directory (%q): %w", nodeagentconfigv1alpha1.TempDir, err) } @@ -164,7 +164,7 @@ func (b *AutonomousBotanist) ensureGardenerNodeAgentDirectories() error { // PrepareGardenerNodeInitConfiguration creates a Secret containing an OperatingSystemConfig with the gardener-node-init // unit. -func (b *AutonomousBotanist) PrepareGardenerNodeInitConfiguration(ctx context.Context, secretName, controlPlaneAddress string, caBundle []byte, bootstrapToken string) error { +func (b *GardenadmBotanist) PrepareGardenerNodeInitConfiguration(ctx context.Context, secretName, controlPlaneAddress string, caBundle []byte, bootstrapToken string) error { osc, err := b.generateGardenerNodeInitOperatingSystemConfig(secretName, controlPlaneAddress, bootstrapToken, caBundle) if err != nil { return fmt.Errorf("failed computing units and files for gardener-node-init: %w", err) @@ -173,7 +173,7 @@ func (b *AutonomousBotanist) PrepareGardenerNodeInitConfiguration(ctx context.Co return b.createOperatingSystemConfigSecretForNodeAgent(ctx, osc, secretName, "") } -func (b *AutonomousBotanist) generateGardenerNodeInitOperatingSystemConfig(secretName, controlPlaneAddress, bootstrapToken string, caBundle []byte) (*extensionsv1alpha1.OperatingSystemConfig, error) { +func (b *GardenadmBotanist) generateGardenerNodeInitOperatingSystemConfig(secretName, controlPlaneAddress, bootstrapToken string, caBundle []byte) (*extensionsv1alpha1.OperatingSystemConfig, error) { image, err := imagevector.Containers().FindImage(imagevector.ContainerImageNameGardenerNodeAgent) if err != nil { return nil, fmt.Errorf("failed finding image %q: %w", imagevector.ContainerImageNameGardenerNodeAgent, err) @@ -207,7 +207,7 @@ func (b *AutonomousBotanist) generateGardenerNodeInitOperatingSystemConfig(secre } // IsGardenerNodeAgentInitialized returns true if the gardener-node-agent systemd unit exists. -func (b *AutonomousBotanist) IsGardenerNodeAgentInitialized(ctx context.Context) (bool, error) { +func (b *GardenadmBotanist) IsGardenerNodeAgentInitialized(ctx context.Context) (bool, error) { unitStatuses, err := b.DBus.List(ctx) if err != nil { return false, fmt.Errorf("failed listing systemd units: %w", err) @@ -229,7 +229,7 @@ func (b *AutonomousBotanist) IsGardenerNodeAgentInitialized(ctx context.Context) // ControlPlaneBootstrapOperatingSystemConfig creates the deployer for the OperatingSystemConfig custom resource that is // used for bootstrapping control plane nodes in `gardenadm bootstrap`. -func (b *AutonomousBotanist) ControlPlaneBootstrapOperatingSystemConfig() (operatingsystemconfig.Interface, error) { +func (b *GardenadmBotanist) ControlPlaneBootstrapOperatingSystemConfig() (operatingsystemconfig.Interface, error) { image, err := imagevector.Containers().FindImage(imagevector.ContainerImageNameGardenadm) if err != nil { return nil, fmt.Errorf("failed finding image %q: %w", imagevector.ContainerImageNameGardenadm, err) diff --git a/pkg/gardenadm/botanist/operatingsystemconfig_test.go b/pkg/gardenadm/botanist/operatingsystemconfig_test.go index 94c3548c544..8564373d9c1 100644 --- a/pkg/gardenadm/botanist/operatingsystemconfig_test.go +++ b/pkg/gardenadm/botanist/operatingsystemconfig_test.go @@ -34,7 +34,7 @@ import ( var _ = Describe("OperatingSystemConfig", func() { var ( ctx = context.Background() - b *AutonomousBotanist + b *GardenadmBotanist fs afero.Afero fakeDBus *fakedbus.DBus @@ -48,7 +48,7 @@ var _ = Describe("OperatingSystemConfig", func() { fakeClient = fakeclient.NewClientBuilder().Build() clientSet = fakekubernetes.NewClientSetBuilder().WithClient(fakeClient).Build() - b = &AutonomousBotanist{ + b = &GardenadmBotanist{ FS: fs, DBus: fakeDBus, HostName: "foo-host", diff --git a/pkg/gardenadm/botanist/secrets.go b/pkg/gardenadm/botanist/secrets.go index 959d2b3829c..28bb15599de 100644 --- a/pkg/gardenadm/botanist/secrets.go +++ b/pkg/gardenadm/botanist/secrets.go @@ -16,7 +16,7 @@ import ( ) // MigrateSecrets exports the secrets generated with the fake client and imports them with the real client. -func (b *AutonomousBotanist) MigrateSecrets(ctx context.Context, fakeClient, realClient client.Client) error { +func (b *GardenadmBotanist) MigrateSecrets(ctx context.Context, fakeClient, realClient client.Client) error { secretList := &corev1.SecretList{} if err := fakeClient.List(ctx, secretList, client.InNamespace(b.Shoot.ControlPlaneNamespace)); err != nil { return fmt.Errorf("failed listing secrets with fake client: %w", err) diff --git a/pkg/gardenadm/botanist/secrets_test.go b/pkg/gardenadm/botanist/secrets_test.go index 94785709337..94a66c89932 100644 --- a/pkg/gardenadm/botanist/secrets_test.go +++ b/pkg/gardenadm/botanist/secrets_test.go @@ -30,7 +30,7 @@ var _ = Describe("Secrets", func() { fakeClient1 client.Client fakeClient2 client.Client - b *AutonomousBotanist + b *GardenadmBotanist ) BeforeEach(func() { @@ -39,7 +39,7 @@ var _ = Describe("Secrets", func() { fakeClient1 = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() fakeClient2 = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() - b = &AutonomousBotanist{ + b = &GardenadmBotanist{ Botanist: &botanistpkg.Botanist{ Operation: &operation.Operation{ SeedClientSet: fakekubernetes. diff --git a/pkg/gardenadm/botanist/ssh.go b/pkg/gardenadm/botanist/ssh.go index f9b5a51923e..7d4bf6e7a49 100644 --- a/pkg/gardenadm/botanist/ssh.go +++ b/pkg/gardenadm/botanist/ssh.go @@ -26,8 +26,8 @@ import ( ) // ConnectToControlPlaneMachine opens an SSH connection via the Bastion to the first control plane machine of the -// autonomous shoot. The connection is stored in the sshConnection field. -func (b *AutonomousBotanist) ConnectToControlPlaneMachine(ctx context.Context) error { +// self-hosted shoot. The connection is stored in the sshConnection field. +func (b *GardenadmBotanist) ConnectToControlPlaneMachine(ctx context.Context) error { machine, err := b.GetMachineByIndex(0) if err != nil { return err @@ -61,7 +61,7 @@ func (b *AutonomousBotanist) ConnectToControlPlaneMachine(ctx context.Context) e } // SSHConnection returns the SSH connection to the control plane machine opened by ConnectToControlPlaneMachine. -func (b *AutonomousBotanist) SSHConnection() *sshutils.Connection { +func (b *GardenadmBotanist) SSHConnection() *sshutils.Connection { return b.sshConnection } @@ -78,7 +78,7 @@ var ( ) // CopyManifests copies all manifests needed for `gardenadm init` to the remote machine under GardenadmBaseDir. -func (b *AutonomousBotanist) CopyManifests(ctx context.Context, configDir fs.FS) error { +func (b *GardenadmBotanist) CopyManifests(ctx context.Context, configDir fs.FS) error { if err := prepareRemoteDirs(ctx, b.sshConnection); err != nil { return err } @@ -135,7 +135,7 @@ func copyImageVectorOverride(ctx context.Context, conn *sshutils.Connection) (er return nil } -func (b *AutonomousBotanist) copyShootState(ctx context.Context) error { +func (b *GardenadmBotanist) copyShootState(ctx context.Context) error { shootState := &gardencorev1beta1.ShootState{} if err := b.GardenClient.Get(ctx, client.ObjectKeyFromObject(b.Shoot.GetInfo()), shootState); err != nil { return fmt.Errorf("error getting ShootState: %w", err) diff --git a/pkg/gardenadm/botanist/system.go b/pkg/gardenadm/botanist/system.go index 9fe160ddab7..9c337c1af51 100644 --- a/pkg/gardenadm/botanist/system.go +++ b/pkg/gardenadm/botanist/system.go @@ -16,12 +16,12 @@ import ( // DeployPriorityClassCritical deploys the gardener-system-critical PriorityClass needed for running // gardener-resource-manager. This is usually deployed to seed clusters via the gardenlet helm chart. When bootstrapping -// an autonomous shoot cluster with `gardenadm bootstrap` there is no gardenlet. Hence, we need to deploy the +// a self-hosted shoot cluster with `gardenadm bootstrap` there is no gardenlet. Hence, we need to deploy the // PriorityClass for gardener-resource-manager manually, as it is not taken over by the seedsystem component (it uses a // ManagedResource for deploying the seed PriorityClasses). // Using system-cluster-critical for gardener-resource-manager could also be good enough, but it's very simple to deploy // the gardener-system-critical PriorityClass, so we can also choose the cleaner way. -func (b *AutonomousBotanist) DeployPriorityClassCritical(ctx context.Context) error { +func (b *GardenadmBotanist) DeployPriorityClassCritical(ctx context.Context) error { priorityClass := &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: v1beta1constants.PriorityClassNameSeedSystemCritical}} _, err := controllerutils.CreateOrGetAndMergePatch(ctx, b.SeedClientSet.Client(), priorityClass, func() error { priorityClass.Value = int32(999998950) diff --git a/pkg/gardenadm/cmd/bootstrap/bootstrap.go b/pkg/gardenadm/cmd/bootstrap/bootstrap.go index a105338b362..be4c9aafd64 100644 --- a/pkg/gardenadm/cmd/bootstrap/bootstrap.go +++ b/pkg/gardenadm/cmd/bootstrap/bootstrap.go @@ -42,8 +42,8 @@ func NewCommand(globalOpts *cmd.Options) *cobra.Command { cmd := &cobra.Command{ Use: "bootstrap", - Short: "Bootstrap the infrastructure for an Autonomous Shoot Cluster", - Long: "Bootstrap the infrastructure for an Autonomous Shoot Cluster (networks, machines, etc.)", + Short: "Bootstrap the infrastructure for a Self-Hosted Shoot Cluster", + Long: "Bootstrap the infrastructure for a Self-Hosted Shoot Cluster (networks, machines, etc.)", Example: `# Bootstrap the infrastructure gardenadm bootstrap --config-dir /path/to/manifests`, @@ -84,7 +84,7 @@ func run(ctx context.Context, opts *Options) error { return err } - b, err := botanist.NewAutonomousBotanistFromManifests(ctx, opts.Log, clientSet, opts.ConfigDir, false) + b, err := botanist.NewGardenadmBotanistFromManifests(ctx, opts.Log, clientSet, opts.ConfigDir, false) if err != nil { return err } @@ -222,7 +222,7 @@ func run(ctx context.Context, opts *Options) error { }) // Scale down machine-controller-manager to prevent it from interfering with Machine objects that will be migrated - // to the autonomous shoot. Scaling down instead of deleting it, allows operators/developers to simply scale it up + // to the self-hosted shoot. Scaling down instead of deleting it, allows operators/developers to simply scale it up // again in case they need to redeploy a control plane machine manually because of errors. scaleDownMachineControllerManager = g.Add(flow.Task{ Name: "Scaling down machine-controller-manager", @@ -245,7 +245,7 @@ func run(ctx context.Context, opts *Options) error { // type "Migrate". Also, this makes it easier to allow re-running `gardenadm bootstrap` in case of failures // down the line. If we deleted the extension objects, we would need to restore them when re-running the flow. migrateExtensionResources = g.Add(flow.Task{ - Name: "Preparing extension resources for migration to autonomous shoot", + Name: "Preparing extension resources for migration to self-hosted shoot", Fn: flow.Parallel( component.MigrateAndWait(b.Shoot.Components.Extensions.Infrastructure), component.MigrateAndWait(b.Shoot.Components.Extensions.Worker), @@ -327,7 +327,7 @@ func run(ctx context.Context, opts *Options) error { fmt.Fprintf(opts.Out, ` Warning: this command is work in progress. -For now, you can connect to the autonomous Shoot cluster control-plane by +For now, you can connect to the self-hosted Shoot cluster control-plane by fetching the kubeconfig from the secret "%[1]s/kubeconfig" on the bootstrap cluster: @@ -335,7 +335,7 @@ on the bootstrap cluster: export KUBECONFIG=$PWD/%[1]s-kubeconfig.yaml kubectl get nodes -Note that the API server of the autonomous Shoot cluster control-plane might +Note that the API server of the self-hosted Shoot cluster control-plane might not be accessible from your current machine. `, b.Shoot.GetInfo().Status.TechnicalID) return nil diff --git a/pkg/gardenadm/cmd/connect/connect.go b/pkg/gardenadm/cmd/connect/connect.go index 0cce7513408..33af9defb1c 100644 --- a/pkg/gardenadm/cmd/connect/connect.go +++ b/pkg/gardenadm/cmd/connect/connect.go @@ -83,13 +83,13 @@ gardenadm connect`, func run(ctx context.Context, opts *Options) error { opts.Log.Info("Using resources from directory", "configDir", opts.ConfigDir) - b, err := botanist.NewAutonomousBotanistFromManifests(ctx, opts.Log, nil, opts.ConfigDir, true) + b, err := botanist.NewGardenadmBotanistFromManifests(ctx, opts.Log, nil, opts.ConfigDir, true) if err != nil { - return fmt.Errorf("failed creating autonomous botanist: %w", err) + return fmt.Errorf("failed creating gardenadm botanist: %w", err) } b.SeedClientSet, err = b.CreateClientSet(ctx) if err != nil { - return fmt.Errorf("failed creating client set for autonomous shoot: %w", err) + return fmt.Errorf("failed creating client set for self-hosted shoot: %w", err) } if alreadyConnected, err := isGardenletDeployed(ctx, b); err != nil { @@ -118,7 +118,7 @@ func run(ctx context.Context, opts *Options) error { Dependencies: flow.NewTaskIDs(retrieveShortLivedKubeconfig), }) deployGardenlet = g.Add(flow.Task{ - Name: "Deploying gardenlet into autonomous shoot cluster", + Name: "Deploying gardenlet into self-hosted shoot cluster", Fn: func(ctx context.Context) error { _, err := newGardenletDeployer(b, bootstrapClientSet).Reconcile( ctx, @@ -153,9 +153,9 @@ func run(ctx context.Context, opts *Options) error { } fmt.Fprintf(opts.Out, ` -Your autonomous shoot cluster has successfully been connected to Gardener! +Your self-hosted shoot cluster has successfully been connected to Gardener! -The gardenlet has been deployed in the %s namespace of your autonomous shoot +The gardenlet has been deployed in the %s namespace of your self-hosted shoot cluster and is now taking over the management and lifecycle of it. All modifications to the Shoot specification should now be performed via the Gardener API, rather than by directly editing resources in the cluster. @@ -178,7 +178,7 @@ Happy Gardening! return nil } -func isGardenletDeployed(ctx context.Context, b *botanist.AutonomousBotanist) (bool, error) { +func isGardenletDeployed(ctx context.Context, b *botanist.GardenadmBotanist) (bool, error) { if err := b.SeedClientSet.Client().Get(ctx, client.ObjectKey{Namespace: b.Shoot.ControlPlaneNamespace, Name: v1beta1constants.DeploymentNameGardenlet}, &appsv1.Deployment{}); err != nil { if !apierrors.IsNotFound(err) { return false, fmt.Errorf("failed checking if gardenlet deployment already exists: %w", err) @@ -200,7 +200,7 @@ func cachedBootstrapKubeconfigPath(fs afero.Afero) string { return filepath.Join(fs.GetTempDir(""), "gardenadm-connect-bootstrap-kubeconfig") } -func initializeTemporaryGardenClient(ctx context.Context, b *botanist.AutonomousBotanist, bootstrapClientSet kubernetes.Interface) error { +func initializeTemporaryGardenClient(ctx context.Context, b *botanist.GardenadmBotanist, bootstrapClientSet kubernetes.Interface) error { bootstrapKubeconfig, cached, err := getCachedBootstrapKubeconfig(b) if err != nil { return fmt.Errorf("failed retrieving cached bootstrap kubeconfig: %w", err) @@ -220,7 +220,7 @@ func initializeTemporaryGardenClient(ctx context.Context, b *botanist.Autonomous return setGardenClientFromKubeconfig(b, bootstrapKubeconfig) } -func getCachedBootstrapKubeconfig(b *botanist.AutonomousBotanist) ([]byte, bool, error) { +func getCachedBootstrapKubeconfig(b *botanist.GardenadmBotanist) ([]byte, bool, error) { fileInfo, err := b.FS.Stat(cachedBootstrapKubeconfigPath(b.FS)) if err != nil || time.Since(fileInfo.ModTime()) > bootstrapKubeconfigValidity-2*time.Minute { // We deliberately ignore the error here - this is just a best-effort attempt to cache the bootstrap kubeconfig. @@ -239,7 +239,7 @@ func getCachedBootstrapKubeconfig(b *botanist.AutonomousBotanist) ([]byte, bool, return data, true, nil } -func requestShortLivedBootstrapKubeconfig(ctx context.Context, b *botanist.AutonomousBotanist, bootstrapClientSet kubernetes.Interface) ([]byte, error) { +func requestShortLivedBootstrapKubeconfig(ctx context.Context, b *botanist.GardenadmBotanist, bootstrapClientSet kubernetes.Interface) ([]byte, error) { certificateSubject := &pkix.Name{ Organization: []string{v1beta1constants.ShootsGroup}, CommonName: v1beta1constants.GardenadmUserNamePrefix + b.Shoot.GetInfo().Namespace + ":" + b.Shoot.GetInfo().Name, @@ -253,7 +253,7 @@ func requestShortLivedBootstrapKubeconfig(ctx context.Context, b *botanist.Auton return gardenletbootstraputil.CreateKubeconfigWithClientCertificate(bootstrapClientSet.RESTConfig(), privateKeyData, certData) } -func setGardenClientFromKubeconfig(b *botanist.AutonomousBotanist, kubeconfig []byte) error { +func setGardenClientFromKubeconfig(b *botanist.GardenadmBotanist, kubeconfig []byte) error { gardenClientSet, err := kubernetes.NewClientFromBytes( kubeconfig, kubernetes.WithClientOptions(client.Options{Scheme: kubernetes.GardenScheme}), @@ -268,7 +268,7 @@ func setGardenClientFromKubeconfig(b *botanist.AutonomousBotanist, kubeconfig [] return nil } -func prepareGardenerResources(ctx context.Context, b *botanist.AutonomousBotanist) error { +func prepareGardenerResources(ctx context.Context, b *botanist.GardenadmBotanist) error { if err := b.GardenClient.Get(ctx, client.ObjectKeyFromObject(b.Resources.CloudProfile), &gardencorev1beta1.CloudProfile{}); err != nil { return fmt.Errorf("failed checking for existence of CloudProfile %s (this is not created by 'gardenadm connect' and must exist in the garden cluster): %w", b.Resources.CloudProfile.Name, err) } @@ -290,7 +290,7 @@ func prepareGardenerResources(ctx context.Context, b *botanist.AutonomousBotanis // We do not handle Project using 'garden' namespace because gardener-apiserver defaults .spec.tolerations for this // Project. This requires special permissions for a custom verb that we do not want to grant to the gardenadm user - // for autonomous shoots. Since this is a special project anyway, it must have been created beforehand. + // for self-hosted shoots. Since this is a special project anyway, it must have been created beforehand. if project := b.Resources.Project.DeepCopy(); ptr.Deref(project.Spec.Namespace, "") != v1beta1constants.GardenNamespace { if err := b.GardenClient.Create(ctx, project); client.IgnoreAlreadyExists(err) != nil { return fmt.Errorf("failed creating Project resource %s in garden cluster: %w", project.Name, err) @@ -334,7 +334,7 @@ func prepareGardenerResources(ctx context.Context, b *botanist.AutonomousBotanis return nil } -func newGardenletDeployer(b *botanist.AutonomousBotanist, gardenClientSet kubernetes.Interface) gardenletdeployer.Interface { +func newGardenletDeployer(b *botanist.GardenadmBotanist, gardenClientSet kubernetes.Interface) gardenletdeployer.Interface { return &gardenletdeployer.Actuator{ GardenConfig: gardenClientSet.RESTConfig(), GardenClient: gardenClientSet.Client(), diff --git a/pkg/gardenadm/cmd/connect/options.go b/pkg/gardenadm/cmd/connect/options.go index 7325e54e5ce..91d14b15b86 100644 --- a/pkg/gardenadm/cmd/connect/options.go +++ b/pkg/gardenadm/cmd/connect/options.go @@ -19,7 +19,7 @@ type Options struct { *cmd.Options cmd.ManifestOptions - // ControlPlaneAddress is the address of the Gardener control plane to which the autonomous shoot should be connected. + // ControlPlaneAddress is the address of the Gardener control plane to which the self-hosted shoot should be connected. ControlPlaneAddress string // BootstrapToken is the bootstrap token to use for connecting the shoot. BootstrapToken string @@ -64,6 +64,6 @@ func (o *Options) Complete() error { return o.ManifestOptions.Complete() } func (o *Options) addFlags(fs *pflag.FlagSet) { o.ManifestOptions.AddFlags(fs) fs.BytesBase64Var(&o.CertificateAuthority, "ca-certificate", nil, "Base64-encoded certificate authority bundle of the Gardener control plane") - fs.StringVar(&o.BootstrapToken, "bootstrap-token", "", "Bootstrap token for connecting the autonomous shoot cluster to a garden cluster (create it with 'gardenadm token' in the garden cluster)") + fs.StringVar(&o.BootstrapToken, "bootstrap-token", "", "Bootstrap token for connecting the self-hosted shoot cluster to a garden cluster (create it with 'gardenadm token' in the garden cluster)") fs.BoolVar(&o.Force, "force", false, "Forces the deployment of gardenlet, even if it already exists") } diff --git a/pkg/gardenadm/cmd/init/init.go b/pkg/gardenadm/cmd/init/init.go index 37083e4ccdb..fb5950fb03f 100644 --- a/pkg/gardenadm/cmd/init/init.go +++ b/pkg/gardenadm/cmd/init/init.go @@ -337,7 +337,7 @@ cluster. Note that the mentioned kubeconfig file will be disabled once you deploy the gardenlet and connect this cluster to an existing Gardener installation. Run this while targeting the garden cluster to which you want to connect this -autonomous shoot cluster: +self-hosted shoot cluster: gardenadm token create --print-connect-command --shoot-namespace=%s --shoot-name=%s @@ -351,8 +351,8 @@ see https://gardener.cloud/docs/gardener/shoot/shoot_access/. return nil } -func bootstrapControlPlane(ctx context.Context, opts *Options) (*botanist.AutonomousBotanist, error) { - b, err := botanist.NewAutonomousBotanistFromManifests(ctx, opts.Log, nil, opts.ConfigDir, true) +func bootstrapControlPlane(ctx context.Context, opts *Options) (*botanist.GardenadmBotanist, error) { + b, err := botanist.NewGardenadmBotanistFromManifests(ctx, opts.Log, nil, opts.ConfigDir, true) if err != nil { return nil, err } @@ -417,5 +417,5 @@ func bootstrapControlPlane(ctx context.Context, opts *Options) (*botanist.Autono return nil, flow.Errors(err) } - return botanist.NewAutonomousBotanistFromManifests(ctx, opts.Log, clientSet, opts.ConfigDir, true) + return botanist.NewGardenadmBotanistFromManifests(ctx, opts.Log, clientSet, opts.ConfigDir, true) } diff --git a/pkg/gardenadm/cmd/join/join.go b/pkg/gardenadm/cmd/join/join.go index 31e8781aa3b..369304b3e8b 100644 --- a/pkg/gardenadm/cmd/join/join.go +++ b/pkg/gardenadm/cmd/join/join.go @@ -25,7 +25,7 @@ func NewCommand(globalOpts *cmd.Options) *cobra.Command { Short: "Bootstrap worker nodes and join them to the cluster", Long: `Bootstrap worker nodes and join them to the cluster. -This command helps to initialize and configure a node to join an existing autonomous shoot cluster. +This command helps to initialize and configure a node to join an existing self-hosted shoot cluster. It ensures that the necessary configurations are applied and the node is properly registered as a worker or control plane node. Note that further control plane nodes cannot be joined currently.`, @@ -57,9 +57,9 @@ gardenadm join --bootstrap-token --ca-certificate --gardener-n } func run(ctx context.Context, opts *Options) error { - b, err := botanist.NewAutonomousBotanistWithoutResources(opts.Log) + b, err := botanist.NewGardenadmBotanistWithoutResources(opts.Log) if err != nil { - return fmt.Errorf("failed creating autonomous botanist: %w", err) + return fmt.Errorf("failed creating gardenadm botanist: %w", err) } version, err := b.DiscoverKubernetesVersion(opts.ControlPlaneAddress, opts.CertificateAuthority, opts.BootstrapToken) diff --git a/pkg/gardenadm/cmd/token/create/options.go b/pkg/gardenadm/cmd/token/create/options.go index 62af1517b0f..1b54a087abc 100644 --- a/pkg/gardenadm/cmd/token/create/options.go +++ b/pkg/gardenadm/cmd/token/create/options.go @@ -31,7 +31,7 @@ type Options struct { // 'worker'. WorkerPoolName string // Shoot contains the namespace and the name of the Shoot which should be connected to Gardener. This is only - // relevant for bootstrap tokens that are used for connecting autonomous shoots via `gardenadm connect`. + // relevant for bootstrap tokens that are used for connecting self-hosted shoots via `gardenadm connect`. Shoot types.NamespacedName // PrintJoinCommand specifies whether to print the full `gardenadm join` command. @@ -114,9 +114,9 @@ func (o *Options) Complete() error { o.Token.ID, o.Token.Secret = split[0], split[1] if o.Shoot.Namespace != "" && o.Shoot.Name != "" { - o.Description = fmt.Sprintf("%s%s to Gardener via 'gardenadm connect'", bootstraptoken.AutonomousShootBootstrapTokenSecretDescriptionPrefix, o.Shoot.String()) + o.Description = fmt.Sprintf("%s%s to Gardener via 'gardenadm connect'", bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix, o.Shoot.String()) } else if len(o.Description) == 0 { - o.Description = "Used for joining nodes to an autonomous shoot cluster via 'gardenadm join'" + o.Description = "Used for joining nodes to a self-hosted shoot cluster via 'gardenadm join'" } return nil diff --git a/pkg/gardenadm/cmd/token/create/options_test.go b/pkg/gardenadm/cmd/token/create/options_test.go index e07fb156b0d..7ae1cbacdaa 100644 --- a/pkg/gardenadm/cmd/token/create/options_test.go +++ b/pkg/gardenadm/cmd/token/create/options_test.go @@ -125,14 +125,14 @@ var _ = Describe("Options", func() { options.Shoot.Namespace = "bar" Expect(options.Complete()).To(Succeed()) - Expect(options.Description).To(Equal("Used for connecting the autonomous Shoot bar/foo to Gardener via 'gardenadm connect'")) + Expect(options.Description).To(Equal("Used for connecting the self-hosted Shoot bar/foo to Gardener via 'gardenadm connect'")) }) It("should default the description for 'gardenadm join' when no shoot info is provided", func() { options.Token.Combined = "foo.bar" Expect(options.Complete()).To(Succeed()) - Expect(options.Description).To(Equal("Used for joining nodes to an autonomous shoot cluster via 'gardenadm join'")) + Expect(options.Description).To(Equal("Used for joining nodes to a self-hosted shoot cluster via 'gardenadm join'")) }) }) }) diff --git a/pkg/gardenadm/cmd/token/generate/generate.go b/pkg/gardenadm/cmd/token/generate/generate.go index 89539ae8b39..dfe5e5759ea 100644 --- a/pkg/gardenadm/cmd/token/generate/generate.go +++ b/pkg/gardenadm/cmd/token/generate/generate.go @@ -23,7 +23,7 @@ func NewCommand(globalOpts *cmd.Options) *cobra.Command { cmd := &cobra.Command{ Use: "generate", Short: "Generate a random bootstrap token for joining a node", - Long: `Generate a random bootstrap token that can be used for joining a node to an autonomous shoot cluster. + Long: `Generate a random bootstrap token that can be used for joining a node to a self-hosted shoot cluster. Note that the token is not created on the server (use 'gardenadm token create' for it). The token is securely generated and follows the format "[a-z0-9]{6}.[a-z0-9]{16}". Read more about it here: https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/`, diff --git a/pkg/gardenadm/cmd/token/utils/clientset.go b/pkg/gardenadm/cmd/token/utils/clientset.go index 323a2ef145d..9a244a5df94 100644 --- a/pkg/gardenadm/cmd/token/utils/clientset.go +++ b/pkg/gardenadm/cmd/token/utils/clientset.go @@ -15,8 +15,8 @@ import ( botanistpkg "github.com/gardener/gardener/pkg/gardenlet/operation/botanist" ) -// CreateClientSet creates a new client set using the AutonomousBotanist to create the client set. +// CreateClientSet creates a new client set using the GardenadmBotanist to create the client set. // Exposed for unit testing. var CreateClientSet = func(ctx context.Context, log logr.Logger) (kubernetes.Interface, error) { - return (&botanist.AutonomousBotanist{Botanist: &botanistpkg.Botanist{Operation: &operation.Operation{Logger: log}}}).CreateClientSet(ctx) + return (&botanist.GardenadmBotanist{Botanist: &botanistpkg.Botanist{Operation: &operation.Operation{Logger: log}}}).CreateClientSet(ctx) } diff --git a/pkg/gardenlet/bootstrap/bootstrap.go b/pkg/gardenlet/bootstrap/bootstrap.go index e5a15b3ee06..1e06b83cfff 100644 --- a/pkg/gardenlet/bootstrap/bootstrap.go +++ b/pkg/gardenlet/bootstrap/bootstrap.go @@ -33,7 +33,7 @@ import ( const ( // SeedCSRPrefix defines the prefix of seed CSR created by gardenlet. SeedCSRPrefix = "seed-csr-" - // ShootCSRPrefix defines the prefix of autonomous shoot CSR created by gardenlet. + // ShootCSRPrefix defines the prefix of self-hosted shoot CSR created by gardenlet. ShootCSRPrefix = "shoot-csr-" ) @@ -46,7 +46,7 @@ func RequestKubeconfigWithBootstrapClient( bootstrapClientSet kubernetes.Interface, kubeconfigKey, bootstrapKubeconfigKey client.ObjectKey, seedConfig *gardenletconfigv1alpha1.SeedConfig, - autonomousShootMeta *types.NamespacedName, + selfHostedShootMeta *types.NamespacedName, validityDuration *metav1.Duration, ) ( []byte, @@ -69,17 +69,17 @@ func RequestKubeconfigWithBootstrapClient( CommonName: v1beta1constants.SeedUserNamePrefix + seedName, } - case gardenlet.IsResponsibleForAutonomousShoot(): - log = log.WithValues("shoot", autonomousShootMeta) + case gardenlet.IsResponsibleForSelfHostedShoot(): + log = log.WithValues("shoot", selfHostedShootMeta) csrPrefix = ShootCSRPrefix certificateSubject = &pkix.Name{ Organization: []string{v1beta1constants.ShootsGroup}, - CommonName: v1beta1constants.ShootUserNamePrefix + autonomousShootMeta.Namespace + ":" + autonomousShootMeta.Name, + CommonName: v1beta1constants.ShootUserNamePrefix + selfHostedShootMeta.Namespace + ":" + selfHostedShootMeta.Name, } default: - return nil, "", fmt.Errorf("failed determining gardenlet bootstrap scenario (seed or autonomous shoot)") + return nil, "", fmt.Errorf("failed determining gardenlet bootstrap scenario (seed or self-hosted shoot)") } certData, privateKeyData, csrName, err := certificatesigningrequest.RequestCertificate(ctx, log, bootstrapClientSet.Kubernetes(), certificateSubject, []string{}, []net.IP{}, validityDuration, csrPrefix) diff --git a/pkg/gardenlet/bootstrap/bootstrap_test.go b/pkg/gardenlet/bootstrap/bootstrap_test.go index a8bb0982db2..f80ebb9e916 100644 --- a/pkg/gardenlet/bootstrap/bootstrap_test.go +++ b/pkg/gardenlet/bootstrap/bootstrap_test.go @@ -74,7 +74,7 @@ var _ = Describe("Bootstrap", func() { gardenClientConnection *gardenletconfigv1alpha1.GardenClientConnection kubeconfigKey client.ObjectKey bootstrapKubeconfigKey client.ObjectKey - autonomousShootMeta *types.NamespacedName + selfHostedShootMeta *types.NamespacedName approvedCSR = certificatesv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ @@ -194,7 +194,7 @@ var _ = Describe("Bootstrap", func() { }, }) - kubeconfig, csrName, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, seedConfig, autonomousShootMeta, nil) + kubeconfig, csrName, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, seedConfig, selfHostedShootMeta, nil) Expect(err).NotTo(HaveOccurred()) Expect(kubeconfig).ToNot(BeEmpty()) @@ -215,7 +215,7 @@ var _ = Describe("Bootstrap", func() { WithKubernetes(kubeClient). Build() - _, _, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, seedConfig, autonomousShootMeta, nil) + _, _, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, seedConfig, selfHostedShootMeta, nil) Expect(err).To(MatchError(ContainSubstring("is denied"))) }) @@ -233,7 +233,7 @@ var _ = Describe("Bootstrap", func() { WithKubernetes(kubeClient). Build() - _, _, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, seedConfig, autonomousShootMeta, nil) + _, _, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, seedConfig, selfHostedShootMeta, nil) Expect(err).To(MatchError(ContainSubstring("failed"))) }) }) @@ -243,7 +243,7 @@ var _ = Describe("Bootstrap", func() { Expect(os.Setenv("NAMESPACE", "kube-system")).To(Succeed()) DeferCleanup(func() { Expect(os.Setenv("NAMESPACE", "")).To(Succeed()) }) - autonomousShootMeta = &types.NamespacedName{Namespace: "shoot-namespace", Name: "shoot-name"} + selfHostedShootMeta = &types.NamespacedName{Namespace: "shoot-namespace", Name: "shoot-name"} }) It("should not return an error", func() { @@ -277,7 +277,7 @@ var _ = Describe("Bootstrap", func() { }, }) - kubeconfig, csrName, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, nil, autonomousShootMeta, nil) + kubeconfig, csrName, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, nil, selfHostedShootMeta, nil) Expect(err).NotTo(HaveOccurred()) Expect(kubeconfig).ToNot(BeEmpty()) @@ -298,7 +298,7 @@ var _ = Describe("Bootstrap", func() { WithKubernetes(kubeClient). Build() - _, _, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, nil, autonomousShootMeta, nil) + _, _, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, nil, selfHostedShootMeta, nil) Expect(err).To(MatchError(ContainSubstring("is denied"))) }) @@ -316,7 +316,7 @@ var _ = Describe("Bootstrap", func() { WithKubernetes(kubeClient). Build() - _, _, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, nil, autonomousShootMeta, nil) + _, _, err := RequestKubeconfigWithBootstrapClient(ctx, testLogger, runtimeClient, bootstrapClientSet, kubeconfigKey, bootstrapKubeconfigKey, nil, selfHostedShootMeta, nil) Expect(err).To(MatchError(ContainSubstring("failed"))) }) }) diff --git a/pkg/gardenlet/bootstrap/certificate/certificate_rotation.go b/pkg/gardenlet/bootstrap/certificate/certificate_rotation.go index 71d11cb20c7..bd2e25749bd 100644 --- a/pkg/gardenlet/bootstrap/certificate/certificate_rotation.go +++ b/pkg/gardenlet/bootstrap/certificate/certificate_rotation.go @@ -56,7 +56,7 @@ func NewCertificateManager( gardenCluster cluster.Cluster, seedClient client.Client, config *gardenletconfigv1alpha1.GardenletConfiguration, - autonomousShootMeta *types.NamespacedName, + selfHostedShootMeta *types.NamespacedName, ) ( *Manager, error, @@ -72,8 +72,8 @@ func NewCertificateManager( } logger := log.WithName("certificate-manager") - if autonomousShootMeta != nil { - logger = logger.WithValues("shootNamespace", autonomousShootMeta.Namespace, "shootName", autonomousShootMeta.Name) + if selfHostedShootMeta != nil { + logger = logger.WithValues("shootNamespace", selfHostedShootMeta.Namespace, "shootName", selfHostedShootMeta.Name) } else { logger = logger.WithValues("seedName", gardenletbootstraputil.GetSeedName(config.SeedConfig)) } @@ -84,8 +84,8 @@ func NewCertificateManager( seedClient: seedClient, gardenClientConnection: config.GardenClientConnection, newTargetedObject: func() client.Object { - if autonomousShootMeta != nil { - return &gardencorev1beta1.Shoot{ObjectMeta: metav1.ObjectMeta{Namespace: autonomousShootMeta.Namespace, Name: autonomousShootMeta.Name}} + if selfHostedShootMeta != nil { + return &gardencorev1beta1.Shoot{ObjectMeta: metav1.ObjectMeta{Namespace: selfHostedShootMeta.Namespace, Name: selfHostedShootMeta.Name}} } return &gardencorev1beta1.Seed{ObjectMeta: metav1.ObjectMeta{Name: gardenletbootstraputil.GetSeedName(config.SeedConfig)}} }, @@ -130,7 +130,7 @@ func (cr *Manager) ScheduleCertificateRotation(ctx context.Context, gardenletCan return nil } -// getTargetedObject returns the Seed or the autonomous Shoot that this Gardenlet is reconciling. +// getTargetedObject returns the Seed or the self-hosted Shoot that this Gardenlet is reconciling. func (cr *Manager) getTargetedObject(ctx context.Context) (client.Object, error) { obj := cr.newTargetedObject() if err := cr.gardenClientSet.Client().Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { diff --git a/pkg/gardenlet/bootstrappers/garden_kubeconfig.go b/pkg/gardenlet/bootstrappers/garden_kubeconfig.go index 5c9b86d3018..ffaaf2f2eb5 100644 --- a/pkg/gardenlet/bootstrappers/garden_kubeconfig.go +++ b/pkg/gardenlet/bootstrappers/garden_kubeconfig.go @@ -22,15 +22,15 @@ import ( // GardenKubeconfig implements manager.Runnable and can be used to fetch a kubeconfig for the garden cluster. type GardenKubeconfig struct { - // RuntimeClient is the runtime cluster client (seed or autonomous shoot). + // RuntimeClient is the runtime cluster client (seed or self-hosted shoot). RuntimeClient client.Client // Log is a logger. Log logr.Logger // Config is the gardenlet component configuration. Config *gardenletconfigv1alpha1.GardenletConfiguration - // AutonomousShootMeta is the NamespacedName of the autonomous shoot the gardenlet is responsible for (if + // SelfHostedShootMeta is the NamespacedName of the self-hosted shoot the gardenlet is responsible for (if // applicable). - AutonomousShootMeta *types.NamespacedName + SelfHostedShootMeta *types.NamespacedName // Result is a structure that will be filled with information about the requested kubeconfig. Must be initialized // by the caller. Result *KubeconfigBootstrapResult @@ -147,7 +147,7 @@ func (g *GardenKubeconfig) getOrBootstrapKubeconfig(ctx context.Context) ([]byte kubeconfigKey, bootstrapKubeconfigKey, g.Config.SeedConfig, - g.AutonomousShootMeta, + g.SelfHostedShootMeta, g.Config.GardenClientConnection.KubeconfigValidity.Validity, ) } diff --git a/pkg/gardenlet/controller/add.go b/pkg/gardenlet/controller/add.go index bd2b4c3a496..be3da54bb35 100644 --- a/pkg/gardenlet/controller/add.go +++ b/pkg/gardenlet/controller/add.go @@ -57,7 +57,7 @@ func AddToManager( // adapt the shoot authorizer (via the resource dependency graph) to allow 'read' access to this ConfigMap (similar // to how it's done for seeds). gardenClusterIdentity := "" - if !gardenletutils.IsResponsibleForAutonomousShoot() { + if !gardenletutils.IsResponsibleForSelfHostedShoot() { configMap := &corev1.ConfigMap{} if err := gardenCluster.GetClient().Get(ctx, client.ObjectKey{Namespace: metav1.NamespaceSystem, Name: v1beta1constants.ClusterIdentity}, configMap); err != nil { return fmt.Errorf("failed getting cluster-identity ConfigMap in garden cluster: %w", err) @@ -79,8 +79,8 @@ func AddToManager( return fmt.Errorf("failed creating seed clientset: %w", err) } - if gardenletutils.IsResponsibleForAutonomousShoot() { - mgr.GetLogger().Info("Running in autonomous shoot, registering minimal set of controllers") + if gardenletutils.IsResponsibleForSelfHostedShoot() { + mgr.GetLogger().Info("Running in self-hosted shoot, registering minimal set of controllers") if err := (&gardenlet.Reconciler{ Config: *cfg, diff --git a/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/reconciler.go b/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/reconciler.go index 064e471d7b8..10ff0d4886e 100644 --- a/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/reconciler.go +++ b/pkg/gardenlet/controller/controllerinstallation/controllerinstallation/reconciler.go @@ -62,7 +62,7 @@ const usablePortsRangeSize = 5 // the process of being deleted when deleting a ControllerInstallation. var RequeueDurationWhenResourceDeletionStillPresent = 5 * time.Second -// Reconciler reconciles ControllerInstallations and deploys them into the seed cluster or the autonomous shoot cluster. +// Reconciler reconciles ControllerInstallations and deploys them into the seed cluster or the self-hosted shoot cluster. type Reconciler struct { GardenClient client.Client GardenConfig *rest.Config @@ -80,7 +80,7 @@ type Reconciler struct { BootstrapControlPlaneNode bool } -// Reconcile reconciles ControllerInstallations and deploys them into the seed cluster or the autonomous shoot cluster. +// Reconcile reconciles ControllerInstallations and deploys them into the seed cluster or the self-hosted shoot cluster. func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) @@ -262,8 +262,8 @@ func (r *Reconciler) reconcile( }, } - if metav1.HasLabel(seed.ObjectMeta, v1beta1constants.LabelAutonomousShootCluster) { - gardenerValues["gardener"].(map[string]any)["autonomousShootCluster"] = true + if metav1.HasLabel(seed.ObjectMeta, v1beta1constants.LabelSelfHostedShootCluster) { + gardenerValues["gardener"].(map[string]any)["selfHostedShootCluster"] = true } if genericGardenKubeconfigSecretName != "" { @@ -582,7 +582,7 @@ func objectEnablesGardenKubeconfig(o runtime.Object) bool { } // MutateSpecForControlPlaneNodeBootstrapping adapts host network, replicas, tolerations and usable ports range for -// autonomous shoot clusters if necessary. +// self-hosted shoot clusters if necessary. func (r *Reconciler) MutateSpecForControlPlaneNodeBootstrapping(obj runtime.Object) error { if !r.BootstrapControlPlaneNode { return nil diff --git a/pkg/gardenlet/controller/gardenlet/add.go b/pkg/gardenlet/controller/gardenlet/add.go index dc787185490..b25450af351 100644 --- a/pkg/gardenlet/controller/gardenlet/add.go +++ b/pkg/gardenlet/controller/gardenlet/add.go @@ -50,7 +50,7 @@ func (r *Reconciler) AddToManager( r.Clock = clock.RealClock{} } if r.GardenNamespace == "" { - if gardenlet.IsResponsibleForAutonomousShoot() { + if gardenlet.IsResponsibleForSelfHostedShoot() { r.GardenNamespace = metav1.NamespaceSystem } else { r.GardenNamespace = v1beta1constants.GardenNamespace diff --git a/pkg/gardenlet/controller/gardenlet/reconciler.go b/pkg/gardenlet/controller/gardenlet/reconciler.go index 80deacb694f..095f741e230 100644 --- a/pkg/gardenlet/controller/gardenlet/reconciler.go +++ b/pkg/gardenlet/controller/gardenlet/reconciler.go @@ -74,7 +74,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( } var seed *gardencorev1beta1.Seed - if !strings.HasPrefix(gardenlet.Name, gardenletutils.ResourcePrefixAutonomousShoot) { + if !strings.HasPrefix(gardenlet.Name, gardenletutils.ResourcePrefixSelfHostedShoot) { seed, err = gardenletdeployer.GetSeed(ctx, r.GardenClient, gardenlet.Name) if err != nil { r.Recorder.Eventf(gardenlet, corev1.EventTypeWarning, gardencorev1beta1.EventReconcileError, err.Error()) diff --git a/pkg/gardenlet/operation/botanist/machinecontrollermanager.go b/pkg/gardenlet/operation/botanist/machinecontrollermanager.go index cf6074531ad..6a6d3f82c7a 100644 --- a/pkg/gardenlet/operation/botanist/machinecontrollermanager.go +++ b/pkg/gardenlet/operation/botanist/machinecontrollermanager.go @@ -31,7 +31,7 @@ func (b *Botanist) DefaultMachineControllerManager() (machinecontrollermanager.I b.SecretsManager, machinecontrollermanager.Values{ Image: image.String(), - AutonomousShoot: b.Shoot.IsAutonomous(), + SelfHostedShoot: b.Shoot.IsSelfHosted(), }, ), nil } diff --git a/pkg/gardenlet/operation/botanist/resource_manager.go b/pkg/gardenlet/operation/botanist/resource_manager.go index 7521df47f16..05e4ca7a5aa 100644 --- a/pkg/gardenlet/operation/botanist/resource_manager.go +++ b/pkg/gardenlet/operation/botanist/resource_manager.go @@ -58,7 +58,7 @@ func (b *Botanist) DefaultResourceManager() (resourcemanager.Interface, error) { } ) - if b.Shoot.IsAutonomous() { + if b.Shoot.IsSelfHosted() { values.KubernetesServiceHost = nil if b.Shoot.RunsControlPlane() { diff --git a/pkg/gardenlet/operation/botanist/secrets.go b/pkg/gardenlet/operation/botanist/secrets.go index 925fa53702d..0f9ef1045da 100644 --- a/pkg/gardenlet/operation/botanist/secrets.go +++ b/pkg/gardenlet/operation/botanist/secrets.go @@ -74,7 +74,7 @@ func (b *Botanist) lastSecretRotationStartTimes() map[string]time.Time { if shootStatus := b.Shoot.GetInfo().Status; shootStatus.Credentials != nil && shootStatus.Credentials.Rotation != nil { if shootStatus.Credentials.Rotation.CertificateAuthorities != nil && shootStatus.Credentials.Rotation.CertificateAuthorities.LastInitiationTime != nil { - for _, config := range caCertConfigurations(b.Shoot.IsWorkerless, b.Shoot.IsAutonomous()) { + for _, config := range caCertConfigurations(b.Shoot.IsWorkerless, b.Shoot.IsSelfHosted()) { rotation[config.GetName()] = shootStatus.Credentials.Rotation.CertificateAuthorities.LastInitiationTime.Time } // The static token secret contains token for the health check of the kube-apiserver. @@ -148,7 +148,7 @@ func (b *Botanist) restoreSecretsFromShootState(ctx context.Context) error { return flow.Parallel(fns...)(ctx) } -func caCertConfigurations(isWorkerless, isAutonomous bool) []secretsutils.ConfigInterface { +func caCertConfigurations(isWorkerless, isSelfHosted bool) []secretsutils.ConfigInterface { certificateSecretConfigs := []secretsutils.ConfigInterface{ // The CommonNames for CA certificates will be overridden with the secret name by the secrets manager when // generated to ensure that each CA has a unique common name. For backwards-compatibility, we still keep the @@ -167,7 +167,7 @@ func caCertConfigurations(isWorkerless, isAutonomous bool) []secretsutils.Config &secretsutils.CertificateSecretConfig{Name: v1beta1constants.SecretNameCAMetricsServer, CommonName: "metrics-server", CertType: secretsutils.CACert}, ) - if !isAutonomous { + if !isSelfHosted { certificateSecretConfigs = append(certificateSecretConfigs, &secretsutils.CertificateSecretConfig{Name: v1beta1constants.SecretNameCAVPN, CommonName: "vpn", CertType: secretsutils.CACert}, ) @@ -209,7 +209,7 @@ func (b *Botanist) caCertGenerateOptionsFor(configName string) []secretsmanager. func (b *Botanist) generateCertificateAuthorities(ctx context.Context) error { var caClientSecret *corev1.Secret - for _, config := range caCertConfigurations(b.Shoot.IsWorkerless, b.Shoot.IsAutonomous()) { + for _, config := range caCertConfigurations(b.Shoot.IsWorkerless, b.Shoot.IsSelfHosted()) { caSecret, err := b.SecretsManager.Generate(ctx, config, b.caCertGenerateOptionsFor(config.GetName())...) if err != nil { return err diff --git a/pkg/gardenlet/operation/botanist/worker.go b/pkg/gardenlet/operation/botanist/worker.go index 9e22cc3b6fd..2b3302233a4 100644 --- a/pkg/gardenlet/operation/botanist/worker.go +++ b/pkg/gardenlet/operation/botanist/worker.go @@ -30,8 +30,8 @@ import ( func (b *Botanist) DefaultWorker() worker.Interface { workers := b.Shoot.GetInfo().Spec.Provider.Workers // In `gardenadm bootstrap` we only deploy the control plane worker pool. When running `gardenadm init` on the - // medium-touch control plane, the full `Worker` with all pools will be deployed. - if b.Shoot.IsAutonomous() && !b.Shoot.RunsControlPlane() { + // created control plane nodes, the full `Worker` with all pools will be deployed. + if b.Shoot.IsSelfHosted() && !b.Shoot.RunsControlPlane() { workers = []gardencorev1beta1.Worker{*v1beta1helper.ControlPlaneWorkerPoolForShoot(b.Shoot.GetInfo().DeepCopy().Spec.Provider.Workers)} } diff --git a/pkg/gardenlet/operation/shoot/shoot.go b/pkg/gardenlet/operation/shoot/shoot.go index a6e7379d620..1be74c87d84 100644 --- a/pkg/gardenlet/operation/shoot/shoot.go +++ b/pkg/gardenlet/operation/shoot/shoot.go @@ -663,20 +663,19 @@ func copyUniqueCIDRs(src []string, dst []net.IPNet, networkType string) ([]net.I return dst, nil } -// IsAutonomous returns true in case of an autonomous shoot cluster. -func (s *Shoot) IsAutonomous() bool { - return v1beta1helper.IsShootAutonomous(s.GetInfo().Spec.Provider.Workers) +// IsSelfHosted returns true in case of a self-hosted shoot cluster. +func (s *Shoot) IsSelfHosted() bool { + return v1beta1helper.IsShootSelfHosted(s.GetInfo().Spec.Provider.Workers) } // RunsControlPlane returns true in case the Kubernetes control plane runs inside the cluster. -// In contrast to IsAutonomous, this function returns false when bootstrapping autonomous shoot clusters using -// `gardenadm bootstrap` (medium-touch scenario). +// In contrast to IsSelfHosted, this function returns false when bootstrapping self-hosted shoot clusters using +// `gardenadm bootstrap` ("managed infrastructure" scenario). func (s *Shoot) RunsControlPlane() bool { return s.ControlPlaneNamespace == metav1.NamespaceSystem } // HasManagedInfrastructure returns true if the shoot's infrastructure (network, machines, etc.) is managed by Gardener. -// I.e., it returns false for high-touch autonomous shoots, where the infrastructure is managed by the user. func (s *Shoot) HasManagedInfrastructure() bool { return v1beta1helper.HasManagedInfrastructure(s.GetInfo()) } diff --git a/pkg/gardenlet/operation/shoot/shoot_test.go b/pkg/gardenlet/operation/shoot/shoot_test.go index 6086bbf100e..0410186f018 100644 --- a/pkg/gardenlet/operation/shoot/shoot_test.go +++ b/pkg/gardenlet/operation/shoot/shoot_test.go @@ -191,7 +191,7 @@ var _ = Describe("shoot", func() { }) Describe("#ComputeInClusterAPIServerAddress", func() { - When("shoot is not autonomous", func() { + When("shoot is not self-hosted", func() { controlPlaneNamespace := "foo" s := &Shoot{ControlPlaneNamespace: controlPlaneNamespace} @@ -204,7 +204,7 @@ var _ = Describe("shoot", func() { }) }) - When("shoot is autonomous", func() { + When("shoot is self-hosted", func() { s := &Shoot{ControlPlaneNamespace: "kube-system"} It("should return kubernetes.default.svc", func() { @@ -265,7 +265,7 @@ var _ = Describe("shoot", func() { }) }) - Describe("#IsAutonomous", func() { + Describe("#IsSelfHosted", func() { It("should return true", func() { shoot.SetInfo(&gardencorev1beta1.Shoot{ Spec: gardencorev1beta1.ShootSpec{ @@ -277,7 +277,7 @@ var _ = Describe("shoot", func() { }, }, }) - Expect(shoot.IsAutonomous()).To(BeTrue()) + Expect(shoot.IsSelfHosted()).To(BeTrue()) }) It("should return false", func() { @@ -290,7 +290,7 @@ var _ = Describe("shoot", func() { }, }, }) - Expect(shoot.IsAutonomous()).To(BeFalse()) + Expect(shoot.IsSelfHosted()).To(BeFalse()) }) }) diff --git a/pkg/gardenlet/operation/shoot/types.go b/pkg/gardenlet/operation/shoot/types.go index 8bc7c39e490..1853e15202c 100644 --- a/pkg/gardenlet/operation/shoot/types.go +++ b/pkg/gardenlet/operation/shoot/types.go @@ -79,7 +79,7 @@ type Shoot struct { ControlPlaneNamespace string KubernetesVersion *semver.Version - // InternalClusterDomain is empty for autonomous shoots, which only have an external domain (Shoot.spec.dns.domain). + // InternalClusterDomain is empty for self-hosted shoots, which only have an external domain (Shoot.spec.dns.domain). InternalClusterDomain *string // ExternalClusterDomain is nil if Shoot.Spec.DNS.Domain is unset. ExternalClusterDomain *string diff --git a/pkg/operator/controller/garden/garden/components.go b/pkg/operator/controller/garden/garden/components.go index 7aa5cee5806..605870a8363 100644 --- a/pkg/operator/controller/garden/garden/components.go +++ b/pkg/operator/controller/garden/garden/components.go @@ -718,7 +718,7 @@ func (r *Reconciler) newKubeAPIServer( ), newAuthorizationWebhook("shoot", kubeconfigShootAuthz, // only intercept request from shoot gardenlets - // TODO(rfranzke): Also handle requests from ServiceAccounts of extensions running in the autonomous shoot. + // TODO(rfranzke): Also handle requests from ServiceAccounts of extensions running in the self-hosted shoot. fmt.Sprintf("'%s' in request.groups", v1beta1constants.ShootsGroup), ), ) diff --git a/pkg/operator/controller/gardenlet/add.go b/pkg/operator/controller/gardenlet/add.go index c03be1c5c20..6e5d97af564 100644 --- a/pkg/operator/controller/gardenlet/add.go +++ b/pkg/operator/controller/gardenlet/add.go @@ -88,7 +88,7 @@ func (r *Reconciler) OperatorResponsiblePredicate(ctx context.Context) predicate if !ok { return false } - return !strings.HasPrefix(gardenlet.Name, gardenletutils.ResourcePrefixAutonomousShoot) && + return !strings.HasPrefix(gardenlet.Name, gardenletutils.ResourcePrefixSelfHostedShoot) && (hasForceRedeployOperationAnnotation(gardenlet) || r.seedDoesNotExist(ctx, gardenlet) || gardenlet.Spec.KubeconfigSecretRef != nil) diff --git a/pkg/operator/controller/gardenlet/add_test.go b/pkg/operator/controller/gardenlet/add_test.go index d5c6cda1159..b69934f3078 100644 --- a/pkg/operator/controller/gardenlet/add_test.go +++ b/pkg/operator/controller/gardenlet/add_test.go @@ -50,8 +50,8 @@ var _ = Describe("Add", func() { test(gardenlet, BeTrue()) }) - It("should return false because gardenlet name is prefixed with 'autonomous-shoot'", func() { - gardenlet.Name = "autonomous-shoot-" + gardenlet.Name + It("should return false because gardenlet name is prefixed with 'self-hosted-shoot'", func() { + gardenlet.Name = "self-hosted-shoot-" + gardenlet.Name test(gardenlet, BeFalse()) }) diff --git a/pkg/provider-local/controller/dnsrecord/actuator.go b/pkg/provider-local/controller/dnsrecord/actuator.go index 55ad746b3d5..905d40f3207 100644 --- a/pkg/provider-local/controller/dnsrecord/actuator.go +++ b/pkg/provider-local/controller/dnsrecord/actuator.go @@ -69,10 +69,10 @@ func (a *Actuator) ForceDelete(ctx context.Context, log logr.Logger, dnsRecord * return a.Delete(ctx, log, dnsRecord, cluster) } -// Migrate removes the DNS record from the CoreDNS config map if the shoot is not autonomous. +// Migrate removes the DNS record from the CoreDNS config map if the shoot is not self-hosted. func (a *Actuator) Migrate(ctx context.Context, log logr.Logger, dnsRecord *extensionsv1alpha1.DNSRecord, cluster *extensionscontroller.Cluster) error { - if v1beta1helper.IsShootAutonomous(cluster.Shoot.Spec.Provider.Workers) { - // Do nothing when migrating DNSRecord of autonomous shoot with managed infrastructure. The CoreDNS + if v1beta1helper.IsShootSelfHosted(cluster.Shoot.Spec.Provider.Workers) { + // Do nothing when migrating DNSRecord of self-hosted shoot with managed infrastructure. The CoreDNS // rewrite rules are still needed for the control plane machines to resolve the kube-apiserver domain. return nil } @@ -99,7 +99,7 @@ func keyForDNSRecord(dnsRecord *extensionsv1alpha1.DNSRecord) string { } func (a *Actuator) configForDNSRecord(ctx context.Context, dnsRecord *extensionsv1alpha1.DNSRecord, cluster *extensionscontroller.Cluster) (string, error) { - if v1beta1helper.IsShootAutonomous(cluster.Shoot.Spec.Provider.Workers) { + if v1beta1helper.IsShootSelfHosted(cluster.Shoot.Spec.Provider.Workers) { switch dnsRecord.Spec.RecordType { case extensionsv1alpha1.DNSRecordTypeA, extensionsv1alpha1.DNSRecordTypeAAAA: // We need to use the `template` plugin because the `hosts` plugin can only be used once per server block. @@ -112,7 +112,7 @@ func (a *Actuator) configForDNSRecord(ctx context.Context, dnsRecord *extensions return config, nil default: - return "", fmt.Errorf("unsupported record type %q for autonomous shoot, only A and AAAA are supported", dnsRecord.Spec.RecordType) + return "", fmt.Errorf("unsupported record type %q for self-hosted shoot, only A and AAAA are supported", dnsRecord.Spec.RecordType) } } diff --git a/pkg/provider-local/controller/dnsrecord/actuator_test.go b/pkg/provider-local/controller/dnsrecord/actuator_test.go index ac3c94d4ba3..fb6edac999f 100644 --- a/pkg/provider-local/controller/dnsrecord/actuator_test.go +++ b/pkg/provider-local/controller/dnsrecord/actuator_test.go @@ -199,7 +199,7 @@ var _ = Describe("Actuator", func() { }) }) - Context("autonomous shoots", func() { + Context("self-hosted shoots", func() { BeforeEach(func() { cluster.Shoot.Spec.Provider.Workers = []gardencorev1beta1.Worker{{ ControlPlane: &gardencorev1beta1.WorkerControlPlane{}, @@ -246,7 +246,7 @@ var _ = Describe("Actuator", func() { apiDNSRecord.Spec.RecordType = "CNAME" apiDNSRecord.Spec.Values = []string{"some.other.name.gardener.cloud"} - Expect(actuator.Reconcile(ctx, log, apiDNSRecord, cluster)).To(MatchError(ContainSubstring(`unsupported record type "CNAME" for autonomous shoot`))) + Expect(actuator.Reconcile(ctx, log, apiDNSRecord, cluster)).To(MatchError(ContainSubstring(`unsupported record type "CNAME" for self-hosted shoot`))) }) }) diff --git a/pkg/provider-local/controller/worker/add.go b/pkg/provider-local/controller/worker/add.go index a660681de23..9f81604a59d 100644 --- a/pkg/provider-local/controller/worker/add.go +++ b/pkg/provider-local/controller/worker/add.go @@ -33,8 +33,8 @@ type AddOptions struct { IgnoreOperationAnnotation bool // ExtensionClass defines the extension class this extension is responsible for. ExtensionClass extensionsv1alpha1.ExtensionClass - // AutonomousShootCluster indicates whether the extension runs in an autonomous shoot cluster. - AutonomousShootCluster bool + // SelfHostedShootCluster indicates whether the extension runs in a self-hosted shoot cluster. + SelfHostedShootCluster bool } // AddToManagerWithOptions adds a controller with the given Options to the given manager. @@ -54,7 +54,7 @@ func AddToManagerWithOptions(ctx context.Context, mgr manager.Manager, opts AddO Predicates: worker.DefaultPredicates(ctx, mgr, opts.IgnoreOperationAnnotation), Type: local.Type, ExtensionClass: opts.ExtensionClass, - AutonomousShootCluster: opts.AutonomousShootCluster, + SelfHostedShootCluster: opts.SelfHostedShootCluster, }) } diff --git a/pkg/provider-local/webhook/dnsconfig/add.go b/pkg/provider-local/webhook/dnsconfig/add.go index 76f0f4f920a..70abd371a59 100644 --- a/pkg/provider-local/webhook/dnsconfig/add.go +++ b/pkg/provider-local/webhook/dnsconfig/add.go @@ -59,9 +59,9 @@ func AddToManagerWithOptions(mgr manager.Manager, _ AddOptions) (*extensionswebh Target: extensionswebhook.TargetSeed, Path: name, Webhook: &admission.Webhook{Handler: handler, RecoverPanic: ptr.To(true)}, - // This webhook should not act on the local setup for the gardenadm high-touch scenario. + // This webhook should not act on the local setup for the gardenadm "unmanaged infrastructure" scenario. NamespaceSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ - {Key: corev1.LabelMetadataName, Operator: metav1.LabelSelectorOpNotIn, Values: []string{"gardenadm-high-touch"}}, + {Key: corev1.LabelMetadataName, Operator: metav1.LabelSelectorOpNotIn, Values: []string{"gardenadm-unmanaged-infra"}}, }}, ObjectSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ {Key: "app", Operator: metav1.LabelSelectorOpIn, Values: []string{ diff --git a/pkg/scheduler/controller/shoot/add.go b/pkg/scheduler/controller/shoot/add.go index 39e17b64d4c..f8e039e2bb8 100644 --- a/pkg/scheduler/controller/shoot/add.go +++ b/pkg/scheduler/controller/shoot/add.go @@ -42,7 +42,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { For(&gardencorev1beta1.Shoot{}, builder.WithPredicates( r.ShootUnassignedPredicate(), r.ShootSpecChangedPredicate(), - r.ShootIsNotAutonomous(), + r.ShootIsNotSelfHosted(), predicate.Not(predicateutils.IsDeleting()), )). WithOptions(controller.Options{ @@ -82,12 +82,12 @@ func (r *Reconciler) ShootSpecChangedPredicate() predicate.Predicate { } } -// ShootIsNotAutonomous is a predicate that returns true if a shoot is not mark as 'autonomous shoot cluster' (meaning, +// ShootIsNotSelfHosted is a predicate that returns true if a shoot is not mark as 'self-hosted shoot cluster' (meaning, // that the control plane components run in the same cluster, hence, no seed selection is required). -func (r *Reconciler) ShootIsNotAutonomous() predicate.Predicate { +func (r *Reconciler) ShootIsNotSelfHosted() predicate.Predicate { return predicate.NewPredicateFuncs(func(obj client.Object) bool { if shoot, ok := obj.(*gardencorev1beta1.Shoot); ok { - return !helper.IsShootAutonomous(shoot.Spec.Provider.Workers) + return !helper.IsShootSelfHosted(shoot.Spec.Provider.Workers) } return false }) diff --git a/pkg/scheduler/controller/shoot/add_test.go b/pkg/scheduler/controller/shoot/add_test.go index 7421db4da54..75c1f97727f 100644 --- a/pkg/scheduler/controller/shoot/add_test.go +++ b/pkg/scheduler/controller/shoot/add_test.go @@ -158,7 +158,7 @@ var _ = Describe("Add", func() { }) }) - Describe("#ShootIsNotAutonomous", func() { + Describe("#ShootIsNotSelfHosted", func() { var ( predicate predicate.Predicate shoot *gardencorev1beta1.Shoot @@ -170,7 +170,7 @@ var _ = Describe("Add", func() { ) BeforeEach(func() { - predicate = reconciler.ShootIsNotAutonomous() + predicate = reconciler.ShootIsNotSelfHosted() shoot = &gardencorev1beta1.Shoot{} createEvent = event.CreateEvent{Object: shoot} @@ -179,7 +179,7 @@ var _ = Describe("Add", func() { genericEvent = event.GenericEvent{Object: shoot} }) - Context("shoot is autonomous", func() { + Context("shoot is self-hosted", func() { It("should be false", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, gardencorev1beta1.Worker{ControlPlane: &gardencorev1beta1.WorkerControlPlane{}}) Expect(predicate.Create(createEvent)).To(BeFalse()) @@ -189,7 +189,7 @@ var _ = Describe("Add", func() { }) }) - Context("shoot is not autonomous", func() { + Context("shoot is not self-hosted", func() { It("should be true", func() { Expect(predicate.Create(createEvent)).To(BeTrue()) Expect(predicate.Update(updateEvent)).To(BeTrue()) diff --git a/pkg/utils/gardener/csr.go b/pkg/utils/gardener/csr.go index 15d01dccb57..f20d0e592c9 100644 --- a/pkg/utils/gardener/csr.go +++ b/pkg/utils/gardener/csr.go @@ -39,20 +39,20 @@ func IsSeedClientCert(x509cr *x509.CertificateRequest, usages []certificatesv1.K } // IsGardenadmClientCert returns true when the given CSR and usages match the requirements for a client -// certificate for an autonomous shoot with the `gardenadm connect` prefix. If false is returned, a reason will be +// certificate for a self-hosted shoot with the `gardenadm connect` prefix. If false is returned, a reason will be // returned explaining which requirement was not met. func IsGardenadmClientCert(x509cr *x509.CertificateRequest, usages []certificatesv1.KeyUsage) (bool, string) { - return isAutonomousShootClientCert(x509cr, usages, v1beta1constants.GardenadmUserNamePrefix) + return isSelfHostedShootClientCert(x509cr, usages, v1beta1constants.GardenadmUserNamePrefix) } // IsShootClientCert returns true when the given CSR and usages match the requirements for a client certificate for an -// autonomous shoot with the gardenlet prefix. If false is returned, a reason will be returned explaining which +// self-hosted shoot with the gardenlet prefix. If false is returned, a reason will be returned explaining which // requirement was not met. func IsShootClientCert(x509cr *x509.CertificateRequest, usages []certificatesv1.KeyUsage) (bool, string) { - return isAutonomousShootClientCert(x509cr, usages, v1beta1constants.ShootUserNamePrefix) + return isSelfHostedShootClientCert(x509cr, usages, v1beta1constants.ShootUserNamePrefix) } -func isAutonomousShootClientCert(x509cr *x509.CertificateRequest, usages []certificatesv1.KeyUsage, prefix string) (bool, string) { +func isSelfHostedShootClientCert(x509cr *x509.CertificateRequest, usages []certificatesv1.KeyUsage, prefix string) (bool, string) { requiredOrganizations := []string{v1beta1constants.ShootsGroup} if !reflect.DeepEqual(requiredOrganizations, x509cr.Subject.Organization) { diff --git a/pkg/utils/gardener/gardenlet/gardenlet.go b/pkg/utils/gardener/gardenlet/gardenlet.go index 39bfeb9d39d..ee8b5724534 100644 --- a/pkg/utils/gardener/gardenlet/gardenlet.go +++ b/pkg/utils/gardener/gardenlet/gardenlet.go @@ -67,12 +67,12 @@ func SetDefaultGardenClusterAddress(log logr.Logger, gardenletConfigRaw runtime. return *newGardenletConfigRaw, nil } -// ResourcePrefixAutonomousShoot is the prefix for resources related to Gardenlet created for autonomous shoots. -const ResourcePrefixAutonomousShoot = "autonomous-shoot-" +// ResourcePrefixSelfHostedShoot is the prefix for resources related to Gardenlet created for self-hosted shoots. +const ResourcePrefixSelfHostedShoot = "self-hosted-shoot-" -// IsResponsibleForAutonomousShoot checks if the current process is responsible for managing autonomous shoots. This is +// IsResponsibleForSelfHostedShoot checks if the current process is responsible for managing self-hosted shoots. This is // determined by checking if the environment variable "NAMESPACE" is set to the kube-system namespace. -func IsResponsibleForAutonomousShoot() bool { +func IsResponsibleForSelfHostedShoot() bool { return os.Getenv("NAMESPACE") == metav1.NamespaceSystem } @@ -90,11 +90,11 @@ func ShootMetaFromBootstrapToken(ctx context.Context, reader client.Reader, boot func extractShootMetaFromBootstrapToken(bootstrapTokenSecret *corev1.Secret) (types.NamespacedName, error) { description := string(bootstrapTokenSecret.Data[bootstraptokenapi.BootstrapTokenDescriptionKey]) - if !strings.HasPrefix(description, bootstraptoken.AutonomousShootBootstrapTokenSecretDescriptionPrefix) { - return types.NamespacedName{}, fmt.Errorf("bootstrap token description does not start with %q: %s", bootstraptoken.AutonomousShootBootstrapTokenSecretDescriptionPrefix, description) + if !strings.HasPrefix(description, bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix) { + return types.NamespacedName{}, fmt.Errorf("bootstrap token description does not start with %q: %s", bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix, description) } - parts := strings.Fields(strings.TrimPrefix(description, bootstraptoken.AutonomousShootBootstrapTokenSecretDescriptionPrefix)) + parts := strings.Fields(strings.TrimPrefix(description, bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix)) if len(parts) == 0 { return types.NamespacedName{}, fmt.Errorf("could not extract shoot meta from bootstrap token description: %s", description) } diff --git a/pkg/utils/gardener/gardenlet/gardenlet_test.go b/pkg/utils/gardener/gardenlet/gardenlet_test.go index 1a943b013cf..f1bcafbc671 100644 --- a/pkg/utils/gardener/gardenlet/gardenlet_test.go +++ b/pkg/utils/gardener/gardenlet/gardenlet_test.go @@ -124,16 +124,16 @@ var _ = Describe("Gardenlet", func() { }) }) - Describe("#IsResponsibleForAutonomousShoot", func() { + Describe("#IsResponsibleForSelfHostedShoot", func() { It("should return true because the NAMESPACE environment variable is set to 'kube-system'", func() { Expect(os.Setenv("NAMESPACE", "kube-system")).To(Succeed()) DeferCleanup(func() { Expect(os.Setenv("NAMESPACE", "")).To(Succeed()) }) - Expect(IsResponsibleForAutonomousShoot()).To(BeTrue()) + Expect(IsResponsibleForSelfHostedShoot()).To(BeTrue()) }) It("should return false because the NAMESPACE environment variable is not set to 'kube-system'", func() { - Expect(IsResponsibleForAutonomousShoot()).To(BeFalse()) + Expect(IsResponsibleForSelfHostedShoot()).To(BeFalse()) }) }) @@ -167,7 +167,7 @@ var _ = Describe("Gardenlet", func() { Namespace: "kube-system", }, Data: map[string][]byte{ - "description": []byte("Used for connecting the autonomous Shoot " + expectedShootNamespace + "/" + expectedShootName + " to Gardener via 'gardenadm connect'"), + "description": []byte("Used for connecting the self-hosted Shoot " + expectedShootNamespace + "/" + expectedShootName + " to Gardener via 'gardenadm connect'"), }, } @@ -211,7 +211,7 @@ var _ = Describe("Gardenlet", func() { Namespace: "kube-system", }, Data: map[string][]byte{ - "description": []byte("Used for connecting the autonomous Shoot "), + "description": []byte("Used for connecting the self-hosted Shoot "), }, } @@ -230,7 +230,7 @@ var _ = Describe("Gardenlet", func() { Namespace: "kube-system", }, Data: map[string][]byte{ - "description": []byte("Used for connecting the autonomous Shoot "), + "description": []byte("Used for connecting the self-hosted Shoot "), }, } @@ -249,7 +249,7 @@ var _ = Describe("Gardenlet", func() { Namespace: "kube-system", }, Data: map[string][]byte{ - "description": []byte("Used for connecting the autonomous Shoot invalid-format-no-slash"), + "description": []byte("Used for connecting the self-hosted Shoot invalid-format-no-slash"), }, } @@ -268,7 +268,7 @@ var _ = Describe("Gardenlet", func() { Namespace: "kube-system", }, Data: map[string][]byte{ - "description": []byte("Used for connecting the autonomous Shoot namespace/shoot/extra"), + "description": []byte("Used for connecting the self-hosted Shoot namespace/shoot/extra"), }, } @@ -287,7 +287,7 @@ var _ = Describe("Gardenlet", func() { Namespace: "kube-system", }, Data: map[string][]byte{ - "description": []byte("Used for connecting the autonomous Shoot /my-shoot"), + "description": []byte("Used for connecting the self-hosted Shoot /my-shoot"), }, } @@ -305,7 +305,7 @@ var _ = Describe("Gardenlet", func() { Namespace: "kube-system", }, Data: map[string][]byte{ - "description": []byte("Used for connecting the autonomous Shoot my-namespace/"), + "description": []byte("Used for connecting the self-hosted Shoot my-namespace/"), }, } @@ -323,7 +323,7 @@ var _ = Describe("Gardenlet", func() { Namespace: "kube-system", }, Data: map[string][]byte{ - "description": []byte("Used for connecting the autonomous Shoot " + expectedShootNamespace + "/" + expectedShootName + " additional text here"), + "description": []byte("Used for connecting the self-hosted Shoot " + expectedShootNamespace + "/" + expectedShootName + " additional text here"), }, } diff --git a/pkg/utils/gardener/shoot.go b/pkg/utils/gardener/shoot.go index 76f33c8efa4..9ef5d5e9e6b 100644 --- a/pkg/utils/gardener/shoot.go +++ b/pkg/utils/gardener/shoot.go @@ -622,9 +622,9 @@ func ConstructExternalDomain(ctx context.Context, c client.Reader, shoot *garden ) switch { - case v1beta1helper.IsShootAutonomous(shoot.Spec.Provider.Workers) && !v1beta1helper.HasManagedInfrastructure(shoot): - // For high-touch autonomous shoots, the external domain is managed outside of Gardener, but must be specified - // in the Shoot resource. Therefore, we do not have to set any secret/zone data here. + case v1beta1helper.IsShootSelfHosted(shoot.Spec.Provider.Workers) && !v1beta1helper.HasManagedInfrastructure(shoot): + // For self-hosted shoots w/o managed infrastructure, the external domain is managed outside of Gardener, but + // must be specified in the Shoot resource. Therefore, we do not have to set any secret/zone data here. // When removing the "unmanaged provider" (https://github.com/gardener/gardener/issues/12212), replace this value // with a bool field on the Domain type that is considered by the NeedsExternalDNS func. externalDomain.Provider = core.DNSUnmanaged diff --git a/pkg/utils/gardener/shoot_test.go b/pkg/utils/gardener/shoot_test.go index 653702965ce..929dfbf3b82 100644 --- a/pkg/utils/gardener/shoot_test.go +++ b/pkg/utils/gardener/shoot_test.go @@ -1070,7 +1070,7 @@ var _ = Describe("Shoot", func() { Expect(err).NotTo(HaveOccurred()) }) - It("returns the unmanaged external domain for autonomous shoots", func(ctx SpecContext) { + It("returns the unmanaged external domain for self-hosted shoots", func(ctx SpecContext) { shoot := &gardencorev1beta1.Shoot{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -1446,7 +1446,7 @@ var _ = Describe("Shoot", func() { ))) }) - It("should compute the correct list of required extensions (autonomous shoot with backup)", func() { + It("should compute the correct list of required extensions (self-hosted shoot with backup)", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, gardencorev1beta1.Worker{ ControlPlane: &gardencorev1beta1.WorkerControlPlane{Backup: &gardencorev1beta1.Backup{Provider: backupProvider}}, }) diff --git a/pkg/utils/graph/eventhandler_gardenlet.go b/pkg/utils/graph/eventhandler_gardenlet.go index 917df4d2a11..987f761199d 100644 --- a/pkg/utils/graph/eventhandler_gardenlet.go +++ b/pkg/utils/graph/eventhandler_gardenlet.go @@ -64,7 +64,7 @@ func (g *graph) handleGardenletCreateOrUpdate(ctx context.Context, gardenlet *se g.lock.Lock() defer g.lock.Unlock() - if g.forAutonomousShoots { + if g.forSelfHostedShoots { g.handleGardenletCreateOrUpdateForShoots(gardenlet) } else { g.handleGardenletCreateOrUpdateForSeeds(ctx, gardenlet) @@ -83,7 +83,7 @@ func (g *graph) handleGardenletDelete(name, namespace string) { } func (g *graph) handleGardenletCreateOrUpdateForSeeds(ctx context.Context, gardenlet *seedmanagementv1alpha1.Gardenlet) { - if strings.HasPrefix(gardenlet.Name, gardenletutils.ResourcePrefixAutonomousShoot) { + if strings.HasPrefix(gardenlet.Name, gardenletutils.ResourcePrefixSelfHostedShoot) { return } @@ -141,7 +141,7 @@ func (g *graph) handleGardenletCreateOrUpdateForSeeds(ctx context.Context, garde } func (g *graph) handleGardenletCreateOrUpdateForShoots(gardenlet *seedmanagementv1alpha1.Gardenlet) { - if !strings.HasPrefix(gardenlet.Name, gardenletutils.ResourcePrefixAutonomousShoot) { + if !strings.HasPrefix(gardenlet.Name, gardenletutils.ResourcePrefixSelfHostedShoot) { return } @@ -150,11 +150,11 @@ func (g *graph) handleGardenletCreateOrUpdateForShoots(gardenlet *seedmanagement var ( gardenletVertex = g.getOrCreateVertex(VertexTypeGardenlet, gardenlet.Namespace, gardenlet.Name) - shootVertex = g.getOrCreateVertex(VertexTypeShoot, gardenlet.Namespace, strings.TrimPrefix(gardenlet.Name, gardenletutils.ResourcePrefixAutonomousShoot)) + shootVertex = g.getOrCreateVertex(VertexTypeShoot, gardenlet.Namespace, strings.TrimPrefix(gardenlet.Name, gardenletutils.ResourcePrefixSelfHostedShoot)) ) g.addEdge(gardenletVertex, shootVertex) - // TODO(rfranzke): Check if we need to support the 'allowBootstrap' logic for autonomous shoots as well (see + // TODO(rfranzke): Check if we need to support the 'allowBootstrap' logic for self-hosted shoots as well (see // handling for seeds). } diff --git a/pkg/utils/graph/eventhandler_shoot.go b/pkg/utils/graph/eventhandler_shoot.go index 175a66d1df2..a102356bb84 100644 --- a/pkg/utils/graph/eventhandler_shoot.go +++ b/pkg/utils/graph/eventhandler_shoot.go @@ -106,7 +106,7 @@ func (g *graph) HandleShootCreateOrUpdate(ctx context.Context, shoot *gardencore g.deleteAllIncomingEdges(VertexTypeSecretBinding, VertexTypeShoot, shoot.Namespace, shoot.Name) g.deleteAllIncomingEdges(VertexTypeCredentialsBinding, VertexTypeShoot, shoot.Namespace, shoot.Name) g.deleteAllIncomingEdges(VertexTypeShootState, VertexTypeShoot, shoot.Namespace, shoot.Name) - if !g.forAutonomousShoots { + if !g.forSelfHostedShoots { g.deleteAllOutgoingEdges(VertexTypeShoot, shoot.Namespace, shoot.Name, VertexTypeSeed) } diff --git a/pkg/utils/graph/graph.go b/pkg/utils/graph/graph.go index 78bf566139f..4f33d7ee26c 100644 --- a/pkg/utils/graph/graph.go +++ b/pkg/utils/graph/graph.go @@ -45,19 +45,19 @@ type graph struct { client client.Client graph *simple.DirectedGraph vertices typeVertexMapping - forAutonomousShoots bool + forSelfHostedShoots bool } var _ Interface = &graph{} // New creates a new graph interface for tracking resource dependencies. -func New(logger logr.Logger, client client.Client, forAutonomousShoots bool) *graph { +func New(logger logr.Logger, client client.Client, forSelfHostedShoots bool) *graph { return &graph{ logger: logger, client: client, graph: simple.NewDirectedGraph(), vertices: make(typeVertexMapping), - forAutonomousShoots: forAutonomousShoots, + forSelfHostedShoots: forSelfHostedShoots, } } @@ -69,7 +69,7 @@ type resourceSetup struct { func (g *graph) Setup(ctx context.Context, c cache.Cache) error { var setups []resourceSetup - if g.forAutonomousShoots { + if g.forSelfHostedShoots { setups = append(setups, resourceSetup{&certificatesv1.CertificateSigningRequest{}, g.setupCertificateSigningRequestWatch}, resourceSetup{&seedmanagementv1alpha1.Gardenlet{}, g.setupGardenletWatch}, diff --git a/pkg/utils/graph/graph_test.go b/pkg/utils/graph/graph_test.go index 59d04ba0795..3a928c11416 100644 --- a/pkg/utils/graph/graph_test.go +++ b/pkg/utils/graph/graph_test.go @@ -2580,7 +2580,7 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d } gardenlet1 = &seedmanagementv1alpha1.Gardenlet{ - ObjectMeta: metav1.ObjectMeta{Name: "autonomous-shoot-" + shootName, Namespace: shootNamespace}, + ObjectMeta: metav1.ObjectMeta{Name: "self-hosted-shoot-" + shootName, Namespace: shootNamespace}, } namespace1 = &corev1.Namespace{ diff --git a/pkg/utils/kubernetes/bootstraptoken/bootstraptoken.go b/pkg/utils/kubernetes/bootstraptoken/bootstraptoken.go index 367d7b90c03..273496860b6 100644 --- a/pkg/utils/kubernetes/bootstraptoken/bootstraptoken.go +++ b/pkg/utils/kubernetes/bootstraptoken/bootstraptoken.go @@ -98,6 +98,6 @@ func TokenID(meta metav1.ObjectMeta) string { return utils.ComputeSHA256Hex([]byte(value))[:IDLength] } -// AutonomousShootBootstrapTokenSecretDescriptionPrefix is the prefix used in the description of a bootstrap token -// secret for autonomous shoots. It indicates that the token is used for connecting the autonomous Shoot to Gardener. -const AutonomousShootBootstrapTokenSecretDescriptionPrefix = "Used for connecting the autonomous Shoot " +// SelfHostedShootBootstrapTokenSecretDescriptionPrefix is the prefix used in the description of a bootstrap token +// secret for self-hosted shoots. It indicates that the token is used for connecting the self-hosted Shoot to Gardener. +const SelfHostedShootBootstrapTokenSecretDescriptionPrefix = "Used for connecting the self-hosted Shoot " diff --git a/plugin/pkg/global/customverbauthorizer/admission.go b/plugin/pkg/global/customverbauthorizer/admission.go index f1fe12d255b..13325f63aa2 100644 --- a/plugin/pkg/global/customverbauthorizer/admission.go +++ b/plugin/pkg/global/customverbauthorizer/admission.go @@ -52,9 +52,9 @@ const ( // `.spec.limits` limits in `NamespacedCloudProfile` resources above values defined in the parent `CloudProfile`. CustomVerbNamespacedCloudProfileRaiseLimits = "raise-spec-limits" - // CustomVerbShootMarkAutonomous is a constant for the custom verb that allows setting the - // `.spec.provider.workers[].controlPlane` field in the `Shoot` spec which marks it as 'autonomous shoot cluster'. - CustomVerbShootMarkAutonomous = "mark-autonomous" + // CustomVerbShootMarkSelfHosted is a constant for the custom verb that allows setting the + // `.spec.provider.workers[].controlPlane` field in the `Shoot` spec which marks it as 'self-hosted shoot cluster'. + CustomVerbShootMarkSelfHosted = "mark-self-hosted" ) // Register registers a plugin. @@ -257,7 +257,7 @@ func (c *CustomVerbAuthorizer) admitShoots(ctx context.Context, a admission.Attr } if mustCheckShootAutonomy(oldObj, obj) { - return c.authorize(ctx, a, CustomVerbShootMarkAutonomous, "modify .spec.provider.workers[].controlPlane") + return c.authorize(ctx, a, CustomVerbShootMarkSelfHosted, "modify .spec.provider.workers[].controlPlane") } return nil diff --git a/plugin/pkg/global/customverbauthorizer/admission_test.go b/plugin/pkg/global/customverbauthorizer/admission_test.go index f32ff2dab18..b298f189dfc 100644 --- a/plugin/pkg/global/customverbauthorizer/admission_test.go +++ b/plugin/pkg/global/customverbauthorizer/admission_test.go @@ -857,9 +857,9 @@ var _ = Describe("customverbauthorizer", func() { Expect(err).NotTo(HaveOccurred()) }) - Context("mark autonomous", func() { + Context("mark self-hosted", func() { BeforeEach(func() { - authorizeAttributes.Verb = "mark-autonomous" + authorizeAttributes.Verb = "mark-self-hosted" }) It("should always allow creating a shoot without whitelist tolerations", func() { @@ -872,14 +872,14 @@ var _ = Describe("customverbauthorizer", func() { auth.EXPECT().Authorize(ctx, authorizeAttributes).Return(authorizer.DecisionAllow, "", nil) }) - It("should allow creating an autonomous shoot", func() { + It("should allow creating a self-hosted shoot", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, core.Worker{ControlPlane: &core.WorkerControlPlane{}}) attrs = admission.NewAttributesRecord(shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) - It("should allow marking an existing shoot as 'autonomous'", func() { + It("should allow marking an existing shoot as 'self-hosted'", func() { // NB: This is already forbidden by validation, but this admission plugin does not know about it, so let's test it for completeness. shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, core.Worker{}) oldShoot := shoot.DeepCopy() @@ -889,7 +889,7 @@ var _ = Describe("customverbauthorizer", func() { Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) - It("should allow removing the control plane field of autonomous shoots", func() { + It("should allow removing the control plane field of self-hosted shoots", func() { // NB: This is already forbidden by validation, but this admission plugin does not know about it, so let's test it for completeness. shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, core.Worker{ControlPlane: &core.WorkerControlPlane{}}) oldShoot := shoot.DeepCopy() @@ -905,14 +905,14 @@ var _ = Describe("customverbauthorizer", func() { auth.EXPECT().Authorize(ctx, authorizeAttributes).Return(authorizer.DecisionDeny, "", nil) }) - It("should forbid creating an autonomous shoot", func() { + It("should forbid creating a self-hosted shoot", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, core.Worker{ControlPlane: &core.WorkerControlPlane{}}) attrs = admission.NewAttributesRecord(shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) Expect(admissionHandler.Validate(ctx, attrs, nil)).NotTo(Succeed()) }) - It("should allow removing the control plane field of autonomous shoots", func() { + It("should allow removing the control plane field of self-hosted shoots", func() { // NB: This is already forbidden by validation, but this admission plugin does not know about it, so let's test it for completeness. shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, core.Worker{ControlPlane: &core.WorkerControlPlane{}}) oldShoot := shoot.DeepCopy() diff --git a/skaffold-gardenadm.yaml b/skaffold-gardenadm.yaml index d40b35d15b9..63fecc64b89 100644 --- a/skaffold-gardenadm.yaml +++ b/skaffold-gardenadm.yaml @@ -997,7 +997,7 @@ build: - bash - -c - > - kubectl --kubeconfig="$VIRTUAL_GARDEN_KUBECONFIG" -n garden patch gardenlet autonomous-shoot-root --type=merge -p "{\"spec\":{\"deployment\":{\"helm\":{\"ociRepository\":{\"ref\":\"$SKAFFOLD_IMAGE\"}}}}}" || true + kubectl --kubeconfig="$VIRTUAL_GARDEN_KUBECONFIG" -n garden patch gardenlet self-hosted-shoot-root --type=merge -p "{\"spec\":{\"deployment\":{\"helm\":{\"ociRepository\":{\"ref\":\"$SKAFFOLD_IMAGE\"}}}}}" || true insecureRegistries: - garden.local.gardener.cloud:5001 @@ -1277,16 +1277,16 @@ resourceSelector: image: - .* profiles: - - name: high-touch + - name: unmanaged-infra manifests: kustomize: paths: - - dev-setup/gardenadm/resources/overlays/high-touch - - name: medium-touch + - dev-setup/gardenadm/resources/overlays/unmanaged-infra + - name: managed-infra manifests: kustomize: paths: - - dev-setup/gardenadm/resources/overlays/medium-touch + - dev-setup/gardenadm/resources/overlays/managed-infra --- apiVersion: skaffold/v4beta12 kind: Config @@ -1315,7 +1315,7 @@ deploy: kubectl -n "$SKAFFOLD_NAMESPACES" cp dev-setup/gardenadm/resources/generated/.skaffold-image "$pod:/tmp/.skaffold-image" kubectl -n "$SKAFFOLD_NAMESPACES" cp dev-setup/gardenadm/resources/generated/.imagevector-overwrite.yaml "$pod:/gardenadm/imagevector-overwrite.yaml" kubectl -n "$SKAFFOLD_NAMESPACES" cp dev-setup/gardenadm/resources/generated/.imagevector-overwrite-charts.yaml "$pod:/gardenadm/imagevector-overwrite-charts.yaml" - kubectl -n "$SKAFFOLD_NAMESPACES" cp dev-setup/gardenadm/resources/generated/high-touch/manifests.yaml "$pod:/gardenadm/resources/manifests.yaml" + kubectl -n "$SKAFFOLD_NAMESPACES" cp dev-setup/gardenadm/resources/generated/unmanaged-infra/manifests.yaml "$pod:/gardenadm/resources/manifests.yaml" done - container: command: diff --git a/test/e2e/gardenadm/e2e_suite_test.go b/test/e2e/gardenadm/e2e_suite_test.go index a5aac4e91c0..0909fb5e27c 100644 --- a/test/e2e/gardenadm/e2e_suite_test.go +++ b/test/e2e/gardenadm/e2e_suite_test.go @@ -14,8 +14,8 @@ import ( "github.com/gardener/gardener/pkg/logger" . "github.com/gardener/gardener/test/e2e/gardenadm/common" - _ "github.com/gardener/gardener/test/e2e/gardenadm/hightouch" - _ "github.com/gardener/gardener/test/e2e/gardenadm/mediumtouch" + _ "github.com/gardener/gardener/test/e2e/gardenadm/managedinfra" + _ "github.com/gardener/gardener/test/e2e/gardenadm/unmanagedinfra" ) func TestE2E(t *testing.T) { diff --git a/test/e2e/gardenadm/mediumtouch/exec.go b/test/e2e/gardenadm/managedinfra/exec.go similarity index 99% rename from test/e2e/gardenadm/mediumtouch/exec.go rename to test/e2e/gardenadm/managedinfra/exec.go index fb098ff7409..a4b5dea74ae 100644 --- a/test/e2e/gardenadm/mediumtouch/exec.go +++ b/test/e2e/gardenadm/managedinfra/exec.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package mediumtouch +package managedinfra import ( "context" diff --git a/test/e2e/gardenadm/mediumtouch/gardenadm.go b/test/e2e/gardenadm/managedinfra/gardenadm.go similarity index 97% rename from test/e2e/gardenadm/mediumtouch/gardenadm.go rename to test/e2e/gardenadm/managedinfra/gardenadm.go index 8f834ea87bf..bc216288ec4 100644 --- a/test/e2e/gardenadm/mediumtouch/gardenadm.go +++ b/test/e2e/gardenadm/managedinfra/gardenadm.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package mediumtouch +package managedinfra import ( "context" @@ -35,7 +35,7 @@ import ( shootmigration "github.com/gardener/gardener/test/utils/shoots/migration" ) -var _ = Describe("gardenadm medium-touch scenario tests", Label("gardenadm", "medium-touch"), func() { +var _ = Describe("gardenadm managed infrastructure scenario tests", Label("gardenadm", "managed-infra"), func() { BeforeEach(OncePerOrdered, func(SpecContext) { PrepareBinary() }, NodeTimeout(5*time.Minute)) @@ -72,7 +72,7 @@ var _ = Describe("gardenadm medium-touch scenario tests", Label("gardenadm", "me It("should start the bootstrap flow", func() { // Start the gardenadm process but don't wait for it to complete so that we can asynchronously perform assertions // on individual steps in the test specs below. - session = Run("bootstrap", "-d", "../../../dev-setup/gardenadm/resources/generated/medium-touch", "--kubeconfig-output", kubeconfigOutputFile) + session = Run("bootstrap", "-d", "../../../dev-setup/gardenadm/resources/generated/managed-infra", "--kubeconfig-output", kubeconfigOutputFile) }) It("should auto-detect the system's public IPs", func(ctx SpecContext) { @@ -186,7 +186,7 @@ var _ = Describe("gardenadm medium-touch scenario tests", Label("gardenadm", "me }, SpecTimeout(15*time.Minute)) It("should write the shoot kubeconfig to the specified file", func(ctx SpecContext) { - Eventually(ctx, session.Err).Should(gbytes.Say("Writing kubeconfig of the autonomous shoot to file")) + Eventually(ctx, session.Err).Should(gbytes.Say("Writing kubeconfig of the self-hosted shoot to file")) // #nosec G304 -- kubeconfigOutputFile is controlled by the test Expect(os.ReadFile(kubeconfigOutputFile)).To(ContainSubstring("server: https://api.root.garden.local.gardener.cloud")) @@ -250,7 +250,7 @@ var _ = Describe("gardenadm medium-touch scenario tests", Label("gardenadm", "me }, SpecTimeout(time.Minute)) It("should run successfully a second time (should be idempotent)", func(ctx SpecContext) { - RunAndWait(ctx, "bootstrap", "-d", "../../../dev-setup/gardenadm/resources/generated/medium-touch") + RunAndWait(ctx, "bootstrap", "-d", "../../../dev-setup/gardenadm/resources/generated/managed-infra") }, SpecTimeout(2*time.Minute)) }) }) diff --git a/test/e2e/gardenadm/hightouch/gardenadm.go b/test/e2e/gardenadm/unmanagedinfra/gardenadm.go similarity index 97% rename from test/e2e/gardenadm/hightouch/gardenadm.go rename to test/e2e/gardenadm/unmanagedinfra/gardenadm.go index e1ebfeab214..5d855321a9d 100644 --- a/test/e2e/gardenadm/hightouch/gardenadm.go +++ b/test/e2e/gardenadm/unmanagedinfra/gardenadm.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package hightouch +package unmanagedinfra import ( "context" @@ -34,7 +34,7 @@ import ( . "github.com/gardener/gardener/test/e2e/gardenadm/common" ) -var _ = Describe("gardenadm high-touch scenario tests", Label("gardenadm", "high-touch"), func() { +var _ = Describe("gardenadm unmanaged infrastructure scenario tests", Label("gardenadm", "unmanaged-infra"), func() { var ( shootNamespace = "garden" shootName = "root" @@ -260,7 +260,7 @@ var _ = Describe("gardenadm high-touch scenario tests", Label("gardenadm", "high }).Should(Succeed()) }, SpecTimeout(time.Minute)) - It("should generate a bootstrap token and connect the autonomous shoot to Gardener", func(ctx SpecContext) { + It("should generate a bootstrap token and connect the self-hosted shoot to Gardener", func(ctx SpecContext) { stdOut, _, err := execute(ctx, 0, "sh", "-c", fmt.Sprintf("KUBECONFIG=%s gardenadm token create --print-connect-command --shoot-namespace=%s --shoot-name=%s", gardenClusterKubeconfigPathOnMachine, shootNamespace, shootName)) Expect(err).NotTo(HaveOccurred()) connectCommand := strings.Split(strings.ReplaceAll(string(stdOut.Contents()), `"`, ``), " ") @@ -268,7 +268,7 @@ var _ = Describe("gardenadm high-touch scenario tests", Label("gardenadm", "high stdOut, _, err = execute(ctx, 0, append(connectCommand, "--log-level=debug")...) Expect(err).NotTo(HaveOccurred()) - Eventually(ctx, stdOut).Should(gbytes.Say("Your autonomous shoot cluster has successfully been connected to Gardener!")) + Eventually(ctx, stdOut).Should(gbytes.Say("Your self-hosted shoot cluster has successfully been connected to Gardener!")) Eventually(ctx, func(g Gomega) { csrList := &certificatesv1.CertificateSigningRequestList{} diff --git a/test/e2e/gardenadm/hightouch/helper.go b/test/e2e/gardenadm/unmanagedinfra/helper.go similarity index 80% rename from test/e2e/gardenadm/hightouch/helper.go rename to test/e2e/gardenadm/unmanagedinfra/helper.go index 10b649fc584..253a0bffde9 100644 --- a/test/e2e/gardenadm/hightouch/helper.go +++ b/test/e2e/gardenadm/unmanagedinfra/helper.go @@ -2,14 +2,14 @@ // // SPDX-License-Identifier: Apache-2.0 -package hightouch +package unmanagedinfra import ( "strconv" ) const ( - namespace = "gardenadm-high-touch" + namespace = "gardenadm-unmanaged-infra" statefulSetName = "machine" ) diff --git a/test/integration/controllermanager/certificatesigningrequest/certificatesigningrequest_test.go b/test/integration/controllermanager/certificatesigningrequest/certificatesigningrequest_test.go index e6a7dee39c5..7bf1f9c9a89 100644 --- a/test/integration/controllermanager/certificatesigningrequest/certificatesigningrequest_test.go +++ b/test/integration/controllermanager/certificatesigningrequest/certificatesigningrequest_test.go @@ -126,7 +126,7 @@ var _ = Describe("CSR autoapprove controller tests", func() { Labels: map[string]string{testID: testRunID}, }, Data: map[string][]byte{ - "description": []byte(fmt.Sprintf("Used for connecting the autonomous Shoot %s/%s", shootNamespace, shootName)), + "description": []byte(fmt.Sprintf("Used for connecting the self-hosted Shoot %s/%s", shootNamespace, shootName)), }, } @@ -175,7 +175,7 @@ var _ = Describe("CSR autoapprove controller tests", func() { Labels: map[string]string{testID: testRunID}, }, Data: map[string][]byte{ - "description": []byte(fmt.Sprintf("Used for connecting the autonomous Shoot %s/%s", shootNamespace, shootName)), + "description": []byte(fmt.Sprintf("Used for connecting the self-hosted Shoot %s/%s", shootNamespace, shootName)), }, } diff --git a/test/integration/gardenlet/controllerinstallation/controllerinstallation/controllerinstallation_test.go b/test/integration/gardenlet/controllerinstallation/controllerinstallation/controllerinstallation_test.go index fd708901b0a..f78eac4ed2d 100644 --- a/test/integration/gardenlet/controllerinstallation/controllerinstallation/controllerinstallation_test.go +++ b/test/integration/gardenlet/controllerinstallation/controllerinstallation/controllerinstallation_test.go @@ -513,21 +513,21 @@ var _ = Describe("ControllerInstallation controller tests", func() { }) }) - When("seed is marked as autonomous shoot clusters", func() { + When("seed is marked as self-hosted shoot clusters", func() { BeforeEach(func() { - By("Mark Seed as autonomous shoot cluster") + By("Mark Seed as self-hosted shoot cluster") patch := client.MergeFrom(seed.DeepCopy()) - metav1.SetMetaDataLabel(&seed.ObjectMeta, "seed.gardener.cloud/autonomous-shoot-cluster", "true") + metav1.SetMetaDataLabel(&seed.ObjectMeta, "seed.gardener.cloud/self-hosted-shoot-cluster", "true") Expect(testClient.Patch(ctx, seed, patch)).To(Succeed()) DeferCleanup(func() { patch := client.MergeFrom(seed.DeepCopy()) - delete(seed.Labels, "seed.gardener.cloud/autonomous-shoot-cluster") + delete(seed.Labels, "seed.gardener.cloud/self-hosted-shoot-cluster") Expect(testClient.Patch(ctx, seed, patch)).To(Succeed()) }) }) - It("should set the autonomousShootCluster value in the chart", func() { + It("should set the selfHostedShootCluster value in the chart", func() { values := make(map[string]any) Eventually(func(g Gomega) { managedResource := &resourcesv1alpha1.ManagedResource{} @@ -544,7 +544,7 @@ var _ = Describe("ControllerInstallation controller tests", func() { valuesBytes, err := yaml.Marshal(values) Expect(err).NotTo(HaveOccurred()) - Expect(string(valuesBytes)).To(ContainSubstring("autonomousShootCluster: true")) + Expect(string(valuesBytes)).To(ContainSubstring("selfHostedShootCluster: true")) }) }) From eb844357ee6ccb01b0cb8b844ca56501556cbb8f Mon Sep 17 00:00:00 2001 From: Rada Dimitrova <117522044+RadaBDimitrova@users.noreply.github.com> Date: Fri, 31 Oct 2025 18:37:11 +0200 Subject: [PATCH 013/176] Bump `vertical-pod-autoscaler` dependency from `1.4.2` to `1.5.1` (#13315) --- go.mod | 15 +++++++-------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 6402cf360d3..0f376e35c27 100644 --- a/go.mod +++ b/go.mod @@ -66,8 +66,7 @@ require ( k8s.io/apiextensions-apiserver v0.34.1 k8s.io/apimachinery v0.34.1 k8s.io/apiserver v0.34.1 - // TODO(vitanovs): Update k8s.io/autoscaler/vertical-pod-autoscaler to v1.5.1 once https://github.com/gardener/gardener/issues/13162 is resolved. - k8s.io/autoscaler/vertical-pod-autoscaler v1.4.2 + k8s.io/autoscaler/vertical-pod-autoscaler v1.5.1 k8s.io/cli-runtime v0.34.1 k8s.io/client-go v0.34.1 k8s.io/cluster-bootstrap v0.34.1 @@ -76,7 +75,7 @@ require ( k8s.io/component-helpers v0.34.1 k8s.io/klog/v2 v2.130.1 k8s.io/kube-aggregator v0.34.1 - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b + k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 k8s.io/kube-proxy v0.34.1 k8s.io/kube-state-metrics/v2 v2.13.0 k8s.io/kubelet v0.34.1 @@ -130,7 +129,7 @@ require ( github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-metrics v0.0.1 // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -139,7 +138,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect - github.com/go-openapi/jsonpointer v0.21.1 // indirect + github.com/go-openapi/jsonpointer v0.21.2 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect @@ -270,19 +269,19 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/grpc v1.75.0 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect - k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f // indirect + k8s.io/gengo/v2 v2.0.0-20250820003526-c297c0c1eb9d // indirect k8s.io/klog v1.0.0 // indirect k8s.io/kms v0.34.1 // indirect k8s.io/sample-controller v0.30.3 // indirect oras.land/oras-go/v2 v2.6.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect ) diff --git a/go.sum b/go.sum index 3b8452fd296..4004cecb47d 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,8 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= -github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -273,8 +273,8 @@ github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUy github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= -github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= +github.com/go-openapi/jsonpointer v0.21.2 h1:AqQaNADVwq/VnkCmQg6ogE+M3FOsKTytwges0JdwVuA= +github.com/go-openapi/jsonpointer v0.21.2/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= @@ -1052,8 +1052,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -1094,8 +1094,8 @@ k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= k8s.io/apiserver v0.34.1 h1:U3JBGdgANK3dfFcyknWde1G6X1F4bg7PXuvlqt8lITA= k8s.io/apiserver v0.34.1/go.mod h1:eOOc9nrVqlBI1AFCvVzsob0OxtPZUCPiUJL45JOTBG0= -k8s.io/autoscaler/vertical-pod-autoscaler v1.4.2 h1:47RLgLhrxXfdBchTeNT2S9Xe9o+Y4kThExLfcGGUQMk= -k8s.io/autoscaler/vertical-pod-autoscaler v1.4.2/go.mod h1:rIBiAf+sK2mw8ryeHIZuY5juhJ4e2rNLwo59SDRXF7I= +k8s.io/autoscaler/vertical-pod-autoscaler v1.5.1 h1:LlVtM3IKqIVHz1ZXC3ahe/mAtDWb7Eob0tyTzqFULqg= +k8s.io/autoscaler/vertical-pod-autoscaler v1.5.1/go.mod h1:znhUnV0Yn+CkZu3TZ2HVqd8GFRMkPj/CXszX1gdBjTU= k8s.io/cli-runtime v0.34.1 h1:btlgAgTrYd4sk8vJTRG6zVtqBKt9ZMDeQZo2PIzbL7M= k8s.io/cli-runtime v0.34.1/go.mod h1:aVA65c+f0MZiMUPbseU/M9l1Wo2byeaGwUuQEQVVveE= k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= @@ -1115,8 +1115,8 @@ k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20201203183100-97869a43a9d9/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 h1:pWEwq4Asjm4vjW7vcsmijwBhOr1/shsbSYiWXmNGlks= k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f h1:SLb+kxmzfA87x4E4brQzB33VBbT2+x7Zq9ROIHmGn9Q= -k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/gengo/v2 v2.0.0-20250820003526-c297c0c1eb9d h1:qUrYOinhdAUL0xxhA4gPqogPBaS9nIq2l2kTb6pmeB0= +k8s.io/gengo/v2 v2.0.0-20250820003526-c297c0c1eb9d/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= @@ -1129,8 +1129,8 @@ k8s.io/kms v0.34.1/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= k8s.io/kube-aggregator v0.34.1 h1:WNLV0dVNoFKmuyvdWLd92iDSyD/TSTjqwaPj0U9XAEU= k8s.io/kube-aggregator v0.34.1/go.mod h1:RU8j+5ERfp0h+gIvWtxRPfsa5nK7rboDm8RST8BJfYQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 h1:liMHz39T5dJO1aOKHLvwaCjDbf07wVh6yaUlTpunnkE= +k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= k8s.io/kube-proxy v0.34.1 h1:cIriNCJY5XmRhXCCyQiazyqi47lbwcBQf0H76fVOpkw= k8s.io/kube-proxy v0.34.1/go.mod h1:syed9c5+gUVFMo6p24SnlTHzsp+BMd4ACcTw2dbArw0= k8s.io/kube-state-metrics/v2 v2.13.0 h1:g5OV0cwrDKvyrQApYxI7Ny+vFUgU3X6MYrnIzdCAdNU= @@ -1157,8 +1157,8 @@ sigs.k8s.io/controller-tools v0.19.0 h1:OU7jrPPiZusryu6YK0jYSjPqg8Vhf8cAzluP9XGI sigs.k8s.io/controller-tools v0.19.0/go.mod h1:y5HY/iNDFkmFla2CfQoVb2AQXMsBk4ad84iR1PLANB0= sigs.k8s.io/gateway-api v1.3.0 h1:q6okN+/UKDATola4JY7zXzx40WO4VISk7i9DIfOvr9M= sigs.k8s.io/gateway-api v1.3.0/go.mod h1:d8NV8nJbaRbEKem+5IuxkL8gJGOZ+FJ+NvOIltV8gDk= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= From c5e73670a1a11d1e50b4de5857dc06fad97edd88 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 31 Oct 2025 20:43:04 +0100 Subject: [PATCH 014/176] Update quay.io/kiwigrid/k8s-sidecar Docker tag to v2 (#13288) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index fdbaa6c03d3..93ad7b214e8 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -337,7 +337,7 @@ images: - name: plutono-data-refresher sourceRepository: github.com/kiwigrid/k8s-sidecar repository: quay.io/kiwigrid/k8s-sidecar - tag: "1.30.9" + tag: "2.0.3" labels: - name: 'gardener.cloud/cve-categorisation' value: From ebe0588fa0083429746d5d063f668e91eb831720 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 31 Oct 2025 20:43:12 +0100 Subject: [PATCH 015/176] Update dependency crate-ci/typos to v1.39.0 (#13320) --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 69b5c6b0515..876511c8741 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -83,7 +83,7 @@ YQ_VERSION ?= v4.48.1 # renovate: datasource=github-releases depName=ironcore-dev/vgopath VGOPATH_VERSION ?= v0.1.9 # renovate: datasource=github-releases depName=crate-ci/typos -TYPOS_VERSION ?= v1.38.1 +TYPOS_VERSION ?= v1.39.0 # tool versions from go.mod CONTROLLER_GEN_VERSION ?= $(call version_gomod,sigs.k8s.io/controller-tools) From 52f5105c6505f21352ba613ee84fdfff70de747d Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Sun, 2 Nov 2025 17:53:57 +0100 Subject: [PATCH 016/176] Update registry.k8s.io/ingress-nginx/controller-chroot Docker tag to v1.13.4 (#13318) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 93ad7b214e8..3690190f464 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -86,7 +86,7 @@ images: - name: nginx-ingress-controller sourceRepository: github.com/kubernetes/ingress-nginx repository: registry.k8s.io/ingress-nginx/controller-chroot - tag: "v1.13.3" + tag: "v1.13.4" labels: - name: 'gardener.cloud/cve-categorisation' value: From 893729010e2e1112c3411cca0e447c43c6e1a669 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Singh Date: Mon, 3 Nov 2025 15:58:59 +0530 Subject: [PATCH 017/176] Validate issuer URLs in ServiceAccountConfig (#13325) --- pkg/apis/core/validation/shoot.go | 7 ++++- pkg/apis/core/validation/shoot_test.go | 43 +++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index 80be1449b40..66938606743 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -1665,14 +1665,19 @@ func ValidateKubeAPIServer(kubeAPIServer *core.KubeAPIServerConfig, version stri allErrs = append(allErrs, field.Forbidden(fldPath.Child("serviceAccountConfig", "maxTokenExpiration"), "must be at most 2160h (90d)")) } } + + if kubeAPIServer.ServiceAccountConfig.Issuer != nil { + allErrs = append(allErrs, ValidateOIDCIssuerURL(*kubeAPIServer.ServiceAccountConfig.Issuer, fldPath.Child("serviceAccountConfig", "issuer"))...) + } if len(kubeAPIServer.ServiceAccountConfig.AcceptedIssuers) > 0 { issuers := sets.New[string]() if kubeAPIServer.ServiceAccountConfig.Issuer != nil { issuers.Insert(*kubeAPIServer.ServiceAccountConfig.Issuer) } for i, acceptedIssuer := range kubeAPIServer.ServiceAccountConfig.AcceptedIssuers { + path := fldPath.Child("serviceAccountConfig", "acceptedIssuers").Index(i) + allErrs = append(allErrs, ValidateOIDCIssuerURL(acceptedIssuer, path)...) if issuers.Has(acceptedIssuer) { - path := fldPath.Child("serviceAccountConfig", "acceptedIssuers").Index(i) if issuer := kubeAPIServer.ServiceAccountConfig.Issuer; issuer != nil && *issuer == acceptedIssuer { allErrs = append(allErrs, field.Invalid(path, acceptedIssuer, fmt.Sprintf("acceptedIssuers cannot contains the issuer field value: %s", acceptedIssuer))) } else { diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index 55742d0b816..8e29a27cd5d 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -2891,8 +2891,8 @@ var _ = Describe("Shoot Validation Tests", func() { It("should not allow to specify duplicates in accepted issuers", func() { shoot.Spec.Kubernetes.KubeAPIServer.ServiceAccountConfig = &core.ServiceAccountConfig{ AcceptedIssuers: []string{ - "foo", - "foo", + "https://issuer.example/auth", + "https://issuer.example/auth", }, } @@ -2906,8 +2906,8 @@ var _ = Describe("Shoot Validation Tests", func() { It("should not allow to duplicate the issuer in accepted issuers", func() { shoot.Spec.Kubernetes.KubeAPIServer.ServiceAccountConfig = &core.ServiceAccountConfig{ - Issuer: ptr.To("foo"), - AcceptedIssuers: []string{"foo"}, + Issuer: ptr.To("https://issuer.example/auth"), + AcceptedIssuers: []string{"https://issuer.example/auth"}, } errorList := ValidateShoot(shoot) @@ -2915,9 +2915,42 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(errorList).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.kubernetes.kubeAPIServer.serviceAccountConfig.acceptedIssuers[0]"), - "Detail": ContainSubstring("acceptedIssuers cannot contains the issuer field value: foo"), + "Detail": ContainSubstring("acceptedIssuers cannot contains the issuer field value: https://issuer.example/auth"), })))) }) + + It("should reject invalid issuer URL", func() { + shoot.Spec.Kubernetes.KubeAPIServer.ServiceAccountConfig = &core.ServiceAccountConfig{ + Issuer: ptr.To("http://issuer.invalid"), + } + + errorList := ValidateShoot(shoot) + + Expect(errorList).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.kubernetes.kubeAPIServer.serviceAccountConfig.issuer"), + })))) + }) + + DescribeTable("should reject invalid accepted issuer URLs", + func(bad string) { + shoot.Spec.Kubernetes.KubeAPIServer.ServiceAccountConfig = &core.ServiceAccountConfig{ + AcceptedIssuers: []string{bad}, + } + + errorList := ValidateShoot(shoot) + + Expect(errorList).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.kubernetes.kubeAPIServer.serviceAccountConfig.acceptedIssuers[0]"), + })))) + }, + Entry("plain string", "foo"), + Entry("non-https scheme", "http://issuer.com"), + Entry("contains fragment", "https://issuer.com#fragment"), + Entry("contains userinfo", "https://user:pass@issuer.com"), + Entry("contains query", "https://issuer.com?x=1"), + ) }) Context("Autoscaling validation", func() { From 6c7f9595a3fec74a3eca667172fcd55e1373d1b0 Mon Sep 17 00:00:00 2001 From: Johannes Roncossek <140696835+Roncossek@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:00:14 +0100 Subject: [PATCH 018/176] [GEP-33] Ensure `NamespacedCloudProfile` status format (#13138) * fix: defaulting strategy and validation for cloudprofiles using one architecture * feat: ensure uniform format for machine images in NamespacedCloudProfile for provider local * feat: implement ensureUniformFormat to synchronize NamespacedCloudProfile with parent CloudProfile * feat: simplify scope of initialMigration to machineCapabilities * adhere to repository standards and other review feedback * feat: simplify scope of initialMigration to machineCapabilities * move restrictToArchitectureCapability function and refine architecture syncing logic * fix: GetArchitecture uses capability definitions * feat: simulate transformation of namespacedcloudprofile format during admission webhook * refactor: move GetCapabilitiesWithAppliedDefaults to types and simplify GetArchitecture handling * feat: clear capability flavors during migration to legacy format * feat: implement simulation of transformation to parent spec format for NamespacedCloudProfile admission pligin * feat: add reference validation for removing machine capabilities in CloudProfile * adhere to review comments * test: update machine image expectations to use ConsistOf matcher --- docs/api-reference/provider-local.md | 1 + .../33-machine-image-capabilities.md | 14 +- .../pkg/controller/worker/cloudprofile.go | 2 +- pkg/apis/core/helper/cloudprofile.go | 21 +- pkg/apis/core/helper/cloudprofile_test.go | 40 +- pkg/apis/core/types_cloudprofile.go | 30 +- pkg/apis/core/types_test.go | 33 ++ pkg/apis/core/v1beta1/helper/cloudprofile.go | 25 +- .../core/v1beta1/helper/cloudprofile_test.go | 34 -- pkg/apis/core/v1beta1/types_cloudprofile.go | 47 +-- pkg/apis/core/v1beta1/types_test.go | 34 ++ pkg/apis/core/validation/cloudprofile.go | 46 ++- pkg/apis/core/validation/cloudprofile_test.go | 57 ++- pkg/apis/core/validation/utils.go | 173 ++++---- .../core/cloudprofile/strategy_test.go | 3 +- .../namespacedcloudprofile/reconciler.go | 38 +- .../namespacedcloudprofile/reconciler_test.go | 267 +++++++++++- .../mutator/namespacedcloudprofile.go | 31 +- .../mutator/namespacedcloudprofile_test.go | 84 +++- .../admission/validator/cloudprofile.go | 6 +- .../validator/namespacedcloudprofile.go | 37 ++ .../apis/local/v1alpha1/types_cloudprofile.go | 3 +- .../apis/local/validation/cloudprofile.go | 8 +- .../controller/worker/machine_images.go | 4 +- pkg/utils/gardener/cloudprofile.go | 147 ++++--- pkg/utils/gardener/cloudprofile_test.go | 120 ++++-- pkg/utils/gardener/namespacedcloudprofile.go | 87 ++++ .../gardener/namespacedcloudprofile_test.go | 380 ++++++++++++++++++ .../resourcereferencemanager/admission.go | 66 ++- .../admission_test.go | 232 +++++++++++ .../validator/admission.go | 63 ++- .../validator/admission_test.go | 225 +++++++---- plugin/pkg/shoot/validator/admission.go | 2 +- plugin/pkg/shoot/validator/admission_test.go | 2 +- skaffold-operator.yaml | 6 + .../namespacedcloudprofile_test.go | 50 +-- 36 files changed, 1930 insertions(+), 488 deletions(-) create mode 100644 pkg/utils/gardener/namespacedcloudprofile.go create mode 100644 pkg/utils/gardener/namespacedcloudprofile_test.go diff --git a/docs/api-reference/provider-local.md b/docs/api-reference/provider-local.md index f416bfb0db6..8392c2647f8 100644 --- a/docs/api-reference/provider-local.md +++ b/docs/api-reference/provider-local.md @@ -251,6 +251,7 @@ string +(Optional)

Image is the image for the machine image.

diff --git a/docs/proposals/33-machine-image-capabilities.md b/docs/proposals/33-machine-image-capabilities.md index b1364a17b8d..3aa92a8fe17 100644 --- a/docs/proposals/33-machine-image-capabilities.md +++ b/docs/proposals/33-machine-image-capabilities.md @@ -188,14 +188,14 @@ In this case it does not result in an error but in performance loss. ## Proposal -Introduce a top level capabilities array in the CloudProfile `spec.capabilities`. +Introduce a top level array in the CloudProfile `spec.machineCapabilities`. ```go type Spec struct { - Capabilities []Capability + MachineCapabilities []CapabilityDefinition } -type Capability struct { +type CapabilityDefinition struct { Name string Values []string } @@ -205,7 +205,7 @@ Capabilities are very specific to the provider and the selected catalog offered To minimize complexity and data size in the cloud profile, the capabilities are defined as a map with string keys and string arrays as values. The key is the capability name and the value is an array of possible values. -For each cloud profile the capabilities are defined in the `spec.capabilities` array. +For each cloud profile the capabilities are defined in the `spec.machineCapabilities` array. The full set of possibilities for each capability is defined here. As some capabilities can have multiple values at the same time an array of possible values is used instead of a single value. @@ -215,7 +215,7 @@ If no further information is provided each machine type and machine image will b ```yaml # CloudProfile Example spec: - capabilities: + machineCapabilities: - name: architecture values: [amd64, arm64] - name: hypervisorType @@ -255,7 +255,7 @@ The architecture is also added to the image version capability flavors. This is ```yaml # CloudProfile spec: - capabilities: # <-- Full list of possible capabilities used as default + machineCapabilities: # <-- Full list of possible capabilities used as default - name: architecture values: [amd64] - name: hypervisorType @@ -361,7 +361,7 @@ If multiple machine image versions are valid for a machine type, the selection i The following implications are to be considered in regards to `NamespacedCloudProfile`s: -* `NamespacedCloudProfile`s won't have a global `capabilities` definition. +* `NamespacedCloudProfile`s won't have a global `machineCapabilities` definition. * For overridden machine image versions, no `capabilities` must be defined, as they are inherited from the machine image versions of the parent `CloudProfile`. * For custom machine image versions, the `capabilities` need to be defined in the `NamespacedCloudProfile`, as they would be in the `CloudProfile` (as well as for the `providerConfig`). diff --git a/extensions/pkg/controller/worker/cloudprofile.go b/extensions/pkg/controller/worker/cloudprofile.go index 503215aa939..4758cc8f6e5 100644 --- a/extensions/pkg/controller/worker/cloudprofile.go +++ b/extensions/pkg/controller/worker/cloudprofile.go @@ -78,7 +78,7 @@ func selectBestImageFlavor[T CapabilitiesAccessor]( capabilitiesWithProviderTypes = append(capabilitiesWithProviderTypes, capabilitiesWithProviderType{ providerEntry: set, // Normalize capabilities copy by applying defaults - capabilities: v1beta1helper.GetCapabilitiesWithAppliedDefaults(set.GetCapabilities(), capabilityDefinitions), + capabilities: v1beta1.GetCapabilitiesWithAppliedDefaults(set.GetCapabilities(), capabilityDefinitions), }) } diff --git a/pkg/apis/core/helper/cloudprofile.go b/pkg/apis/core/helper/cloudprofile.go index 18b8745a293..26722737cb5 100644 --- a/pkg/apis/core/helper/cloudprofile.go +++ b/pkg/apis/core/helper/cloudprofile.go @@ -282,30 +282,17 @@ func CapabilityDefinitionsToCapabilities(capabilityDefinitions []core.Capability return capabilities } -// GetCapabilitiesWithAppliedDefaults returns new capabilities with applied defaults from the capability definitions. -func GetCapabilitiesWithAppliedDefaults(capabilities core.Capabilities, capabilityDefinitions []core.CapabilityDefinition) core.Capabilities { - result := make(core.Capabilities, len(capabilityDefinitions)) - for _, capabilityDefinition := range capabilityDefinitions { - if values, ok := capabilities[capabilityDefinition.Name]; ok { - result[capabilityDefinition.Name] = values - } else { - result[capabilityDefinition.Name] = capabilityDefinition.Values - } - } - return result -} - -// GetImageFlavorWithAppliedDefaults returns MachineImageFlavors sets with applied defaults from the capability definitions. -func GetImageFlavorWithAppliedDefaults(imageFlavors []core.MachineImageFlavor, capabilityDefinitions []core.CapabilityDefinition) []core.MachineImageFlavor { +// GetImageFlavorsWithAppliedDefaults returns MachineImageFlavors sets with applied defaults from the capability definitions. +func GetImageFlavorsWithAppliedDefaults(imageFlavors []core.MachineImageFlavor, capabilityDefinitions []core.CapabilityDefinition) []core.MachineImageFlavor { if len(imageFlavors) == 0 { // If no capabilityFlavors are defined, assume all capabilities are supported. - return []core.MachineImageFlavor{{Capabilities: GetCapabilitiesWithAppliedDefaults(core.Capabilities{}, capabilityDefinitions)}} + return []core.MachineImageFlavor{{Capabilities: core.GetCapabilitiesWithAppliedDefaults(core.Capabilities{}, capabilityDefinitions)}} } result := make([]core.MachineImageFlavor, len(imageFlavors)) for i, imageFlavor := range imageFlavors { result[i] = core.MachineImageFlavor{ - Capabilities: GetCapabilitiesWithAppliedDefaults(imageFlavor.Capabilities, capabilityDefinitions), + Capabilities: core.GetCapabilitiesWithAppliedDefaults(imageFlavor.Capabilities, capabilityDefinitions), } } return result diff --git a/pkg/apis/core/helper/cloudprofile_test.go b/pkg/apis/core/helper/cloudprofile_test.go index bf0f9533b19..7cabf2d987a 100644 --- a/pkg/apis/core/helper/cloudprofile_test.go +++ b/pkg/apis/core/helper/cloudprofile_test.go @@ -518,41 +518,7 @@ var _ = Describe("CloudProfile Helper", func() { Entry("with capabilities", "architecture", "network"), ) - Describe("#GetCapabilitiesWithAppliedDefaults", func() { - It("should apply default values when capabilities are nil", func() { - var capabilities core.Capabilities - capabilityDefinitions := []core.CapabilityDefinition{ - {Name: "capability1", Values: []string{"value1", "value2"}}, - {Name: "architecture", Values: []string{"amd64"}}, - } - - result := GetCapabilitiesWithAppliedDefaults(capabilities, capabilityDefinitions) - - Expect(result).To(Equal(core.Capabilities{ - "capability1": []string{"value1", "value2"}, - "architecture": []string{"amd64"}, - })) - }) - - It("should retain existing values and apply defaults for missing capabilities", func() { - capabilities := core.Capabilities{ - "capability1": []string{"value1"}, - } - capabilityDefinitions := []core.CapabilityDefinition{ - {Name: "capability1", Values: []string{"value1", "value2"}}, - {Name: "architecture", Values: []string{"amd64"}}, - } - - result := GetCapabilitiesWithAppliedDefaults(capabilities, capabilityDefinitions) - - Expect(result).To(Equal(core.Capabilities{ - "capability1": []string{"value1"}, - "architecture": []string{"amd64"}, - })) - }) - }) - - Describe("#GetImageFlavorWithAppliedDefaults", func() { + Describe("#GetImageFlavorsWithAppliedDefaults", func() { It("should apply defaults when capabilityFlavors are empty", func() { var imageFlavors []core.MachineImageFlavor capabilityDefinitions := []core.CapabilityDefinition{ @@ -560,7 +526,7 @@ var _ = Describe("CloudProfile Helper", func() { {Name: "architecture", Values: []string{"amd64"}}, } - result := GetImageFlavorWithAppliedDefaults(imageFlavors, capabilityDefinitions) + result := GetImageFlavorsWithAppliedDefaults(imageFlavors, capabilityDefinitions) Expect(result).To(HaveLen(1)) Expect(result[0].Capabilities).To(Equal(core.Capabilities{ @@ -579,7 +545,7 @@ var _ = Describe("CloudProfile Helper", func() { {Name: "architecture", Values: []string{"amd64", "arm64"}}, } - result := GetImageFlavorWithAppliedDefaults(imageFlavors, capabilityDefinitions) + result := GetImageFlavorsWithAppliedDefaults(imageFlavors, capabilityDefinitions) Expect(result).To(HaveLen(2)) Expect(result[0].Capabilities).To(Equal(core.Capabilities{ diff --git a/pkg/apis/core/types_cloudprofile.go b/pkg/apis/core/types_cloudprofile.go index 9fc9794f765..d03c2986b4e 100644 --- a/pkg/apis/core/types_cloudprofile.go +++ b/pkg/apis/core/types_cloudprofile.go @@ -173,14 +173,6 @@ type MachineType struct { Capabilities Capabilities } -// GetArchitecture returns the architecture of the machine type. -func (m *MachineType) GetArchitecture() string { - if len(m.Capabilities[constants.ArchitectureName]) == 1 { - return m.Capabilities[constants.ArchitectureName][0] - } - return ptr.Deref(m.Architecture, "") -} - // MachineTypeStorage is the amount of storage associated with the root volume of this machine type. type MachineTypeStorage struct { // Class is the class of the storage type. @@ -332,3 +324,25 @@ type Capabilities map[string]CapabilityValues type MachineImageFlavor struct { Capabilities } + +// GetArchitecture returns the architecture of the machine type. +func (m *MachineType) GetArchitecture(capabilityDefinitions []CapabilityDefinition) string { + capabilityArchitecture := GetCapabilitiesWithAppliedDefaults(m.Capabilities, capabilityDefinitions)[constants.ArchitectureName] + if len(capabilityArchitecture) == 1 { + return capabilityArchitecture[0] + } + return ptr.Deref(m.Architecture, "") +} + +// GetCapabilitiesWithAppliedDefaults returns new capabilities with applied defaults from the capability definitions. +func GetCapabilitiesWithAppliedDefaults(capabilities Capabilities, capabilityDefinitions []CapabilityDefinition) Capabilities { + result := make(Capabilities, len(capabilityDefinitions)) + for _, capabilityDefinition := range capabilityDefinitions { + if values, ok := capabilities[capabilityDefinition.Name]; ok { + result[capabilityDefinition.Name] = values + } else { + result[capabilityDefinition.Name] = capabilityDefinition.Values + } + } + return result +} diff --git a/pkg/apis/core/types_test.go b/pkg/apis/core/types_test.go index 3b43a01bac1..f85ea00df66 100644 --- a/pkg/apis/core/types_test.go +++ b/pkg/apis/core/types_test.go @@ -53,4 +53,37 @@ var _ = Describe("API Types", func() { Entry("#ClassificationDeprecated is active", ClassificationDeprecated, true), Entry("#ClassificationExpired is not active", ClassificationExpired, false), ) + Describe("#GetCapabilitiesWithAppliedDefaults", func() { + It("should apply default values when capabilities are nil", func() { + var capabilities Capabilities + capabilityDefinitions := []CapabilityDefinition{ + {Name: "capability1", Values: []string{"value1", "value2"}}, + {Name: "architecture", Values: []string{"amd64"}}, + } + + result := GetCapabilitiesWithAppliedDefaults(capabilities, capabilityDefinitions) + + Expect(result).To(Equal(Capabilities{ + "capability1": []string{"value1", "value2"}, + "architecture": []string{"amd64"}, + })) + }) + + It("should retain existing values and apply defaults for missing capabilities", func() { + capabilities := Capabilities{ + "capability1": []string{"value1"}, + } + capabilityDefinitions := []CapabilityDefinition{ + {Name: "capability1", Values: []string{"value1", "value2"}}, + {Name: "architecture", Values: []string{"amd64"}}, + } + + result := GetCapabilitiesWithAppliedDefaults(capabilities, capabilityDefinitions) + + Expect(result).To(Equal(Capabilities{ + "capability1": []string{"value1"}, + "architecture": []string{"amd64"}, + })) + }) + }) }) diff --git a/pkg/apis/core/v1beta1/helper/cloudprofile.go b/pkg/apis/core/v1beta1/helper/cloudprofile.go index 9c7d086c4db..91c94bb5791 100644 --- a/pkg/apis/core/v1beta1/helper/cloudprofile.go +++ b/pkg/apis/core/v1beta1/helper/cloudprofile.go @@ -533,7 +533,7 @@ func FilterDeprecatedVersion() func(expirableVersion gardencorev1beta1.Expirable } } -func extractArchitecturesFromImageFlavor(imageFlavors []gardencorev1beta1.MachineImageFlavor, capabilityDefinitions []gardencorev1beta1.CapabilityDefinition) []string { +func extractArchitecturesFromImageFlavors(imageFlavors []gardencorev1beta1.MachineImageFlavor, capabilityDefinitions []gardencorev1beta1.CapabilityDefinition) []string { if len(imageFlavors) == 0 { for _, capabilityDefinition := range capabilityDefinitions { if capabilityDefinition.Name == constants.ArchitectureName { @@ -554,7 +554,7 @@ func extractArchitecturesFromImageFlavor(imageFlavors []gardencorev1beta1.Machin // GetArchitecturesFromImageVersion returns the list of supported architectures for the machine image version. // It first tries to retrieve the architectures from the capability flavors and falls back to the architectures field if none are found. func GetArchitecturesFromImageVersion(imageVersion gardencorev1beta1.MachineImageVersion, capabilityDefinitions []gardencorev1beta1.CapabilityDefinition) []string { - if architectures := extractArchitecturesFromImageFlavor(imageVersion.CapabilityFlavors, capabilityDefinitions); len(architectures) > 0 { + if architectures := extractArchitecturesFromImageFlavors(imageVersion.CapabilityFlavors, capabilityDefinitions); len(architectures) > 0 { return architectures } return imageVersion.Architectures @@ -567,30 +567,17 @@ func ArchitectureSupportedByImageVersion(version gardencorev1beta1.MachineImageV return slices.Contains(supportedArchitectures, architecture) } -// GetCapabilitiesWithAppliedDefaults returns new capabilities with applied defaults from the capability definitions. -func GetCapabilitiesWithAppliedDefaults(capabilities gardencorev1beta1.Capabilities, capabilityDefinitions []gardencorev1beta1.CapabilityDefinition) gardencorev1beta1.Capabilities { - result := make(gardencorev1beta1.Capabilities, len(capabilityDefinitions)) - for _, capabilityDefinition := range capabilityDefinitions { - if values, ok := capabilities[capabilityDefinition.Name]; ok { - result[capabilityDefinition.Name] = values - } else { - result[capabilityDefinition.Name] = capabilityDefinition.Values - } - } - return result -} - // GetImageFlavorsWithAppliedDefaults returns new MachineImageFlavors with applied defaults from the capability definitions. func GetImageFlavorsWithAppliedDefaults(imageFlavors []gardencorev1beta1.MachineImageFlavor, capabilityDefinitions []gardencorev1beta1.CapabilityDefinition) []gardencorev1beta1.MachineImageFlavor { if len(imageFlavors) == 0 { // If no capabilityFlavors are defined, assume all capabilities are supported. - return []gardencorev1beta1.MachineImageFlavor{{Capabilities: GetCapabilitiesWithAppliedDefaults(gardencorev1beta1.Capabilities{}, capabilityDefinitions)}} + return []gardencorev1beta1.MachineImageFlavor{{Capabilities: gardencorev1beta1.GetCapabilitiesWithAppliedDefaults(gardencorev1beta1.Capabilities{}, capabilityDefinitions)}} } result := make([]gardencorev1beta1.MachineImageFlavor, len(imageFlavors)) for i, imageFlavor := range imageFlavors { result[i] = gardencorev1beta1.MachineImageFlavor{ - Capabilities: GetCapabilitiesWithAppliedDefaults(imageFlavor.Capabilities, capabilityDefinitions), + Capabilities: gardencorev1beta1.GetCapabilitiesWithAppliedDefaults(imageFlavor.Capabilities, capabilityDefinitions), } } return result @@ -649,8 +636,8 @@ func AreCapabilitiesSupportedByImageFlavors( // AreCapabilitiesCompatible checks if two sets of capabilities are compatible. // It applies defaults from the capability definitions to both sets before checking compatibility. func AreCapabilitiesCompatible(capabilities1, capabilities2 gardencorev1beta1.Capabilities, capabilityDefinitions []gardencorev1beta1.CapabilityDefinition) bool { - defaultedCapabilities1 := GetCapabilitiesWithAppliedDefaults(capabilities1, capabilityDefinitions) - defaultedCapabilities2 := GetCapabilitiesWithAppliedDefaults(capabilities2, capabilityDefinitions) + defaultedCapabilities1 := gardencorev1beta1.GetCapabilitiesWithAppliedDefaults(capabilities1, capabilityDefinitions) + defaultedCapabilities2 := gardencorev1beta1.GetCapabilitiesWithAppliedDefaults(capabilities2, capabilityDefinitions) isSupported := true commonCapabilities := GetCapabilitiesIntersection(defaultedCapabilities1, defaultedCapabilities2) diff --git a/pkg/apis/core/v1beta1/helper/cloudprofile_test.go b/pkg/apis/core/v1beta1/helper/cloudprofile_test.go index 455fb86fd33..029e10bb385 100644 --- a/pkg/apis/core/v1beta1/helper/cloudprofile_test.go +++ b/pkg/apis/core/v1beta1/helper/cloudprofile_test.go @@ -1506,40 +1506,6 @@ var _ = Describe("CloudProfile Helper", func() { Entry("Should be true for supported architecture", []string{"amd64", "arm64"}, "arm64", true), ) - Describe("#GetCapabilitiesWithAppliedDefaults", func() { - It("should apply default values when capabilities are nil", func() { - var capabilities gardencorev1beta1.Capabilities - capabilityDefinitions := []gardencorev1beta1.CapabilityDefinition{ - {Name: "capability1", Values: []string{"value1", "value2"}}, - {Name: "architecture", Values: []string{"amd64"}}, - } - - result := GetCapabilitiesWithAppliedDefaults(capabilities, capabilityDefinitions) - - Expect(result).To(Equal(gardencorev1beta1.Capabilities{ - "capability1": []string{"value1", "value2"}, - "architecture": []string{"amd64"}, - })) - }) - - It("should retain existing values and apply defaults for missing capabilities", func() { - capabilities := gardencorev1beta1.Capabilities{ - "capability1": []string{"value1"}, - } - capabilityDefinitions := []gardencorev1beta1.CapabilityDefinition{ - {Name: "capability1", Values: []string{"value1", "value2"}}, - {Name: "architecture", Values: []string{"amd64"}}, - } - - result := GetCapabilitiesWithAppliedDefaults(capabilities, capabilityDefinitions) - - Expect(result).To(Equal(gardencorev1beta1.Capabilities{ - "capability1": []string{"value1"}, - "architecture": []string{"amd64"}, - })) - }) - }) - Describe("#GetImageFlavorsWithAppliedDefaults", func() { It("should apply defaults when capabilityFlavors are empty", func() { var imageFlavors []gardencorev1beta1.MachineImageFlavor diff --git a/pkg/apis/core/v1beta1/types_cloudprofile.go b/pkg/apis/core/v1beta1/types_cloudprofile.go index 821e4127a4b..97b45fa3f11 100644 --- a/pkg/apis/core/v1beta1/types_cloudprofile.go +++ b/pkg/apis/core/v1beta1/types_cloudprofile.go @@ -196,31 +196,6 @@ type MachineType struct { Capabilities Capabilities `json:"capabilities,omitempty" protobuf:"bytes,8,rep,name=capabilities,casttype=Capabilities"` } -// GetArchitecture returns the architecture of the machine type. -func (m *MachineType) GetArchitecture(capabilityDefinitions []CapabilityDefinition) string { - if len(capabilityDefinitions) == 0 { - return ptr.Deref(m.Architecture, "") - } - - if len(m.Capabilities[constants.ArchitectureName]) == 1 { - return m.Capabilities[constants.ArchitectureName][0] - } - - if len(m.Capabilities[constants.ArchitectureName]) == 0 { - for _, capabilityDefinition := range capabilityDefinitions { - if capabilityDefinition.Name == constants.ArchitectureName && len(capabilityDefinition.Values) == 1 { - return capabilityDefinition.Values[0] - } - } - } - - // constants.ArchitectureName is a required capability and - // machineType.Capabilities[constants.ArchitectureName] can only - // be empty for cloudprofiles supporting exactly one architecture. - // we should never reach this point. - return "" -} - // MachineTypeStorage is the amount of storage associated with the root volume of this machine type. type MachineTypeStorage struct { // Class is the class of the storage type. @@ -409,3 +384,25 @@ func (c *MachineImageFlavor) UnmarshalJSON(data []byte) error { func (c *MachineImageFlavor) MarshalJSON() ([]byte, error) { return json.Marshal(c.Capabilities) } + +// GetArchitecture returns the architecture of the machine type. +func (m *MachineType) GetArchitecture(capabilityDefinitions []CapabilityDefinition) string { + capabilityArchitecture := GetCapabilitiesWithAppliedDefaults(m.Capabilities, capabilityDefinitions)[constants.ArchitectureName] + if len(capabilityArchitecture) == 1 { + return capabilityArchitecture[0] + } + return ptr.Deref(m.Architecture, "") +} + +// GetCapabilitiesWithAppliedDefaults returns new capabilities with applied defaults from the capability definitions. +func GetCapabilitiesWithAppliedDefaults(capabilities Capabilities, capabilityDefinitions []CapabilityDefinition) Capabilities { + result := make(Capabilities, len(capabilityDefinitions)) + for _, capabilityDefinition := range capabilityDefinitions { + if values, ok := capabilities[capabilityDefinition.Name]; ok { + result[capabilityDefinition.Name] = values + } else { + result[capabilityDefinition.Name] = capabilityDefinition.Values + } + } + return result +} diff --git a/pkg/apis/core/v1beta1/types_test.go b/pkg/apis/core/v1beta1/types_test.go index 9a137b511fe..3ffd5946408 100644 --- a/pkg/apis/core/v1beta1/types_test.go +++ b/pkg/apis/core/v1beta1/types_test.go @@ -53,4 +53,38 @@ var _ = Describe("Types", func() { Entry("#ClassificationDeprecated is active", ClassificationDeprecated, true), Entry("#ClassificationExpired is not active", ClassificationExpired, false), ) + + Describe("#GetCapabilitiesWithAppliedDefaults", func() { + It("should apply default values when capabilities are nil", func() { + var capabilities Capabilities + capabilityDefinitions := []CapabilityDefinition{ + {Name: "capability1", Values: []string{"value1", "value2"}}, + {Name: "architecture", Values: []string{"amd64"}}, + } + + result := GetCapabilitiesWithAppliedDefaults(capabilities, capabilityDefinitions) + + Expect(result).To(Equal(Capabilities{ + "capability1": []string{"value1", "value2"}, + "architecture": []string{"amd64"}, + })) + }) + + It("should retain existing values and apply defaults for missing capabilities", func() { + capabilities := Capabilities{ + "capability1": []string{"value1"}, + } + capabilityDefinitions := []CapabilityDefinition{ + {Name: "capability1", Values: []string{"value1", "value2"}}, + {Name: "architecture", Values: []string{"amd64"}}, + } + + result := GetCapabilitiesWithAppliedDefaults(capabilities, capabilityDefinitions) + + Expect(result).To(Equal(Capabilities{ + "capability1": []string{"value1"}, + "architecture": []string{"amd64"}, + })) + }) + }) }) diff --git a/pkg/apis/core/validation/cloudprofile.go b/pkg/apis/core/validation/cloudprofile.go index 620dbd35a6a..a8e74025a2d 100644 --- a/pkg/apis/core/validation/cloudprofile.go +++ b/pkg/apis/core/validation/cloudprofile.go @@ -56,18 +56,18 @@ func ValidateCloudProfileUpdate(newProfile, oldProfile *core.CloudProfile) field // ValidateCloudProfileSpec validates the specification of a CloudProfile object. func ValidateCloudProfileSpec(spec *core.CloudProfileSpec, fldPath *field.Path) field.ErrorList { var ( - allErrs = field.ErrorList{} - capabilities = helper.CapabilityDefinitionsToCapabilities(spec.MachineCapabilities) + allErrs = field.ErrorList{} + machineCapabilities = helper.CapabilityDefinitionsToCapabilities(spec.MachineCapabilities) ) if len(spec.Type) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("type"), "must provide a provider type")) } - allErrs = append(allErrs, validateCloudProfileKubernetesSettings(spec.Kubernetes, fldPath.Child("kubernetes"))...) - allErrs = append(allErrs, ValidateCloudProfileMachineImages(spec.MachineImages, capabilities, fldPath.Child("machineImages"))...) - allErrs = append(allErrs, validateCloudProfileMachineTypes(spec.MachineTypes, capabilities, fldPath.Child("machineTypes"))...) allErrs = append(allErrs, validateCapabilityDefinitions(spec.MachineCapabilities, fldPath.Child("machineCapabilities"))...) + allErrs = append(allErrs, validateCloudProfileKubernetesSettings(spec.Kubernetes, fldPath.Child("kubernetes"))...) + allErrs = append(allErrs, ValidateCloudProfileMachineImages(spec.MachineImages, machineCapabilities, fldPath.Child("machineImages"))...) + allErrs = append(allErrs, validateCloudProfileMachineTypes(spec.MachineTypes, machineCapabilities, fldPath.Child("machineTypes"))...) allErrs = append(allErrs, validateVolumeTypes(spec.VolumeTypes, fldPath.Child("volumeTypes"))...) allErrs = append(allErrs, validateCloudProfileRegions(spec.Regions, fldPath.Child("regions"))...) allErrs = append(allErrs, validateCloudProfileBastion(spec, fldPath.Child("bastion"))...) @@ -184,23 +184,31 @@ func ValidateCloudProfileMachineImages(machineImages []core.MachineImage, capabi for index, machineVersion := range image.Versions { versionsPath := idxPath.Child("versions").Index(index) - allErrs = append(allErrs, validateContainerRuntimesInterfaces(machineVersion.CRI, versionsPath.Child("cri"))...) - allErrs = append(allErrs, validateSupportedVersionsConfiguration(machineVersion.ExpirableVersion, helper.ToExpirableVersions(image.Versions), versionsPath)...) - - if len(capabilities) == 0 { - if len(machineVersion.Architectures) == 0 { - allErrs = append(allErrs, field.Required(versionsPath.Child("architectures"), "must provide at least one architecture")) - } - if len(machineVersion.CapabilityFlavors) > 0 { - allErrs = append(allErrs, field.Forbidden(versionsPath.Child("capabilityFlavors"), "must not provide capabilities without global definition")) - } - } + allErrs = append(allErrs, ValidateMachineImageAdditionalConfig(machineVersion, versionsPath, image, capabilities)...) } } return allErrs } +// ValidateMachineImageAdditionalConfig validates RuntimeInterfaces and supported versions configuration of a MachineImageVersion. +func ValidateMachineImageAdditionalConfig(machineVersion core.MachineImageVersion, versionsPath *field.Path, image core.MachineImage, capabilities core.Capabilities) field.ErrorList { + allErrs := field.ErrorList{} + + allErrs = append(allErrs, validateContainerRuntimesInterfaces(machineVersion.CRI, versionsPath.Child("cri"))...) + allErrs = append(allErrs, validateSupportedVersionsConfiguration(machineVersion.ExpirableVersion, helper.ToExpirableVersions(image.Versions), versionsPath)...) + + if len(capabilities) == 0 { + if len(machineVersion.Architectures) == 0 { + allErrs = append(allErrs, field.Required(versionsPath.Child("architectures"), "must provide at least one architecture")) + } + if len(machineVersion.CapabilityFlavors) > 0 { + allErrs = append(allErrs, field.Forbidden(versionsPath.Child("capabilityFlavors"), "must not provide capabilities without global definition")) + } + } + return allErrs +} + func validateContainerRuntimesInterfaces(cris []core.CRI, fldPath *field.Path) field.ErrorList { var ( allErrs = field.ErrorList{} @@ -301,7 +309,7 @@ func validateCloudProfileBastion(spec *core.CloudProfileSpec, fldPath *field.Pat if spec.Bastion.MachineType != nil { var validationErrors field.ErrorList - machineArch, validationErrors = validateBastionMachineType(spec.Bastion.MachineType, spec.MachineTypes, fldPath.Child("machineType")) + machineArch, validationErrors = validateBastionMachineType(spec.Bastion.MachineType, spec.MachineTypes, spec.MachineCapabilities, fldPath.Child("machineType")) allErrs = append(allErrs, validationErrors...) } @@ -312,7 +320,7 @@ func validateCloudProfileBastion(spec *core.CloudProfileSpec, fldPath *field.Pat return allErrs } -func validateBastionMachineType(bastionMachineType *core.BastionMachineType, machineTypes []core.MachineType, fldPath *field.Path) (*string, field.ErrorList) { +func validateBastionMachineType(bastionMachineType *core.BastionMachineType, machineTypes []core.MachineType, capabilityDefinitions []core.CapabilityDefinition, fldPath *field.Path) (*string, field.ErrorList) { machineIndex := slices.IndexFunc(machineTypes, func(machineType core.MachineType) bool { return machineType.Name == bastionMachineType.Name }) @@ -321,7 +329,7 @@ func validateBastionMachineType(bastionMachineType *core.BastionMachineType, mac return nil, field.ErrorList{field.Invalid(fldPath.Child("name"), bastionMachineType.Name, "machine type not found in spec.machineTypes")} } - return ptr.To(machineTypes[machineIndex].GetArchitecture()), nil + return ptr.To(machineTypes[machineIndex].GetArchitecture(capabilityDefinitions)), nil } func validateBastionImage(bastionImage *core.BastionMachineImage, machineImages []core.MachineImage, capabilities core.Capabilities, machineArch *string, fldPath *field.Path) field.ErrorList { diff --git a/pkg/apis/core/validation/cloudprofile_test.go b/pkg/apis/core/validation/cloudprofile_test.go index 21c88498f67..5c69b1ae353 100644 --- a/pkg/apis/core/validation/cloudprofile_test.go +++ b/pkg/apis/core/validation/cloudprofile_test.go @@ -1340,11 +1340,51 @@ var _ = Describe("CloudProfile Validation Tests ", func() { {Name: "architecture", Values: []string{"amd64", "arm64"}}, {Name: "anotherCapability", Values: []string{"value1"}}, } + cloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = []core.MachineImageFlavor{ + {Capabilities: core.Capabilities{"architecture": []string{"amd64"}}}, + } + }) + + When("Capability Architecture has one supported value", func() { + BeforeEach(func() { + cloudProfile.Spec.MachineCapabilities = []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"arm64"}}, + {Name: "anotherCapability", Values: []string{"value1"}}, + } + }) + + It("should succeed to validate with neither architectures nor capabilities set for machine images and types", func() { + cloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = nil + allErrs := ValidateCloudProfile(cloudProfile) + Expect(allErrs).To(BeEmpty()) + }) }) - It("should succeed to validate with neither architectures nor capability capabilityFlavors set for machine images", func() { + It("should succeed to validate with architecture(s) and capabilities set for machine images and types", func() { cloudProfile.Spec.MachineTypes[0].Architecture = ptr.To("arm64") - Expect(ValidateCloudProfile(cloudProfile)).To(BeEmpty()) + cloudProfile.Spec.MachineImages[0].Versions[0].Architectures = []string{"arm64"} + + cloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = []core.MachineImageFlavor{ + {Capabilities: core.Capabilities{"architecture": []string{"arm64"}}}, + } + cloudProfile.Spec.MachineTypes[0].Capabilities = core.Capabilities{ + "architecture": []string{"arm64"}, + } + allErrs := ValidateCloudProfile(cloudProfile) + Expect(allErrs).To(BeEmpty()) + }) + + It("should fail to validate with machineImages without capabilityFlavors", func() { + cloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = nil + cloudProfile.Spec.MachineTypes[0].Architecture = ptr.To("arm64") + + Expect(ValidateCloudProfile(cloudProfile)).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("spec.machineImages[0].versions[0].capabilityFlavors"), + "Detail": Equal("must provide at least one image flavor when multiple architectures are defined in spec.machineCapabilities"), + })), + )) }) It("should fail to validate with no architecture capability defined in a machine image flavor", func() { @@ -1357,7 +1397,7 @@ var _ = Describe("CloudProfile Validation Tests ", func() { PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeRequired), "Field": Equal("spec.machineImages[0].versions[0].capabilityFlavors[0].architecture"), - "Detail": Equal("must provide at least one architecture"), + "Detail": Equal("must specify one architecture explicitly as multiple architectures are defined in spec.machineCapabilities"), })), )) }) @@ -1371,7 +1411,7 @@ var _ = Describe("CloudProfile Validation Tests ", func() { "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.machineImages[0].versions[0].architectures"), "BadValue": Equal([]string{"arm64"}), - "Detail": Equal("architecture field values set (arm64) conflict with the capability architectures ()"), + "Detail": Equal("architecture field values set (arm64) conflict with the capability architectures (amd64)"), })), )) }) @@ -1446,6 +1486,8 @@ var _ = Describe("CloudProfile Validation Tests ", func() { }) It("should fail to validate capabilities with empty keys or values", func() { + cloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = nil + cloudProfile.Spec.MachineCapabilities = []core.CapabilityDefinition{ {Name: "", Values: core.CapabilityValues{}}, {Name: "hasNoValues", Values: []string{}}, @@ -1484,7 +1526,7 @@ var _ = Describe("CloudProfile Validation Tests ", func() { It("should fail to validate invalid architecture capability values", func() { cloudProfile.Spec.MachineCapabilities = []core.CapabilityDefinition{ - {Name: "architecture", Values: []string{"arm64", "custom"}}, + {Name: "architecture", Values: []string{"amd64", "custom"}}, } Expect(ValidateCloudProfile(cloudProfile)).To(ConsistOf( @@ -1947,6 +1989,9 @@ var _ = Describe("CloudProfile Validation Tests ", func() { cloudProfileNew.Spec.MachineCapabilities = []core.CapabilityDefinition{ {Name: "architecture", Values: []string{"amd64", "arm64"}}, } + cloudProfileNew.Spec.MachineImages[0].Versions[0].CapabilityFlavors = []core.MachineImageFlavor{ + {Capabilities: core.Capabilities{"architecture": []string{"amd64"}}}, + } if !isInitialSwitch { cloudProfileOld.Spec.MachineCapabilities = cloudProfileNew.Spec.MachineCapabilities @@ -1970,7 +2015,7 @@ var _ = Describe("CloudProfile Validation Tests ", func() { "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.machineImages[0].versions[0].architectures"), "BadValue": Equal([]string{"arm64"}), - "Detail": Equal("architecture field values set (arm64) conflict with the capability architectures ()"), + "Detail": Equal("architecture field values set (arm64) conflict with the capability architectures (amd64)"), })), )) }) diff --git a/pkg/apis/core/validation/utils.go b/pkg/apis/core/validation/utils.go index 85a0bf8c966..2adeccc76d8 100644 --- a/pkg/apis/core/validation/utils.go +++ b/pkg/apis/core/validation/utils.go @@ -199,63 +199,79 @@ func ValidateMachineImages(machineImages []core.MachineImage, capabilities core. duplicateName := sets.Set[string]{} for i, image := range machineImages { idxPath := fldPath.Index(i) - if duplicateName.Has(image.Name) { - allErrs = append(allErrs, field.Duplicate(idxPath, image.Name)) - } - duplicateName.Insert(image.Name) + allErrs = append(allErrs, ValidateMachineImage(image, capabilities, duplicateName, duplicateNameVersion, allowEmptyVersions, idxPath)...) + } - if len(image.Name) == 0 { - allErrs = append(allErrs, field.Required(idxPath.Child("name"), "machine image name must not be empty")) - } else if errs := validateUnprefixedQualifiedName(image.Name); len(errs) != 0 { - allErrs = append(allErrs, field.Invalid(idxPath.Child("name"), image.Name, fmt.Sprintf("machine image name must be a qualified name: %v", errs))) - } + return allErrs +} - if len(image.Versions) == 0 && !allowEmptyVersions { - allErrs = append(allErrs, field.Required(idxPath.Child("versions"), fmt.Sprintf("must provide at least one version for the machine image '%s'", image.Name))) - } +// ValidateMachineImage validates a single machine image for valid values and combinations. +func ValidateMachineImage(image core.MachineImage, capabilities core.Capabilities, duplicateName sets.Set[string], duplicateNameVersion sets.Set[string], allowEmptyVersions bool, idxPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} - if image.UpdateStrategy != nil { - if !availableUpdateStrategiesForMachineImage.Has(string(*image.UpdateStrategy)) { - allErrs = append(allErrs, field.NotSupported(idxPath.Child("updateStrategy"), *image.UpdateStrategy, sets.List(availableUpdateStrategiesForMachineImage))) - } - } + if duplicateName.Has(image.Name) { + allErrs = append(allErrs, field.Duplicate(idxPath, image.Name)) + } + duplicateName.Insert(image.Name) - for index, machineVersion := range image.Versions { - versionsPath := idxPath.Child("versions").Index(index) - key := fmt.Sprintf("%s-%s", image.Name, machineVersion.Version) - if duplicateNameVersion.Has(key) { - allErrs = append(allErrs, field.Duplicate(versionsPath, key)) - } - duplicateNameVersion.Insert(key) - if len(machineVersion.Version) == 0 { - allErrs = append(allErrs, field.Required(versionsPath.Child("version"), machineVersion.Version)) - } + if len(image.Name) == 0 { + allErrs = append(allErrs, field.Required(idxPath.Child("name"), "machine image name must not be empty")) + } else if errs := validateUnprefixedQualifiedName(image.Name); len(errs) != 0 { + allErrs = append(allErrs, field.Invalid(idxPath.Child("name"), image.Name, fmt.Sprintf("machine image name must be a qualified name: %v", errs))) + } - _, err := semver.NewVersion(machineVersion.Version) - if err != nil { - allErrs = append(allErrs, field.Invalid(versionsPath.Child("version"), machineVersion.Version, "could not parse version. Use a semantic version. In case there is no semantic version for this image use the extensibility provider (define mapping in the CloudProfile) to map to the actual non semantic version")) - } + if len(image.Versions) == 0 && !allowEmptyVersions { + allErrs = append(allErrs, field.Required(idxPath.Child("versions"), fmt.Sprintf("must provide at least one version for the machine image '%s'", image.Name))) + } - if machineVersion.InPlaceUpdates != nil && machineVersion.InPlaceUpdates.MinVersionForUpdate != nil { - if _, err = semver.NewVersion(*machineVersion.InPlaceUpdates.MinVersionForUpdate); err != nil { - allErrs = append(allErrs, field.Invalid(versionsPath.Child("minVersionForInPlaceUpdate"), *machineVersion.InPlaceUpdates.MinVersionForUpdate, "could not parse version. Use a semantic version.")) - } - } + if image.UpdateStrategy != nil { + if !availableUpdateStrategiesForMachineImage.Has(string(*image.UpdateStrategy)) { + allErrs = append(allErrs, field.NotSupported(idxPath.Child("updateStrategy"), *image.UpdateStrategy, sets.List(availableUpdateStrategiesForMachineImage))) + } + } - if machineVersion.Classification != nil && !supportedVersionClassifications.Has(string(*machineVersion.Classification)) { - allErrs = append(allErrs, field.NotSupported(versionsPath.Child("classification"), *machineVersion.Classification, sets.List(supportedVersionClassifications))) - } + for index, machineVersion := range image.Versions { + versionsPath := idxPath.Child("versions").Index(index) + allErrs = append(allErrs, ValidateMachineImageVersion(image, machineVersion, capabilities, duplicateNameVersion, versionsPath)...) + } + return allErrs +} - allErrs = append(allErrs, validateMachineImageVersionFlavors(machineVersion, capabilities, versionsPath)...) +// ValidateMachineImageVersion validates a single machine image version for valid values and combinations. +func ValidateMachineImageVersion(image core.MachineImage, machineVersion core.MachineImageVersion, capabilities core.Capabilities, duplicateNameVersion sets.Set[string], versionsPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} - if machineVersion.KubeletVersionConstraint != nil { - if _, err := semver.NewConstraint(*machineVersion.KubeletVersionConstraint); err != nil { - allErrs = append(allErrs, field.Invalid(versionsPath.Child("kubeletVersionConstraint"), machineVersion.KubeletVersionConstraint, fmt.Sprintf("cannot parse the kubeletVersionConstraint: %s", err.Error()))) - } - } + key := fmt.Sprintf("%s-%s", image.Name, machineVersion.Version) + if duplicateNameVersion.Has(key) { + allErrs = append(allErrs, field.Duplicate(versionsPath, key)) + } + duplicateNameVersion.Insert(key) + if len(machineVersion.Version) == 0 { + allErrs = append(allErrs, field.Required(versionsPath.Child("version"), machineVersion.Version)) + } + + _, err := semver.NewVersion(machineVersion.Version) + if err != nil { + allErrs = append(allErrs, field.Invalid(versionsPath.Child("version"), machineVersion.Version, "could not parse version. Use a semantic version. In case there is no semantic version for this image use the extensibility provider (define mapping in the CloudProfile) to map to the actual non semantic version")) + } + + if machineVersion.InPlaceUpdates != nil && machineVersion.InPlaceUpdates.MinVersionForUpdate != nil { + if _, err = semver.NewVersion(*machineVersion.InPlaceUpdates.MinVersionForUpdate); err != nil { + allErrs = append(allErrs, field.Invalid(versionsPath.Child("minVersionForInPlaceUpdate"), *machineVersion.InPlaceUpdates.MinVersionForUpdate, "could not parse version. Use a semantic version.")) } } + if machineVersion.Classification != nil && !supportedVersionClassifications.Has(string(*machineVersion.Classification)) { + allErrs = append(allErrs, field.NotSupported(versionsPath.Child("classification"), *machineVersion.Classification, sets.List(supportedVersionClassifications))) + } + + allErrs = append(allErrs, validateMachineImageVersionFlavors(machineVersion, capabilities, versionsPath)...) + + if machineVersion.KubeletVersionConstraint != nil { + if _, err := semver.NewConstraint(*machineVersion.KubeletVersionConstraint); err != nil { + allErrs = append(allErrs, field.Invalid(versionsPath.Child("kubeletVersionConstraint"), machineVersion.KubeletVersionConstraint, fmt.Sprintf("cannot parse the kubeletVersionConstraint: %s", err.Error()))) + } + } return allErrs } @@ -277,33 +293,40 @@ func validateMachineTypes(machineTypes []core.MachineType, capabilities core.Cap for i, machineType := range machineTypes { idxPath := fldPath.Index(i) - namePath := idxPath.Child("name") - cpuPath := idxPath.Child("cpu") - gpuPath := idxPath.Child("gpu") - memoryPath := idxPath.Child("memory") + allErrs = append(allErrs, ValidateMachineType(machineType, names, capabilities, idxPath)...) + } - if len(machineType.Name) == 0 { - allErrs = append(allErrs, field.Required(namePath, "must provide a name")) - } else if errs := validateUnprefixedQualifiedName(machineType.Name); len(errs) != 0 { - allErrs = append(allErrs, field.Invalid(fldPath.Index(i).Child("name"), machineType.Name, fmt.Sprintf("machine type name must be a qualified name: %v", errs))) - } + return allErrs +} - if names.Has(machineType.Name) { - allErrs = append(allErrs, field.Duplicate(namePath, machineType.Name)) - break - } - names.Insert(machineType.Name) +// ValidateMachineType validates a single machine type for valid values and combinations. +func ValidateMachineType(machineType core.MachineType, names sets.Set[string], capabilities core.Capabilities, idxPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} - allErrs = append(allErrs, kubernetescorevalidation.ValidateResourceQuantityValue("cpu", machineType.CPU, cpuPath)...) - allErrs = append(allErrs, kubernetescorevalidation.ValidateResourceQuantityValue("gpu", machineType.GPU, gpuPath)...) - allErrs = append(allErrs, kubernetescorevalidation.ValidateResourceQuantityValue("memory", machineType.Memory, memoryPath)...) - allErrs = append(allErrs, validateMachineTypeCapabilities(machineType, capabilities, idxPath)...) + namePath := idxPath.Child("name") + cpuPath := idxPath.Child("cpu") + gpuPath := idxPath.Child("gpu") + memoryPath := idxPath.Child("memory") - if machineType.Storage != nil { - allErrs = append(allErrs, validateMachineTypeStorage(*machineType.Storage, idxPath.Child("storage"))...) - } + if len(machineType.Name) == 0 { + allErrs = append(allErrs, field.Required(namePath, "must provide a name")) + } else if errs := validateUnprefixedQualifiedName(machineType.Name); len(errs) != 0 { + allErrs = append(allErrs, field.Invalid(idxPath.Child("name"), machineType.Name, fmt.Sprintf("machine type name must be a qualified name: %v", errs))) + } + + if names.Has(machineType.Name) { + return append(allErrs, field.Duplicate(namePath, machineType.Name)) } + names.Insert(machineType.Name) + allErrs = append(allErrs, kubernetescorevalidation.ValidateResourceQuantityValue("cpu", machineType.CPU, cpuPath)...) + allErrs = append(allErrs, kubernetescorevalidation.ValidateResourceQuantityValue("gpu", machineType.GPU, gpuPath)...) + allErrs = append(allErrs, kubernetescorevalidation.ValidateResourceQuantityValue("memory", machineType.Memory, memoryPath)...) + allErrs = append(allErrs, validateMachineTypeCapabilities(machineType, capabilities, idxPath)...) + + if machineType.Storage != nil { + allErrs = append(allErrs, validateMachineTypeStorage(*machineType.Storage, idxPath.Child("storage"))...) + } return allErrs } @@ -379,25 +402,35 @@ func validateMachineImageVersionArchitecture(machineImageVersion core.MachineIma // assert that the architecture values defined do not conflict if len(capabilities) > 0 { supportedArchitectures = capabilities[v1beta1constants.ArchitectureName] + + if len(supportedArchitectures) > 1 && len(machineImageVersion.CapabilityFlavors) == 0 { + return append(allErrs, field.Required(fldPath.Child("capabilityFlavors"), + "must provide at least one image flavor when multiple architectures are defined in spec.machineCapabilities")) + } + for flavorIdx, flavor := range machineImageVersion.CapabilityFlavors { architectureCapabilityValues := flavor.Capabilities[v1beta1constants.ArchitectureName] architectureFieldPath := fldPath.Child("capabilityFlavors").Index(flavorIdx).Child("architecture") - if len(architectureCapabilityValues) == 0 { - allErrs = append(allErrs, field.Required(architectureFieldPath, "must provide at least one architecture")) + if len(architectureCapabilityValues) == 0 && len(supportedArchitectures) > 1 { + allErrs = append(allErrs, field.Required(architectureFieldPath, "must specify one architecture explicitly as multiple architectures are defined in spec.machineCapabilities")) } else if len(architectureCapabilityValues) > 1 { allErrs = append(allErrs, field.Invalid(architectureFieldPath, architectureCapabilityValues, "must not define more than one architecture within an image flavor")) } } allCapabilityArchitectures := sets.New(gardencorehelper.ExtractArchitecturesFromImageFlavors(machineImageVersion.CapabilityFlavors)...) + if len(allCapabilityArchitectures) == 0 && len(supportedArchitectures) == 1 { + allCapabilityArchitectures = sets.New(supportedArchitectures...) + } if len(machineImageVersion.Architectures) > 0 && !allCapabilityArchitectures.HasAll(machineImageVersion.Architectures...) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("architectures"), machineImageVersion.Architectures, fmt.Sprintf("architecture field values set (%s) conflict with the capability architectures (%s)", strings.Join(machineImageVersion.Architectures, ","), strings.Join(allCapabilityArchitectures.UnsortedList(), ",")))) + allErrs = append(allErrs, field.Invalid(fldPath.Child("architectures"), machineImageVersion.Architectures, + fmt.Sprintf("architecture field values set (%s) conflict with the capability architectures (%s)", strings.Join(machineImageVersion.Architectures, ","), strings.Join(allCapabilityArchitectures.UnsortedList(), ",")))) } } for archIdx, arch := range machineImageVersion.Architectures { if !slices.Contains(supportedArchitectures, arch) { - allErrs = append(allErrs, field.NotSupported(fldPath.Child("architectures").Index(archIdx), arch, v1beta1constants.ValidArchitectures)) + allErrs = append(allErrs, field.NotSupported(fldPath.Child("architectures").Index(archIdx), arch, supportedArchitectures)) } } diff --git a/pkg/apiserver/registry/core/cloudprofile/strategy_test.go b/pkg/apiserver/registry/core/cloudprofile/strategy_test.go index de3717ba878..f78b177fee1 100644 --- a/pkg/apiserver/registry/core/cloudprofile/strategy_test.go +++ b/pkg/apiserver/registry/core/cloudprofile/strategy_test.go @@ -186,7 +186,7 @@ var _ = Describe("Strategy", func() { } newCloudProfile := oldCloudProfile.DeepCopy() newCloudProfile.Spec.MachineCapabilities = []core.CapabilityDefinition{ - {Name: "architecture", Values: []string{"amd64"}}, + {Name: "architecture", Values: []string{"amd64", "arm64"}}, } cloudprofileregistry.Strategy.PrepareForUpdate(context.Background(), newCloudProfile, oldCloudProfile) @@ -200,7 +200,6 @@ var _ = Describe("Strategy", func() { })) }) }) - Describe("#Canonicalize", func() { It("should sync architecture capabilities to empty architecture fields", func() { cloudProfile := &core.CloudProfile{ diff --git a/pkg/controllermanager/controller/namespacedcloudprofile/reconciler.go b/pkg/controllermanager/controller/namespacedcloudprofile/reconciler.go index b7b5fde79bf..edc129dfc77 100644 --- a/pkg/controllermanager/controller/namespacedcloudprofile/reconciler.go +++ b/pkg/controllermanager/controller/namespacedcloudprofile/reconciler.go @@ -21,6 +21,7 @@ import ( "github.com/gardener/gardener/pkg/api" gardencore "github.com/gardener/gardener/pkg/apis/core" + gardencorehelper "github.com/gardener/gardener/pkg/apis/core/helper" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" @@ -118,8 +119,11 @@ func MergeCloudProfiles(namespacedCloudProfile *gardencorev1beta1.NamespacedClou if namespacedCloudProfile.Spec.Kubernetes != nil { namespacedCloudProfile.Status.CloudProfileSpec.Kubernetes.Versions = mergeDeep(namespacedCloudProfile.Status.CloudProfileSpec.Kubernetes.Versions, namespacedCloudProfile.Spec.Kubernetes.Versions, expirableVersionKeyFunc, mergeExpirationDates, false) } - namespacedCloudProfile.Status.CloudProfileSpec.MachineImages = mergeDeep(namespacedCloudProfile.Status.CloudProfileSpec.MachineImages, namespacedCloudProfile.Spec.MachineImages, machineImageKeyFunc, mergeMachineImages, true) - namespacedCloudProfile.Status.CloudProfileSpec.MachineTypes = mergeDeep(namespacedCloudProfile.Status.CloudProfileSpec.MachineTypes, namespacedCloudProfile.Spec.MachineTypes, machineTypeKeyFunc, nil, true) + + // TODO(Roncossek): Remove TransformSpecToParentFormat once all CloudProfiles have been migrated to use CapabilityFlavors and the Architecture fields are effectively forbidden or have been removed. + uniformNamespacedCloudProfileSpec := gardenerutils.TransformSpecToParentFormat(namespacedCloudProfile.Spec, cloudProfile.Spec.MachineCapabilities) + namespacedCloudProfile.Status.CloudProfileSpec.MachineImages = mergeDeep(namespacedCloudProfile.Status.CloudProfileSpec.MachineImages, uniformNamespacedCloudProfileSpec.MachineImages, machineImageKeyFunc, mergeMachineImages, true) + namespacedCloudProfile.Status.CloudProfileSpec.MachineTypes = mergeDeep(namespacedCloudProfile.Status.CloudProfileSpec.MachineTypes, uniformNamespacedCloudProfileSpec.MachineTypes, machineTypeKeyFunc, nil, true) namespacedCloudProfile.Status.CloudProfileSpec.VolumeTypes = mergeDeep(namespacedCloudProfile.Status.CloudProfileSpec.VolumeTypes, namespacedCloudProfile.Spec.VolumeTypes, volumeTypeKeyFunc, nil, true) if namespacedCloudProfile.Spec.CABundle != nil { mergedCABundles := fmt.Sprintf("%s%s", ptr.Deref(namespacedCloudProfile.Status.CloudProfileSpec.CABundle, ""), ptr.Deref(namespacedCloudProfile.Spec.CABundle, "")) @@ -140,17 +144,39 @@ func MergeCloudProfiles(namespacedCloudProfile *gardencorev1beta1.NamespacedClou func syncArchitectureCapabilities(namespacedCloudProfile *gardencorev1beta1.NamespacedCloudProfile) { var coreCloudProfileSpec gardencore.CloudProfileSpec _ = api.Scheme.Convert(&namespacedCloudProfile.Status.CloudProfileSpec, &coreCloudProfileSpec, nil) - gardenerutils.SyncArchitectureCapabilityFields(coreCloudProfileSpec, gardencore.CloudProfileSpec{}) - defaultMachineTypeArchitectures(coreCloudProfileSpec) + defaultMachineTypeArchitectures(coreCloudProfileSpec, coreCloudProfileSpec.MachineCapabilities) + defaultMachineImageArchitectures(coreCloudProfileSpec, coreCloudProfileSpec.MachineCapabilities) _ = api.Scheme.Convert(&coreCloudProfileSpec, &namespacedCloudProfile.Status.CloudProfileSpec, nil) } // defaultMachineTypeArchitectures defaults the architectures of the machine types for NamespacedCloudProfiles. // The sync can only happen after having had a look at the parent CloudProfile and whether it uses capabilities. -func defaultMachineTypeArchitectures(cloudProfile gardencore.CloudProfileSpec) { +func defaultMachineTypeArchitectures(cloudProfile gardencore.CloudProfileSpec, capabilitiesDefinitions []gardencore.CapabilityDefinition) { for i, machineType := range cloudProfile.MachineTypes { - if machineType.GetArchitecture() == "" { + effectiveArchitecture := machineType.GetArchitecture(capabilitiesDefinitions) + if effectiveArchitecture == "" { cloudProfile.MachineTypes[i].Architecture = ptr.To(v1beta1constants.ArchitectureAMD64) + } else if cloudProfile.MachineTypes[i].Architecture == nil { + cloudProfile.MachineTypes[i].Architecture = ptr.To(effectiveArchitecture) + } + } +} + +// defaultMachineImageArchitectures defaults the architectures of the machine images for NamespacedCloudProfiles. +// The sync can only happen after having had a look at the parent CloudProfile and whether it uses capabilities. +func defaultMachineImageArchitectures(cloudProfile gardencore.CloudProfileSpec, capabilitiesDefinitions []gardencore.CapabilityDefinition) { + for i, machineImage := range cloudProfile.MachineImages { + for j, version := range machineImage.Versions { + if len(version.Architectures) > 0 { + continue + } + capabilityFlavors := gardencorehelper.GetImageFlavorsWithAppliedDefaults(version.CapabilityFlavors, capabilitiesDefinitions) + capabilityArchitectures := gardencorehelper.ExtractArchitecturesFromImageFlavors(capabilityFlavors) + if len(capabilityArchitectures) == 0 { + cloudProfile.MachineImages[i].Versions[j].Architectures = []string{v1beta1constants.ArchitectureAMD64} + } else if len(capabilityArchitectures) > 0 { + cloudProfile.MachineImages[i].Versions[j].Architectures = capabilityArchitectures + } } } } diff --git a/pkg/controllermanager/controller/namespacedcloudprofile/reconciler_test.go b/pkg/controllermanager/controller/namespacedcloudprofile/reconciler_test.go index 2e56f27c1db..786d8ab300b 100644 --- a/pkg/controllermanager/controller/namespacedcloudprofile/reconciler_test.go +++ b/pkg/controllermanager/controller/namespacedcloudprofile/reconciler_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" "go.uber.org/mock/gomock" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -531,7 +532,7 @@ var _ = Describe("NamespacedCloudProfile Reconciler", func() { c.EXPECT().Status().Return(sw), sw.EXPECT().Patch(gomock.Any(), gomock.AssignableToTypeOf(&gardencorev1beta1.NamespacedCloudProfile{}), gomock.Any()).DoAndReturn(func(_ context.Context, o client.Object, patch client.Patch, _ ...client.PatchOption) error { versionOverride := fmt.Sprintf(`{"architectures":["amd64"],"cri":[{"name":"containerd"}],"expirationDate":"%s","kubeletVersionConstraint":"==1.30.0","version":"1.0.0"}`, newExpiryDate.UTC().Format(time.RFC3339)) - versionAdded := `{"version":"1.1.2"}` + versionAdded := `{"architectures":["amd64"],"version":"1.1.2"}` Expect(patch.Data(o)).To(And( // The order is (currently) indeterministic. ContainSubstring(`{"status":{"cloudProfileSpec":{"machineImages":[{"name":"test-image","updateStrategy":"major","versions":[`), @@ -755,10 +756,16 @@ var _ = Describe("NamespacedCloudProfile Reconciler", func() { Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(namespacedCloudProfile), namespacedCloudProfile)).To(Succeed()) Expect(namespacedCloudProfile.Status.CloudProfileSpec.MachineImages).To(ConsistOf( gardencorev1beta1.MachineImage{Name: "machine-image-1", Versions: []gardencorev1beta1.MachineImageVersion{ - {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "1.0.0"}}, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "1.0.0"}, + Architectures: []string{"amd64"}, + }, }}, gardencorev1beta1.MachineImage{Name: "machine-image-2", Versions: []gardencorev1beta1.MachineImageVersion{ - {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "2.0.0"}}, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "2.0.0"}, + Architectures: []string{"amd64"}, + }, }}, )) Expect(namespacedCloudProfile.Status.CloudProfileSpec.ProviderConfig.Raw).To(Equal( @@ -803,10 +810,16 @@ var _ = Describe("NamespacedCloudProfile Reconciler", func() { Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(namespacedCloudProfile), namespacedCloudProfile)).To(Succeed()) Expect(namespacedCloudProfile.Status.CloudProfileSpec.MachineImages).To(ConsistOf( gardencorev1beta1.MachineImage{Name: "machine-image-1", Versions: []gardencorev1beta1.MachineImageVersion{ - {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "1.0.0"}}, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "1.0.0"}, + Architectures: []string{"amd64"}, + }, }}, gardencorev1beta1.MachineImage{Name: "machine-image-2", Versions: []gardencorev1beta1.MachineImageVersion{ - {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "3.0.0"}}, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "3.0.0"}, + Architectures: []string{"amd64"}, + }, }}, )) // Make sure that status providerConfig is set to cloudProfile providerConfig again. @@ -908,6 +921,250 @@ var _ = Describe("NamespacedCloudProfile Reconciler", func() { Expect(namespacedCloudProfile.Status.CloudProfileSpec.Limits.MaxNodesTotal).To(Equal(ptr.To(int32(20)))) }) }) + + Describe("Transform to parent CloudProfile capability/legacy format functionality", func() { + var ( + cloudProfile *gardencorev1beta1.CloudProfile + namespacedCloudProfile *gardencorev1beta1.NamespacedCloudProfile + ) + + BeforeEach(func() { + cloudProfile = &gardencorev1beta1.CloudProfile{} + namespacedCloudProfile = &gardencorev1beta1.NamespacedCloudProfile{} + }) + + When("parent CloudProfile has capability definitions", func() { + BeforeEach(func() { + cloudProfile.Spec.MachineCapabilities = []gardencorev1beta1.CapabilityDefinition{ + {Name: "architecture", Values: []string{"amd64", "arm64"}}, + } + }) + + It("should transform legacy architectures to capability flavors when parent is in capability format", func() { + namespacedCloudProfile.Spec.MachineImages = []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "20.04"}, + Architectures: []string{"amd64", "arm64"}, + }, + }, + }, + } + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + Expect(namespacedCloudProfile.Status.CloudProfileSpec.MachineImages).To(HaveLen(1)) + Expect(namespacedCloudProfile.Status.CloudProfileSpec.MachineImages[0].Versions).To(HaveLen(1)) + version := namespacedCloudProfile.Status.CloudProfileSpec.MachineImages[0].Versions[0] + Expect(version.CapabilityFlavors).To(HaveLen(2)) + Expect(version.CapabilityFlavors[0].Capabilities).To(HaveKeyWithValue("architecture", gardencorev1beta1.CapabilityValues{"amd64"})) + Expect(version.CapabilityFlavors[1].Capabilities).To(HaveKeyWithValue("architecture", gardencorev1beta1.CapabilityValues{"arm64"})) + Expect(version.Architectures).To(ConsistOf("amd64", "arm64")) + }) + + It("should add architecture to capabilityFlavors", func() { + namespacedCloudProfile.Spec.MachineImages = []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "20.04"}, + Architectures: []string{"amd64"}, + }, + }, + }, + } + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + version := namespacedCloudProfile.Status.CloudProfileSpec.MachineImages[0].Versions[0] + Expect(version.CapabilityFlavors).To(HaveLen(1)) + Expect(version.CapabilityFlavors[0].Capabilities).To(HaveKeyWithValue("architecture", gardencorev1beta1.CapabilityValues{"amd64"})) + Expect(version.Architectures).To(ConsistOf("amd64")) + }) + + It("should preserve existing capability flavors", func() { + namespacedCloudProfile.Spec.MachineImages = []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "20.04"}, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{"arm64"}, + "gpu": []string{"nvidia"}, + }, + }, + }, + Architectures: []string{"arm64"}, + }, + }, + }, + } + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + version := namespacedCloudProfile.Status.CloudProfileSpec.MachineImages[0].Versions[0] + Expect(version.CapabilityFlavors).To(HaveLen(1)) + Expect(version.CapabilityFlavors[0].Capabilities).To(HaveKeyWithValue("architecture", gardencorev1beta1.CapabilityValues{"arm64"})) + Expect(version.CapabilityFlavors[0].Capabilities).To(HaveKeyWithValue("gpu", gardencorev1beta1.CapabilityValues{"nvidia"})) + }) + + It("should set architecture capabilities for machine types", func() { + namespacedCloudProfile.Spec.MachineTypes = []gardencorev1beta1.MachineType{ + { + Name: "m5.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + }, + { + Name: "m5.arm.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + Architecture: ptr.To("arm64"), + }, + } + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + machineTypes := namespacedCloudProfile.Status.CloudProfileSpec.MachineTypes + Expect(machineTypes).To(HaveLen(2)) + + Expect(machineTypes).To(ConsistOf( + MatchFields(IgnoreExtras, Fields{ + "Architecture": Equal(ptr.To("amd64")), + "Capabilities": HaveKeyWithValue("architecture", gardencorev1beta1.CapabilityValues{"amd64"}), + }), + MatchFields(IgnoreExtras, Fields{ + "Architecture": Equal(ptr.To("arm64")), + "Capabilities": HaveKeyWithValue("architecture", gardencorev1beta1.CapabilityValues{"arm64"}), + }), + )) + }) + }) + + When("parent CloudProfile has no capability definitions", func() { + BeforeEach(func() { + cloudProfile.Spec.MachineCapabilities = nil + }) + + It("should preserve existing architectures", func() { + namespacedCloudProfile.Spec.MachineImages = []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "20.04"}, + Architectures: []string{"amd64", "arm64"}, + }, + }, + }, + } + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + version := namespacedCloudProfile.Status.CloudProfileSpec.MachineImages[0].Versions[0] + Expect(version.Architectures).To(ConsistOf("amd64", "arm64")) + Expect(version.CapabilityFlavors).To(BeNil()) + }) + + It("should clear capabilities for machine types", func() { + namespacedCloudProfile.Spec.MachineTypes = []gardencorev1beta1.MachineType{ + { + Name: "m5.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{"amd64"}, + "gpu": []string{"nvidia"}, + }, + }, + } + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + machineType := namespacedCloudProfile.Status.CloudProfileSpec.MachineTypes[0] + Expect(machineType.Architecture).To(Equal(ptr.To("amd64"))) + Expect(machineType.Capabilities).To(BeNil()) + }) + + It("should handle multiple capability flavors with different architectures", func() { + namespacedCloudProfile.Spec.MachineImages = []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "20.04"}, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{"amd64"}, + }, + }, + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{"arm64"}, + }, + }, + }, + }, + }, + }, + } + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + version := namespacedCloudProfile.Status.CloudProfileSpec.MachineImages[0].Versions[0] + Expect(version.Architectures).To(ConsistOf("amd64", "arm64")) + Expect(version.CapabilityFlavors).To(BeNil()) + }) + }) + + Context("edge cases", func() { + It("should handle empty machine images list", func() { + namespacedCloudProfile.Spec.MachineImages = []gardencorev1beta1.MachineImage{} + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + Expect(namespacedCloudProfile.Status.CloudProfileSpec.MachineImages).To(BeEmpty()) + }) + + It("should handle empty machine types list", func() { + namespacedCloudProfile.Spec.MachineTypes = []gardencorev1beta1.MachineType{} + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + Expect(namespacedCloudProfile.Status.CloudProfileSpec.MachineTypes).To(BeEmpty()) + }) + + It("should handle machine image versions with no architectures and no capability flavors", func() { + cloudProfile.Spec.MachineCapabilities = []gardencorev1beta1.CapabilityDefinition{ + {Name: "architecture", Values: []string{"amd64"}}, + } + namespacedCloudProfile.Spec.MachineImages = []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "20.04"}, + }, + }, + }, + } + + namespacedcloudprofilecontroller.MergeCloudProfiles(namespacedCloudProfile, cloudProfile) + + version := namespacedCloudProfile.Status.CloudProfileSpec.MachineImages[0].Versions[0] + // no architectures specified so should default to amd64 as per parent capability definition + Expect(version.CapabilityFlavors).To(BeEmpty()) + Expect(version.Architectures).To(ConsistOf("amd64")) + }) + }) + }) }) }) }) diff --git a/pkg/provider-local/admission/mutator/namespacedcloudprofile.go b/pkg/provider-local/admission/mutator/namespacedcloudprofile.go index f20ba7f7822..afd3c5e656f 100644 --- a/pkg/provider-local/admission/mutator/namespacedcloudprofile.go +++ b/pkg/provider-local/admission/mutator/namespacedcloudprofile.go @@ -57,7 +57,10 @@ func (p *namespacedCloudProfile) Mutate(_ context.Context, newObj, _ client.Obje return fmt.Errorf("could not decode providerConfig of namespacedCloudProfile status for '%s': %w", profile.Name, err) } - statusConfig.MachineImages = mergeMachineImages(specConfig.MachineImages, statusConfig.MachineImages) + // TODO(Roncossek): Remove TransformProviderConfigToParentFormat once all CloudProfiles have been migrated to use CapabilityFlavors and the Architecture fields are effectively forbidden or have been removed. + uniformSpecConfig := TransformProviderConfigToParentFormat(specConfig, profile.Status.CloudProfileSpec.MachineCapabilities) + + statusConfig.MachineImages = mergeMachineImages(uniformSpecConfig.MachineImages, statusConfig.MachineImages) modifiedStatusConfig, err := json.Marshal(statusConfig) if err != nil { @@ -68,6 +71,32 @@ func (p *namespacedCloudProfile) Mutate(_ context.Context, newObj, _ client.Obje return nil } +// TransformProviderConfigToParentFormat supports the migration from the deprecated architecture fields to architecture capabilities during a transition period. +// Depending on whether the parent CloudProfile is in capability format or not, it transforms the given config to +// the capability format or the deprecated architecture fields format respectively. +// It assumes that the given config is either completely in the capability format or in the deprecated architecture fields format. +func TransformProviderConfigToParentFormat(cloudProfileConfig *v1alpha1.CloudProfileConfig, capabilityDefinitions []gardencorev1beta1.CapabilityDefinition) *v1alpha1.CloudProfileConfig { + isParentInCapabilityFormat := len(capabilityDefinitions) != 0 + for idx, machineImage := range cloudProfileConfig.MachineImages { + cloudProfileConfig.MachineImages[idx].Versions = make([]v1alpha1.MachineImageVersion, 0, len(machineImage.Versions)) + for _, version := range machineImage.Versions { + isVersionInCapabilityFormat := len(version.CapabilityFlavors) != 0 + transformedVersion := v1alpha1.MachineImageVersion{Version: version.Version} + if isParentInCapabilityFormat && !isVersionInCapabilityFormat { + // transform to capability format + transformedVersion.CapabilityFlavors = []v1alpha1.MachineImageFlavor{{Image: version.Image}} + } else if !isParentInCapabilityFormat && isVersionInCapabilityFormat { + // transform to old format + transformedVersion.Image = version.CapabilityFlavors[0].Image + } else { + transformedVersion = version + } + cloudProfileConfig.MachineImages[idx].Versions = append(cloudProfileConfig.MachineImages[idx].Versions, transformedVersion) + } + } + return cloudProfileConfig +} + func mergeMachineImages(specMachineImages, statusMachineImages []v1alpha1.MachineImages) []v1alpha1.MachineImages { specImages := utils.CreateMapFromSlice(specMachineImages, func(mi v1alpha1.MachineImages) string { return mi.Name }) statusImages := utils.CreateMapFromSlice(statusMachineImages, func(mi v1alpha1.MachineImages) string { return mi.Name }) diff --git a/pkg/provider-local/admission/mutator/namespacedcloudprofile_test.go b/pkg/provider-local/admission/mutator/namespacedcloudprofile_test.go index 475275ba592..f5184874816 100644 --- a/pkg/provider-local/admission/mutator/namespacedcloudprofile_test.go +++ b/pkg/provider-local/admission/mutator/namespacedcloudprofile_test.go @@ -21,10 +21,11 @@ import ( "github.com/gardener/gardener/extensions/pkg/util" extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook" - "github.com/gardener/gardener/pkg/apis/core/v1beta1" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" "github.com/gardener/gardener/pkg/provider-local/admission/mutator" api "github.com/gardener/gardener/pkg/provider-local/apis/local" "github.com/gardener/gardener/pkg/provider-local/apis/local/install" + "github.com/gardener/gardener/pkg/provider-local/apis/local/v1alpha1" "github.com/gardener/gardener/pkg/utils/test" . "github.com/gardener/gardener/pkg/utils/test/matchers" ) @@ -38,13 +39,13 @@ var _ = Describe("NamespacedCloudProfile Mutator", func() { decoder runtime.Decoder namespacedCloudProfileMutator extensionswebhook.Mutator - namespacedCloudProfile *v1beta1.NamespacedCloudProfile + namespacedCloudProfile *gardencorev1beta1.NamespacedCloudProfile ) BeforeEach(func() { scheme := runtime.NewScheme() utilruntime.Must(install.AddToScheme(scheme)) - utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(gardencorev1beta1.AddToScheme(scheme)) fakeClient = fakeclient.NewClientBuilder().WithScheme(scheme).Build() fakeManager = &test.FakeManager{ Client: fakeClient, @@ -54,7 +55,7 @@ var _ = Describe("NamespacedCloudProfile Mutator", func() { decoder = serializer.NewCodecFactory(fakeManager.GetScheme(), serializer.EnableStrict).UniversalDecoder() namespacedCloudProfileMutator = mutator.NewNamespacedCloudProfileMutator(fakeManager) - namespacedCloudProfile = &v1beta1.NamespacedCloudProfile{ + namespacedCloudProfile = &gardencorev1beta1.NamespacedCloudProfile{ ObjectMeta: metav1.ObjectMeta{ Name: "profile-1", Namespace: namespace, @@ -62,6 +63,81 @@ var _ = Describe("NamespacedCloudProfile Mutator", func() { } }) + Describe("TransformProviderConfigToParentFormat", func() { + var parentCloudProfile *gardencorev1beta1.CloudProfile + var cpConfig *v1alpha1.CloudProfileConfig + var capabilityMachineImage v1alpha1.MachineImages + var legacyMachineImage v1alpha1.MachineImages + + BeforeEach(func() { + parentCloudProfile = &gardencorev1beta1.CloudProfile{ + Spec: gardencorev1beta1.CloudProfileSpec{}, + } + cpConfig = &v1alpha1.CloudProfileConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: "CloudProfileConfig", + APIVersion: "local.provider.extensions.gardener.cloud/v1alpha1", + }, + } + capabilityMachineImage = v1alpha1.MachineImages{ + Name: "image-1", + Versions: []v1alpha1.MachineImageVersion{{ + Version: "1.0", + CapabilityFlavors: []v1alpha1.MachineImageFlavor{{ + Image: "local/image:1.0-amd64", + }}, + }}, + } + legacyMachineImage = v1alpha1.MachineImages{ + Name: "image-1", + Versions: []v1alpha1.MachineImageVersion{{ + Version: "1.0", + Image: "local/image:1.0-amd64", + }}, + } + }) + + When("the parent CloudProfile has machineCapabilities defined", func() { + BeforeEach(func() { + parentCloudProfile.Spec.MachineCapabilities = []gardencorev1beta1.CapabilityDefinition{ + {Name: "architecture", Values: []string{"amd64", "arm64"}}, + } + }) + + It("should do nothing if the NamespacedCloudProfile spec is in capability format", func() { + machineImages := []v1alpha1.MachineImages{capabilityMachineImage} + cpConfig.MachineImages = machineImages + + uniformSpecConfig := mutator.TransformProviderConfigToParentFormat(cpConfig, parentCloudProfile.Spec.MachineCapabilities) + Expect(uniformSpecConfig.MachineImages).To(ContainElements(machineImages)) + Expect(uniformSpecConfig).To(BeEquivalentTo(cpConfig)) + }) + + It("should transform the status to capability format if the NamespacedCloudProfile spec is in old format", func() { + cpConfig.MachineImages = []v1alpha1.MachineImages{legacyMachineImage} + uniformSpecConfig := mutator.TransformProviderConfigToParentFormat(cpConfig, parentCloudProfile.Spec.MachineCapabilities) + Expect(uniformSpecConfig.MachineImages[0]).To(Equal(capabilityMachineImage)) + }) + }) + + When("the parentCloudProfile has NO machineCapabilities defined", func() { + It("should do nothing if the NamespacedCloudProfile spec is in legacy format", func() { + machineImages := []v1alpha1.MachineImages{legacyMachineImage} + cpConfig.MachineImages = machineImages + + uniformSpecConfig := mutator.TransformProviderConfigToParentFormat(cpConfig, parentCloudProfile.Spec.MachineCapabilities) + Expect(uniformSpecConfig.MachineImages).To(ContainElements(machineImages)) + Expect(uniformSpecConfig).To(BeEquivalentTo(cpConfig)) + }) + + It("should transform the status to legacy format if the NamespacedCloudProfile spec is in capability format", func() { + cpConfig.MachineImages = []v1alpha1.MachineImages{capabilityMachineImage} + uniformSpecConfig := mutator.TransformProviderConfigToParentFormat(cpConfig, parentCloudProfile.Spec.MachineCapabilities) + Expect(uniformSpecConfig.MachineImages[0]).To(Equal(legacyMachineImage)) + }) + }) + }) + Describe("#Mutate", func() { It("should succeed for NamespacedCloudProfile without provider config", func() { Expect(namespacedCloudProfileMutator.Mutate(ctx, namespacedCloudProfile, nil)).To(Succeed()) diff --git a/pkg/provider-local/admission/validator/cloudprofile.go b/pkg/provider-local/admission/validator/cloudprofile.go index 692dd3a2f90..91c0d1af6bb 100644 --- a/pkg/provider-local/admission/validator/cloudprofile.go +++ b/pkg/provider-local/admission/validator/cloudprofile.go @@ -48,9 +48,11 @@ func (cp *cloudProfileValidator) Validate(_ context.Context, newObj, _ client.Ob if err != nil { return fmt.Errorf("could not decode providerConfig of CloudProfile %q: %w", cloudProfile.Name, err) } - CapabilityDefinition, err := helper.ConvertV1beta1CapabilityDefinitions(cloudProfile.Spec.MachineCapabilities) + + capabilityDefinitions, err := helper.ConvertV1beta1CapabilityDefinitions(cloudProfile.Spec.MachineCapabilities) if err != nil { return field.InternalError(field.NewPath("spec").Child("machineCapabilities"), err) } - return validation.ValidateCloudProfileConfig(cpConfig, cloudProfile.Spec.MachineImages, CapabilityDefinition, providerConfigPath).ToAggregate() + + return validation.ValidateCloudProfileConfig(cpConfig, cloudProfile.Spec.MachineImages, capabilityDefinitions, providerConfigPath).ToAggregate() } diff --git a/pkg/provider-local/admission/validator/namespacedcloudprofile.go b/pkg/provider-local/admission/validator/namespacedcloudprofile.go index a5c4afcef0c..781a562da85 100644 --- a/pkg/provider-local/admission/validator/namespacedcloudprofile.go +++ b/pkg/provider-local/admission/validator/namespacedcloudprofile.go @@ -15,11 +15,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook" + gardencoreapi "github.com/gardener/gardener/pkg/api" "github.com/gardener/gardener/pkg/apis/core" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/provider-local/admission" + "github.com/gardener/gardener/pkg/provider-local/admission/mutator" api "github.com/gardener/gardener/pkg/provider-local/apis/local" + "github.com/gardener/gardener/pkg/provider-local/apis/local/helper" + "github.com/gardener/gardener/pkg/provider-local/apis/local/v1alpha1" "github.com/gardener/gardener/pkg/provider-local/apis/local/validation" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -66,6 +70,10 @@ func (p *namespacedCloudProfileValidator) Validate(ctx context.Context, new, _ c return err } + if err := SimulateTransformToParentFormat(cloudProfileConfig, cloudProfile, parentProfile.Spec.MachineCapabilities); err != nil { + return err + } + return p.validateNamespacedCloudProfileProviderConfig(cloudProfileConfig, cloudProfile.Spec.MachineImages, parentProfile).ToAggregate() } @@ -130,3 +138,32 @@ func (p *namespacedCloudProfileValidator) validateNamespacedCloudProfileProvider return allErrs } + +// SimulateTransformToParentFormat simulates the transformation of the given NamespacedCloudProfile and its providerConfig +// to the parent CloudProfile format. This includes the transformation of both the providerConfig and the spec. +func SimulateTransformToParentFormat(cloudProfileConfig *api.CloudProfileConfig, cloudProfile *core.NamespacedCloudProfile, capabilityDefinitions []gardencorev1beta1.CapabilityDefinition) error { + cloudProfileConfigV1alpha1 := &v1alpha1.CloudProfileConfig{} + path := field.NewPath("spec").Child("providerConfig") + + if err := helper.Scheme.Convert(cloudProfileConfig, cloudProfileConfigV1alpha1, nil); err != nil { + return field.InternalError(path, err) + } + namespacedCloudProfileSpecV1beta1 := gardencorev1beta1.NamespacedCloudProfileSpec{} + if err := gardencoreapi.Scheme.Convert(&cloudProfile.Spec, &namespacedCloudProfileSpecV1beta1, nil); err != nil { + return field.InternalError(path, err) + } + + // simulate transformation to parent spec format + // - performed in mutating extension webhook + transformedSpecConfig := mutator.TransformProviderConfigToParentFormat(cloudProfileConfigV1alpha1, capabilityDefinitions) + // - performed in namespaced cloud profile controller + transformedSpec := gardenerutils.TransformSpecToParentFormat(namespacedCloudProfileSpecV1beta1, capabilityDefinitions) + + if err := helper.Scheme.Convert(transformedSpecConfig, cloudProfileConfig, nil); err != nil { + return field.InternalError(path, err) + } + if err := gardencoreapi.Scheme.Convert(&transformedSpec, &cloudProfile.Spec, nil); err != nil { + return field.InternalError(path, err) + } + return nil +} diff --git a/pkg/provider-local/apis/local/v1alpha1/types_cloudprofile.go b/pkg/provider-local/apis/local/v1alpha1/types_cloudprofile.go index 872f004b13c..7b95dad5ea3 100644 --- a/pkg/provider-local/apis/local/v1alpha1/types_cloudprofile.go +++ b/pkg/provider-local/apis/local/v1alpha1/types_cloudprofile.go @@ -36,7 +36,8 @@ type MachineImageVersion struct { // Version is the version of the image. Version string `json:"version"` // Image is the image for the machine image. - Image string `json:"image"` + // +optional + Image string `json:"image,omitempty"` // CapabilityFlavors contains provider-specific image identifiers of this version with their capabilities. CapabilityFlavors []MachineImageFlavor `json:"capabilityFlavors"` } diff --git a/pkg/provider-local/apis/local/validation/cloudprofile.go b/pkg/provider-local/apis/local/validation/cloudprofile.go index 440b74fa081..eacaa36752c 100644 --- a/pkg/provider-local/apis/local/validation/cloudprofile.go +++ b/pkg/provider-local/apis/local/validation/cloudprofile.go @@ -149,8 +149,7 @@ func validateMachineImageMapping(coreMachineImages []core.MachineImage, machineI // validate that for each machine image there is a corresponding cpConfig image if _, existsInConfig := providerImages.GetImage(machineImage.Name); !existsInConfig { - allErrs = append(allErrs, field.Required(machineImagePath, - fmt.Sprintf("must provide an image mapping for image %q in providerConfig", machineImage.Name))) + allErrs = append(allErrs, field.Required(machineImagePath, fmt.Sprintf("must provide an image mapping for image %q in providerConfig", machineImage.Name))) continue } @@ -177,8 +176,7 @@ func validateMachineImageVersionMapping(machineImage core.MachineImage, provider imageVersion, exists := providerImages.GetImageVersion(machineImage.Name, version.Version) if !exists { allErrs = append(allErrs, field.Required(machineImageVersionPath, - fmt.Sprintf("machine image version %s@%s is not defined in the providerConfig", - machineImage.Name, version.Version), + fmt.Sprintf("machine image version %s@%s is not defined in the providerConfig", machineImage.Name, version.Version), )) continue // Skip further validation if version doesn't exist } @@ -204,7 +202,7 @@ func validateImageFlavorMapping(imageName string, version core.MachineImageVersi isFound := false // search for the corresponding imageVersion.MachineImageFlavor for _, providerFlavor := range imageVersion.CapabilityFlavors { - providerDefaultedCapabilities := v1beta1helper.GetCapabilitiesWithAppliedDefaults(providerFlavor.Capabilities, capabilityDefinitions) + providerDefaultedCapabilities := gardencorev1beta1.GetCapabilitiesWithAppliedDefaults(providerFlavor.Capabilities, capabilityDefinitions) if v1beta1helper.AreCapabilitiesEqual(coreDefaultedFlavor.Capabilities, providerDefaultedCapabilities) { isFound = true break diff --git a/pkg/provider-local/controller/worker/machine_images.go b/pkg/provider-local/controller/worker/machine_images.go index deecc266075..62ee1654ac1 100644 --- a/pkg/provider-local/controller/worker/machine_images.go +++ b/pkg/provider-local/controller/worker/machine_images.go @@ -93,10 +93,10 @@ func appendMachineImage(machineImages []api.MachineImage, machineImage api.Machi }) } - defaultedCapabilities := v1beta1helper.GetCapabilitiesWithAppliedDefaults(machineImage.Capabilities, capabilityDefinitions) + defaultedCapabilities := gardencorev1beta1.GetCapabilitiesWithAppliedDefaults(machineImage.Capabilities, capabilityDefinitions) for _, existingMachineImage := range machineImages { - existingDefaultedCapabilities := v1beta1helper.GetCapabilitiesWithAppliedDefaults(existingMachineImage.Capabilities, capabilityDefinitions) + existingDefaultedCapabilities := gardencorev1beta1.GetCapabilitiesWithAppliedDefaults(existingMachineImage.Capabilities, capabilityDefinitions) if existingMachineImage.Name == machineImage.Name && existingMachineImage.Version == machineImage.Version && v1beta1helper.AreCapabilitiesEqual(defaultedCapabilities, existingDefaultedCapabilities) { diff --git a/pkg/utils/gardener/cloudprofile.go b/pkg/utils/gardener/cloudprofile.go index 1355adede2c..5630d1d2b7e 100644 --- a/pkg/utils/gardener/cloudprofile.go +++ b/pkg/utils/gardener/cloudprofile.go @@ -289,92 +289,111 @@ func SyncCloudProfileFields(oldShoot, newShoot *core.Shoot) { // SyncArchitectureCapabilityFields syncs the architecture capabilities and the architecture fields. func SyncArchitectureCapabilityFields(newCloudProfileSpec core.CloudProfileSpec, oldCloudProfileSpec core.CloudProfileSpec) { - hasCapabilities := len(newCloudProfileSpec.MachineCapabilities) > 0 - if !hasCapabilities || !gardencorehelper.HasCapability(newCloudProfileSpec.MachineCapabilities, v1beta1constants.ArchitectureName) { - return + hasNewCapabilities := len(newCloudProfileSpec.MachineCapabilities) > 0 + hasOldCapabilities := len(oldCloudProfileSpec.MachineCapabilities) > 0 + isInitialMigration := !hasOldCapabilities && hasNewCapabilities + + if isInitialMigration { + machineCapabilities := gardencorehelper.CapabilityDefinitionsToCapabilities(newCloudProfileSpec.MachineCapabilities) + numberOfCloudProfileArchitectures := len(machineCapabilities[v1beta1constants.ArchitectureName]) + // syncing is only required if there is more than 1 architecture in spec.machineCapabilities + // 0: means the MachineCapabilities are invalid and will be caught by validation + // 1: means we have only one architecture and no syncing is required + if numberOfCloudProfileArchitectures > 1 { + // During the initial migration to capabilities, synchronize the legacy architecture fields with the architecture capability values. + // Any mismatch between capabilities and architecture fields will result in a validation error. + syncMachineImageArchitectureCapabilities(newCloudProfileSpec.MachineImages) + syncMachineTypeArchitectureCapabilities(newCloudProfileSpec.MachineTypes) + } + } + if hasNewCapabilities || hasOldCapabilities { + // sync back the legacy architecture fields from the capabilities on every update + // or if only the old cloud profile has capabilities defined assign them to nil to allow switching back to legacy mode + syncMachineImageLegacyArchitecture(newCloudProfileSpec.MachineImages, newCloudProfileSpec.MachineCapabilities) + syncMachineTypeLegacyArchitecture(newCloudProfileSpec.MachineTypes, newCloudProfileSpec.MachineCapabilities) } - - isInitialMigration := hasCapabilities && len(oldCloudProfileSpec.MachineCapabilities) == 0 - - // During the initial migration to capabilities, synchronize the architecture fields with the capability definitions. - // After the migration, only sync architectures from the capability definitions back to the architecture fields. - // This approach ensures that capabilities are consistently used once defined. - // Any mismatch between capabilities and architecture fields will result in a validation error. - syncMachineImageArchitectureCapabilities(newCloudProfileSpec.MachineImages, oldCloudProfileSpec.MachineImages, isInitialMigration) - syncMachineTypeArchitectureCapabilities(newCloudProfileSpec.MachineTypes, oldCloudProfileSpec.MachineTypes, isInitialMigration) } -func syncMachineImageArchitectureCapabilities(newMachineImages, oldMachineImages []core.MachineImage, isInitialMigration bool) { - oldMachineImagesMap := NewCoreImagesContext(oldMachineImages) - - for imageIdx, image := range newMachineImages { - for versionIdx, version := range newMachineImages[imageIdx].Versions { - oldMachineImageVersion, oldVersionExists := oldMachineImagesMap.GetImageVersion(image.Name, version.Version) - capabilityArchitectures := gardencorehelper.ExtractArchitecturesFromImageFlavors(version.CapabilityFlavors) - - // Skip any architecture field syncing if - // - architecture field has been modified and changed to any value other than empty. - architecturesFieldHasBeenChanged := oldVersionExists && len(version.Architectures) > 0 && - !apiequality.Semantic.DeepEqual(oldMachineImageVersion.Architectures, version.Architectures) - - // - both the architecture field and the architecture capability are empty or filled equally. - if architecturesFieldHasBeenChanged || slices.Equal(capabilityArchitectures, version.Architectures) { +func syncMachineImageLegacyArchitecture(newMachineImages []core.MachineImage, capabilityDefinitions []core.CapabilityDefinition) { + for imageIdx := range newMachineImages { + for versionIdx, imageVersion := range newMachineImages[imageIdx].Versions { + // If there are no capability definitions, clear capability flavors. + if len(capabilityDefinitions) == 0 { + newMachineImages[imageIdx].Versions[versionIdx].CapabilityFlavors = nil continue } - // Sync architecture field to capabilities if filled on initial migration. - if isInitialMigration && len(version.Architectures) > 0 && len(version.CapabilityFlavors) == 0 { - for _, architecture := range version.Architectures { - newMachineImages[imageIdx].Versions[versionIdx].CapabilityFlavors = append(newMachineImages[imageIdx].Versions[versionIdx].CapabilityFlavors, - core.MachineImageFlavor{ - Capabilities: core.Capabilities{ - v1beta1constants.ArchitectureName: []string{architecture}, - }, - }) - } + // don't sync if architectures are set by the user + if len(imageVersion.Architectures) > 0 { continue } - // Sync capability architectures to architectures field. - if len(capabilityArchitectures) > 0 { - newMachineImages[imageIdx].Versions[versionIdx].Architectures = capabilityArchitectures + defaultedImageFlavors := gardencorehelper.GetImageFlavorsWithAppliedDefaults(imageVersion.CapabilityFlavors, capabilityDefinitions) + defaultArchitectures := gardencorehelper.ExtractArchitecturesFromImageFlavors(defaultedImageFlavors) + if len(defaultArchitectures) > 0 { + newMachineImages[imageIdx].Versions[versionIdx].Architectures = defaultArchitectures } } } } -func syncMachineTypeArchitectureCapabilities(newMachineTypes, oldMachineTypes []core.MachineType, isInitialMigration bool) { - oldMachineTypesMap := utils.CreateMapFromSlice(oldMachineTypes, func(machineType core.MachineType) string { return machineType.Name }) - +func syncMachineTypeLegacyArchitecture(newMachineTypes []core.MachineType, capabilityDefinitions []core.CapabilityDefinition) { for i, machineType := range newMachineTypes { - oldMachineType, oldMachineTypeExists := oldMachineTypesMap[machineType.Name] - architectureValue := ptr.Deref(machineType.Architecture, "") - oldArchitectureValue := ptr.Deref(oldMachineType.Architecture, "") - capabilityArchitectures := machineType.Capabilities[v1beta1constants.ArchitectureName] - - // Skip any architecture field syncing if - // - architecture field has been modified and changed to any value other than empty. - architectureFieldHasBeenChanged := oldMachineTypeExists && architectureValue != "" && - (oldArchitectureValue == "" || oldArchitectureValue != architectureValue) - // - both the architecture field and the architecture capability are empty or filled equally. - architecturesInSync := len(capabilityArchitectures) == 0 && architectureValue == "" || - len(capabilityArchitectures) == 1 && capabilityArchitectures[0] == architectureValue - if architectureFieldHasBeenChanged || architecturesInSync { + // If there are no capability definitions, clear capability flavors. + if len(capabilityDefinitions) == 0 { + newMachineTypes[i].Capabilities = nil continue } - // Sync architecture field to capabilities if filled on initial migration. - if isInitialMigration && architectureValue != "" && len(capabilityArchitectures) == 0 { - if newMachineTypes[i].Capabilities == nil { - newMachineTypes[i].Capabilities = make(core.Capabilities) - } - newMachineTypes[i].Capabilities[v1beta1constants.ArchitectureName] = []string{architectureValue} + // don't sync if architecture is set by the user + if machineType.Architecture != nil { continue } // Sync capability architecture to architecture field. - if len(capabilityArchitectures) == 1 { - newMachineTypes[i].Architecture = ptr.To(capabilityArchitectures[0]) + defaultedCapabilities := core.GetCapabilitiesWithAppliedDefaults(newMachineTypes[i].Capabilities, capabilityDefinitions) + if len(defaultedCapabilities[v1beta1constants.ArchitectureName]) == 1 { + newMachineTypes[i].Architecture = ptr.To(defaultedCapabilities[v1beta1constants.ArchitectureName][0]) + } + } +} + +func syncMachineImageArchitectureCapabilities(newMachineImages []core.MachineImage) { + for imageIdx := range newMachineImages { + for versionIdx, version := range newMachineImages[imageIdx].Versions { + // don't sync if capabilities are set by the user + if len(version.CapabilityFlavors) > 0 { + continue + } + + legacyArchitectures := version.Architectures + if len(legacyArchitectures) == 0 { + // default to AMD64 if empty as this is the default for the architecture field + legacyArchitectures = []string{v1beta1constants.ArchitectureAMD64} + } + for _, architecture := range legacyArchitectures { + newMachineImages[imageIdx].Versions[versionIdx].CapabilityFlavors = append(newMachineImages[imageIdx].Versions[versionIdx].CapabilityFlavors, + core.MachineImageFlavor{Capabilities: core.Capabilities{ + v1beta1constants.ArchitectureName: []string{architecture}, + }}, + ) + } + } + } +} + +func syncMachineTypeArchitectureCapabilities(newMachineTypes []core.MachineType) { + for i, machineType := range newMachineTypes { + // don't sync if capabilities are set by the user + if len(machineType.Capabilities) > 0 { + continue + } + + // default to AMD64 if empty as this is the default for the architecture field + legacyArchitecture := ptr.Deref(machineType.Architecture, v1beta1constants.ArchitectureAMD64) + + newMachineTypes[i].Capabilities = core.Capabilities{ + v1beta1constants.ArchitectureName: []string{legacyArchitecture}, } } } diff --git a/pkg/utils/gardener/cloudprofile_test.go b/pkg/utils/gardener/cloudprofile_test.go index 47ef61fee76..a9e919664d6 100644 --- a/pkg/utils/gardener/cloudprofile_test.go +++ b/pkg/utils/gardener/cloudprofile_test.go @@ -748,6 +748,9 @@ var _ = Describe("CloudProfile", func() { BeforeEach(func() { cloudProfileSpecNew = core.CloudProfileSpec{ + MachineCapabilities: []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"arm64", "amd64", "custom"}}, + }, MachineImages: []core.MachineImage{ {Versions: []core.MachineImageVersion{{}}}, }, @@ -755,51 +758,102 @@ var _ = Describe("CloudProfile", func() { {}, }, } - cloudProfileSpecOld = cloudProfileSpecNew + cloudProfileSpecOld = *cloudProfileSpecNew.DeepCopy() + cloudProfileSpecOld.MachineCapabilities = []core.CapabilityDefinition{} }) - Describe("Initial migration", func() { - BeforeEach(func() { - cloudProfileSpecNew.MachineCapabilities = []core.CapabilityDefinition{ - {Name: "architecture", Values: []string{"arm64", "amd64", "custom"}}, - } - }) + It("should do nothing if its NOT the initial migration", func() { + // its no initial migration if both old and new have machineCapabilities defined + cloudProfileSpecOld.MachineCapabilities = []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"arm64", "amd64"}}, + } - It("It should do nothing for empty architectures and empty capabilities", func() { - cloudProfileSpecNewBefore := cloudProfileSpecNew - // With the update, the old fields are unset: - cloudProfileSpecOld.MachineImages[0].Versions[0].Architectures = []string{"amd64"} - cloudProfileSpecOld.MachineTypes[0].Architecture = ptr.To("amd64") + cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures = []string{"amd64"} + cloudProfileSpecNew.MachineTypes[0].Architecture = ptr.To("amd64") + initialNewSpec := *cloudProfileSpecNew.DeepCopy() - gardenerutils.SyncArchitectureCapabilityFields(cloudProfileSpecNew, cloudProfileSpecOld) + gardenerutils.SyncArchitectureCapabilityFields(cloudProfileSpecNew, cloudProfileSpecOld) + Expect(cloudProfileSpecNew).To(Equal(initialNewSpec)) + }) - Expect(cloudProfileSpecNew).To(Equal(cloudProfileSpecNewBefore)) - }) + It("should do nothing if less than two architectures are supported", func() { + cloudProfileSpecNew.MachineCapabilities = []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"amd64"}}, + } + cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures = []string{"amd64"} + cloudProfileSpecNew.MachineTypes[0].Architecture = ptr.To("amd64") - It("It should correctly handle split-up machine image version capability architectures", func() { - cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors = []core.MachineImageFlavor{ - {Capabilities: core.Capabilities{"architecture": []string{"custom"}}}, - {Capabilities: core.Capabilities{"architecture": []string{"amd64"}}}, - {Capabilities: core.Capabilities{"architecture": []string{"arm64"}}}, - } + initialNewSpec := *cloudProfileSpecNew.DeepCopy() - gardenerutils.SyncArchitectureCapabilityFields(cloudProfileSpecNew, cloudProfileSpecOld) + gardenerutils.SyncArchitectureCapabilityFields(cloudProfileSpecNew, cloudProfileSpecOld) + Expect(cloudProfileSpecNew).To(Equal(initialNewSpec)) + }) - Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures).To(ConsistOf("amd64", "arm64", "custom")) - }) + It("should create capabilities and capabilityFlavors from architecture fields if they are empty", func() { + cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures = []string{"amd64", "arm64"} + cloudProfileSpecNew.MachineTypes[0].Architecture = ptr.To("arm64") - It("It should sync filled architecture fields to empty capabilities", func() { - cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures = []string{"amd64", "arm64"} - cloudProfileSpecNew.MachineTypes[0].Architecture = ptr.To("amd64") + gardenerutils.SyncArchitectureCapabilityFields(cloudProfileSpecNew, cloudProfileSpecOld) - gardenerutils.SyncArchitectureCapabilityFields(cloudProfileSpecNew, cloudProfileSpecOld) + Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures).To(Equal([]string{"amd64", "arm64"})) + Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors[0].Capabilities["architecture"]).To(BeEquivalentTo([]string{"amd64"})) + Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors[1].Capabilities["architecture"]).To(BeEquivalentTo([]string{"arm64"})) + Expect(cloudProfileSpecNew.MachineTypes[0].Architecture).To(Equal(ptr.To("arm64"))) + Expect(cloudProfileSpecNew.MachineTypes[0].Capabilities["architecture"]).To(BeEquivalentTo([]string{"arm64"})) + }) - Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures).To(Equal([]string{"amd64", "arm64"})) - Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors[0].Capabilities["architecture"]).To(BeEquivalentTo([]string{"amd64"})) - Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors[1].Capabilities["architecture"]).To(BeEquivalentTo([]string{"arm64"})) - Expect(cloudProfileSpecNew.MachineTypes[0].Architecture).To(Equal(ptr.To("amd64"))) - Expect(cloudProfileSpecNew.MachineTypes[0].Capabilities["architecture"]).To(BeEquivalentTo([]string{"amd64"})) + It("should not write to capabilities if they are already set", func() { + cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures = []string{"amd64", "arm64"} + cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors = []core.MachineImageFlavor{ + {Capabilities: core.Capabilities{"architecture": []string{"custom"}}}, + } + cloudProfileSpecNew.MachineTypes[0].Architecture = ptr.To("amd64") + cloudProfileSpecNew.MachineTypes[0].Capabilities = core.Capabilities{ + "architecture": []string{"custom"}, + } + cloudProfileSpecNew.MachineTypes = append(cloudProfileSpecNew.MachineTypes, core.MachineType{ + Architecture: ptr.To("amd64"), }) + + gardenerutils.SyncArchitectureCapabilityFields(cloudProfileSpecNew, cloudProfileSpecOld) + + Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures).To(Equal([]string{"amd64", "arm64"})) + Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors).To(HaveLen(1)) + Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors[0].Capabilities["architecture"]).To(BeEquivalentTo([]string{"custom"})) + Expect(cloudProfileSpecNew.MachineTypes[0].Architecture).To(Equal(ptr.To("amd64"))) + Expect(cloudProfileSpecNew.MachineTypes[0].Capabilities["architecture"]).To(BeEquivalentTo([]string{"custom"})) + Expect(cloudProfileSpecNew.MachineTypes[1].Architecture).To(Equal(ptr.To("amd64"))) + Expect(cloudProfileSpecNew.MachineTypes[1].Capabilities["architecture"]).To(BeEquivalentTo([]string{"amd64"})) + }) + + It("should use the default architecture amd64 if architecture field is empty and capabilities are empty", func() { + gardenerutils.SyncArchitectureCapabilityFields(cloudProfileSpecNew, cloudProfileSpecOld) + Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].Architectures).To(Equal([]string{"amd64"})) + Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors[0].Capabilities["architecture"]).To(BeEquivalentTo([]string{"amd64"})) + Expect(cloudProfileSpecNew.MachineTypes[0].Architecture).To(Equal(ptr.To("amd64"))) + Expect(cloudProfileSpecNew.MachineTypes[0].Capabilities["architecture"]).To(BeEquivalentTo([]string{"amd64"})) + }) + + It("should set capabilities and capabilityFlavors to nil on migration to legacy format", func() { + // migration to legacy is indicated by old spec having machineCapabilities defined + // and new spec having empty machineCapabilities + cloudProfileSpecOld.MachineCapabilities = []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"arm64", "amd64"}}, + } + cloudProfileSpecNew.MachineCapabilities = []core.CapabilityDefinition{} + + cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors = []core.MachineImageFlavor{ + {Capabilities: core.Capabilities{"architecture": []string{"amd64"}}}, + {Capabilities: core.Capabilities{"architecture": []string{"arm64"}}}, + } + cloudProfileSpecNew.MachineTypes[0].Capabilities = core.Capabilities{ + "architecture": []string{"arm64"}, + } + + gardenerutils.SyncArchitectureCapabilityFields(cloudProfileSpecNew, cloudProfileSpecOld) + + Expect(cloudProfileSpecNew.MachineImages[0].Versions[0].CapabilityFlavors).To(BeNil()) + Expect(cloudProfileSpecNew.MachineTypes[0].Capabilities).To(BeNil()) }) }) diff --git a/pkg/utils/gardener/namespacedcloudprofile.go b/pkg/utils/gardener/namespacedcloudprofile.go new file mode 100644 index 00000000000..780cdbad038 --- /dev/null +++ b/pkg/utils/gardener/namespacedcloudprofile.go @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package gardener + +import ( + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/utils/ptr" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" +) + +// TransformSpecToParentFormat ensures that the given NamespacedCloudProfileSpec is in a uniform format with its parent CloudProfileSpec. +// If the parent CloudProfileSpec uses capability definitions, then the NamespacedCloudProfileSpec is transformed to also use capabilities +// and vice versa. +// TODO(Roncossek): Remove TransformSpecToParentFormat once all CloudProfiles have been migrated to use CapabilityFlavors and the Architecture fields are effectively forbidden or have been removed. +func TransformSpecToParentFormat( + spec gardencorev1beta1.NamespacedCloudProfileSpec, + capabilityDefinitions []gardencorev1beta1.CapabilityDefinition, +) gardencorev1beta1.NamespacedCloudProfileSpec { + isParentInCapabilityFormat := len(capabilityDefinitions) > 0 + transformedSpec := spec.DeepCopy() + + // Normalize MachineImages + for idx, machineImage := range transformedSpec.MachineImages { + for idy, version := range machineImage.Versions { + legacyArchitectures := version.Architectures + + if isParentInCapabilityFormat && len(version.CapabilityFlavors) == 0 { + // Convert legacy architectures to capability flavors + version.CapabilityFlavors = []gardencorev1beta1.MachineImageFlavor{} + for _, arch := range legacyArchitectures { + version.CapabilityFlavors = append(version.CapabilityFlavors, gardencorev1beta1.MachineImageFlavor{ + Capabilities: gardencorev1beta1.Capabilities{ + v1beta1constants.ArchitectureName: []string{arch}, + }, + }) + } + version.Architectures = legacyArchitectures + } else if !isParentInCapabilityFormat { + // Convert capability flavors to legacy architectures + if len(legacyArchitectures) == 0 { + architectureSet := sets.New[string]() + if len(version.CapabilityFlavors) > 0 { + for _, flavor := range version.CapabilityFlavors { + architectureSet.Insert(flavor.Capabilities[v1beta1constants.ArchitectureName]...) + } + } + version.Architectures = architectureSet.UnsortedList() + } + version.CapabilityFlavors = nil + } + + transformedSpec.MachineImages[idx].Versions[idy] = version + } + } + + // Normalize MachineTypes + for idx, machineType := range transformedSpec.MachineTypes { + if isParentInCapabilityFormat { + if len(machineType.Capabilities) > 0 { + continue + } + architecture := machineType.GetArchitecture(capabilityDefinitions) + if architecture == "" { + architecture = ptr.Deref(machineType.Architecture, v1beta1constants.ArchitectureAMD64) + } + machineType.Capabilities = gardencorev1beta1.Capabilities{ + v1beta1constants.ArchitectureName: []string{architecture}, + } + } else { + if machineType.Architecture == nil { + if len(machineType.Capabilities) > 0 { + architecture := machineType.Capabilities[v1beta1constants.ArchitectureName] + if len(architecture) > 0 { + machineType.Architecture = &architecture[0] + } + } + } + machineType.Capabilities = nil + } + transformedSpec.MachineTypes[idx] = machineType + } + return *transformedSpec +} diff --git a/pkg/utils/gardener/namespacedcloudprofile_test.go b/pkg/utils/gardener/namespacedcloudprofile_test.go new file mode 100644 index 00000000000..d0bb148ce31 --- /dev/null +++ b/pkg/utils/gardener/namespacedcloudprofile_test.go @@ -0,0 +1,380 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package gardener_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + . "github.com/gardener/gardener/pkg/utils/gardener" +) + +var _ = Describe("TransformSpecToParentFormat", func() { + When("parent uses capabilities", func() { + var capabilityDefinitions []gardencorev1beta1.CapabilityDefinition + + BeforeEach(func() { + capabilityDefinitions = []gardencorev1beta1.CapabilityDefinition{ + { + Name: v1beta1constants.ArchitectureName, + Values: []string{"amd64", "arm64"}, + }, + } + }) + + It("should transform legacy architectures to capability flavors", func() { + spec := gardencorev1beta1.NamespacedCloudProfileSpec{ + MachineImages: []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: "20.04", + }, + Architectures: []string{"amd64", "arm64"}, + }, + }, + }, + }, + MachineTypes: []gardencorev1beta1.MachineType{ + { + Name: "m5.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + Architecture: ptr.To("amd64"), + }, + }, + } + + result := TransformSpecToParentFormat(spec, capabilityDefinitions) + + Expect(result.MachineImages).To(HaveLen(1)) + Expect(result.MachineImages[0].Versions).To(HaveLen(1)) + version := result.MachineImages[0].Versions[0] + Expect(version.Architectures).To(Equal([]string{"amd64", "arm64"})) + Expect(version.CapabilityFlavors).To(HaveLen(2)) + Expect(version.CapabilityFlavors[0].Capabilities[v1beta1constants.ArchitectureName]).To(BeEquivalentTo([]string{"amd64"})) + Expect(version.CapabilityFlavors[1].Capabilities[v1beta1constants.ArchitectureName]).To(BeEquivalentTo([]string{"arm64"})) + + Expect(result.MachineTypes).To(HaveLen(1)) + machineType := result.MachineTypes[0] + Expect(machineType.Architecture).To(Equal(ptr.To("amd64"))) + Expect(machineType.Capabilities[v1beta1constants.ArchitectureName]).To(BeEquivalentTo([]string{"amd64"})) + }) + + It("should preserve existing capability flavors when already present", func() { + spec := gardencorev1beta1.NamespacedCloudProfileSpec{ + MachineImages: []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: "20.04", + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + { + Capabilities: gardencorev1beta1.Capabilities{ + v1beta1constants.ArchitectureName: []string{"amd64"}, + }, + }, + }, + }, + }, + }, + }, + MachineTypes: []gardencorev1beta1.MachineType{ + { + Name: "m5.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + Capabilities: gardencorev1beta1.Capabilities{ + v1beta1constants.ArchitectureName: []string{"amd64"}, + }, + }, + }, + } + + result := TransformSpecToParentFormat(spec, capabilityDefinitions) + + Expect(result.MachineImages[0].Versions[0].CapabilityFlavors).To(HaveLen(1)) + Expect(result.MachineImages[0].Versions[0].CapabilityFlavors[0].Capabilities[v1beta1constants.ArchitectureName]).To(BeEquivalentTo([]string{"amd64"})) + Expect(result.MachineTypes[0].Capabilities[v1beta1constants.ArchitectureName]).To(BeEquivalentTo([]string{"amd64"})) + }) + + It("should default to AMD64 for machine types without architecture", func() { + spec := gardencorev1beta1.NamespacedCloudProfileSpec{ + MachineTypes: []gardencorev1beta1.MachineType{ + { + Name: "m5.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + }, + }, + } + + result := TransformSpecToParentFormat(spec, capabilityDefinitions) + + Expect(result.MachineTypes[0].Capabilities[v1beta1constants.ArchitectureName]).To(BeEquivalentTo([]string{"amd64"})) + }) + }) + + When("parent doesn't use capabilities", func() { + It("should transform capability flavors to legacy architectures", func() { + spec := gardencorev1beta1.NamespacedCloudProfileSpec{ + MachineImages: []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: "20.04", + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + { + Capabilities: gardencorev1beta1.Capabilities{ + v1beta1constants.ArchitectureName: []string{"amd64"}, + }, + }, + { + Capabilities: gardencorev1beta1.Capabilities{ + v1beta1constants.ArchitectureName: []string{"arm64"}, + }, + }, + }, + }, + }, + }, + }, + MachineTypes: []gardencorev1beta1.MachineType{ + { + Name: "m5.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + Capabilities: gardencorev1beta1.Capabilities{ + v1beta1constants.ArchitectureName: []string{"amd64"}, + }, + }, + }, + } + + result := TransformSpecToParentFormat(spec, nil) + + Expect(result.MachineImages[0].Versions[0].Architectures).To(ConsistOf("amd64", "arm64")) + Expect(result.MachineImages[0].Versions[0].CapabilityFlavors).To(BeNil()) + Expect(result.MachineTypes[0].Capabilities).To(BeNil()) + Expect(result.MachineTypes[0].Architecture).To(Equal(ptr.To("amd64"))) + }) + + It("should leave architectures empty for machine image versions with empty capability flavors, to not overwrite images during deepMerge", func() { + spec := gardencorev1beta1.NamespacedCloudProfileSpec{ + MachineImages: []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: "20.04", + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{}, + }, + }, + }, + }, + } + + result := TransformSpecToParentFormat(spec, nil) + + Expect(result.MachineImages[0].Versions[0].Architectures).To(BeEmpty()) + }) + }) + + It("should handle multiple machine images and types correctly", func() { + spec := gardencorev1beta1.NamespacedCloudProfileSpec{ + MachineImages: []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: "20.04", + }, + Architectures: []string{"amd64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: "22.04", + }, + Architectures: []string{"arm64"}, + }, + }, + }, + { + Name: "debian", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: "11", + }, + Architectures: []string{"amd64", "arm64"}, + }, + }, + }, + }, + MachineTypes: []gardencorev1beta1.MachineType{ + { + Name: "m5.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + Architecture: ptr.To("amd64"), + }, + { + Name: "m6g.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + Architecture: ptr.To("arm64"), + }, + }, + } + + capabilityDefinitions := []gardencorev1beta1.CapabilityDefinition{ + { + Name: v1beta1constants.ArchitectureName, + Values: []string{"amd64", "arm64"}, + }, + } + + result := TransformSpecToParentFormat(spec, capabilityDefinitions) + + // Verify all machine images have capability flavors + Expect(result.MachineImages).To(ConsistOf(gardencorev1beta1.MachineImage{ + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "20.04"}, + Architectures: []string{"amd64"}, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": {"amd64"}, + }, + }, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "22.04"}, + Architectures: []string{"arm64"}, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": {"arm64"}, + }, + }, + }, + }, + }, + }, + gardencorev1beta1.MachineImage{ + Name: "debian", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "11"}, + Architectures: []string{"amd64", "arm64"}, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": {"amd64"}, + }, + }, + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": {"arm64"}, + }, + }, + }, + }, + }, + })) + // Verify all machine types have capabilities + Expect(result.MachineTypes).To(Equal([]gardencorev1beta1.MachineType{ + { + Name: "m5.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + Architecture: ptr.To("amd64"), + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": {"amd64"}, + }, + }, + { + Name: "m6g.large", + CPU: resource.MustParse("2"), + Memory: resource.MustParse("8Gi"), + Architecture: ptr.To("arm64"), + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": {"arm64"}, + }, + }, + })) + }) + + It("should return empty spec for empty input", func() { + spec := gardencorev1beta1.NamespacedCloudProfileSpec{} + capabilityDefinitions := []gardencorev1beta1.CapabilityDefinition{ + { + Name: v1beta1constants.ArchitectureName, + Values: []string{"amd64", "arm64"}, + }, + } + + result := TransformSpecToParentFormat(spec, capabilityDefinitions) + + Expect(result).To(Equal(gardencorev1beta1.NamespacedCloudProfileSpec{})) + }) + + It("should not modify the original spec (deep copy behavior)", func() { + originalSpec := gardencorev1beta1.NamespacedCloudProfileSpec{ + MachineImages: []gardencorev1beta1.MachineImage{ + { + Name: "ubuntu", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: "20.04", + }, + Architectures: []string{"amd64"}, + }, + }, + }, + }, + } + + capabilityDefinitions := []gardencorev1beta1.CapabilityDefinition{ + { + Name: v1beta1constants.ArchitectureName, + Values: []string{"amd64", "arm64"}, + }, + } + + // Keep references to verify original is unchanged + originalArchitectures := originalSpec.MachineImages[0].Versions[0].Architectures + originalCapabilityFlavors := originalSpec.MachineImages[0].Versions[0].CapabilityFlavors + + result := TransformSpecToParentFormat(originalSpec, capabilityDefinitions) + + // Original should remain unchanged + Expect(originalSpec.MachineImages[0].Versions[0].Architectures).To(Equal(originalArchitectures)) + Expect(originalSpec.MachineImages[0].Versions[0].CapabilityFlavors).To(Equal(originalCapabilityFlavors)) + + // Result should have capability flavors added + Expect(result.MachineImages[0].Versions[0].CapabilityFlavors).ToNot(BeNil()) + Expect(result.MachineImages[0].Versions[0].CapabilityFlavors).To(HaveLen(1)) + }) +}) diff --git a/plugin/pkg/global/resourcereferencemanager/admission.go b/plugin/pkg/global/resourcereferencemanager/admission.go index 5388fd21854..1fbaf4ed2bd 100644 --- a/plugin/pkg/global/resourcereferencemanager/admission.go +++ b/plugin/pkg/global/resourcereferencemanager/admission.go @@ -489,9 +489,12 @@ func (r *ReferenceManager) Validate(ctx context.Context, a admission.Attributes, // getting Machine image versions that have been removed from or added to the CloudProfile removedMachineImages, removedMachineImageVersions, addedMachineImages, addedMachineImageVersions := helper.GetMachineImageDiff(oldCloudProfile.Spec.MachineImages, cloudProfile.Spec.MachineImages) + // getting removed capabilities + removedCapabilities := getRemovedMachineCapabilities(oldCloudProfile.Spec.MachineCapabilities, cloudProfile.Spec.MachineCapabilities) + wasLimitAdded := !apiequality.Semantic.DeepEqual(cloudProfile.Spec.Limits, oldCloudProfile.Spec.Limits) - if len(removedKubernetesVersions) > 0 || len(removedMachineImageVersions) > 0 || len(addedMachineImageVersions) > 0 || wasLimitAdded { + if len(removedKubernetesVersions) > 0 || len(removedMachineImageVersions) > 0 || len(addedMachineImageVersions) > 0 || wasLimitAdded || len(removedCapabilities) > 0 { shootList, err1 := r.shootLister.List(labels.Everything()) if err1 != nil { return apierrors.NewInternalError(fmt.Errorf("could not list shoots to verify that Kubernetes and/or Machine image version can be removed: %v", err1)) @@ -547,6 +550,10 @@ func (r *ReferenceManager) Validate(ctx context.Context, a admission.Attributes, } } } + checkMachineImageForRemovedCapabilities(machineImage, removedCapabilities, nscpfl, cloudProfile, channel) + } + for _, machineType := range nscpfl.Spec.MachineTypes { + checkMachineTypeForRemovedCapabilities(machineType, removedCapabilities, nscpfl, cloudProfile, channel) } }(ncp) } @@ -712,6 +719,34 @@ func (r *ReferenceManager) Validate(ctx context.Context, a admission.Attributes, return nil } +func checkMachineTypeForRemovedCapabilities(machineType gardencorev1beta1.MachineType, removedCapabilities []core.CapabilityDefinition, nscpfl *gardencorev1beta1.NamespacedCloudProfile, cloudProfile *core.CloudProfile, channel chan error) { + for _, removedCapability := range removedCapabilities { + if capabilityValues, exist := machineType.Capabilities[removedCapability.Name]; exist { + for _, capabilityValue := range capabilityValues { + if slices.Contains(removedCapability.Values, capabilityValue) { + channel <- fmt.Errorf("unable to delete MachineCapability %q with value %q from CloudProfile %q - capability value is still in use by NamespacedCloudProfile '%s/%s'", removedCapability.Name, capabilityValue, cloudProfile.Name, nscpfl.Namespace, nscpfl.Name) + } + } + } + } +} + +func checkMachineImageForRemovedCapabilities(machineImage gardencorev1beta1.MachineImage, removedCapabilities []core.CapabilityDefinition, nscpfl *gardencorev1beta1.NamespacedCloudProfile, cloudProfile *core.CloudProfile, channel chan error) { + for _, imageVersion := range machineImage.Versions { + for _, imageFlavor := range imageVersion.CapabilityFlavors { + for _, removedCapability := range removedCapabilities { + if capabilityValues, exist := imageFlavor.Capabilities[removedCapability.Name]; exist { + for _, capabilityValue := range capabilityValues { + if slices.Contains(removedCapability.Values, capabilityValue) { + channel <- fmt.Errorf("unable to delete MachineCapability %q with value %q from CloudProfile %q - capability value is still in use by NamespacedCloudProfile '%s/%s'", removedCapability.Name, capabilityValue, cloudProfile.Name, nscpfl.Namespace, nscpfl.Name) + } + } + } + } + } + } +} + func (r *ReferenceManager) ensureControllerRegistrationReferences(ctx context.Context, ctrlReg *core.ControllerRegistration) error { deployment := ctrlReg.Spec.Deployment if ctrlReg.Spec.Deployment == nil { @@ -1399,3 +1434,32 @@ func (r *ReferenceManager) sanityCheckProviderSecret(ctx context.Context, namesp func isGardenadmUser(userInfo user.Info) bool { return slices.Contains(userInfo.GetGroups(), v1beta1constants.ShootsGroup) && strings.HasPrefix(userInfo.GetName(), v1beta1constants.GardenadmUserNamePrefix) } + +// getRemovedMachineCapabilities returns the removed capabilities and their removed values. +func getRemovedMachineCapabilities(old, new []core.CapabilityDefinition) []core.CapabilityDefinition { + var ( + removedCapabilities []core.CapabilityDefinition + oldCapabilitiesMap = utils.CreateMapFromSlice(old, func(capability core.CapabilityDefinition) string { return capability.Name }) + newCapabilitiesMap = utils.CreateMapFromSlice(new, func(capability core.CapabilityDefinition) string { return capability.Name }) + ) + + for capabilityName, oldCapability := range oldCapabilitiesMap { + newCapability, exists := newCapabilitiesMap[capabilityName] + if !exists { + // Completely removed capability. + removedCapabilities = append(removedCapabilities, oldCapability) + } else { + // Collect removed capability values. + oldValuesSet := sets.New[string](oldCapability.Values...) + newValuesSet := sets.New[string](newCapability.Values...) + + if removedValues := oldValuesSet.Difference(newValuesSet); removedValues.Len() > 0 { + removedCapabilities = append(removedCapabilities, core.CapabilityDefinition{ + Name: capabilityName, + Values: removedValues.UnsortedList(), + }) + } + } + } + return removedCapabilities +} diff --git a/plugin/pkg/global/resourcereferencemanager/admission_test.go b/plugin/pkg/global/resourcereferencemanager/admission_test.go index 8f85efc8b27..345091ad47e 100644 --- a/plugin/pkg/global/resourcereferencemanager/admission_test.go +++ b/plugin/pkg/global/resourcereferencemanager/admission_test.go @@ -3001,6 +3001,238 @@ var _ = Describe("resourcereferencemanager", func() { }) }) + Context("CloudProfile - Remove MachineCapabilities", func() { + var ( + ctx context.Context + cloudProfile, oldCloudProfile *core.CloudProfile + namespacedCloudProfile *gardencorev1beta1.NamespacedCloudProfile + ) + + BeforeEach(func() { + ctx = context.Background() + + // Base CloudProfile with MachineCapabilities + cloudProfile = &core.CloudProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: cloudProfileName, + }, + Spec: core.CloudProfileSpec{ + MachineCapabilities: []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"amd64", "arm64"}}, + {Name: "network", Values: []string{"public", "private"}}, + {Name: "storage", Values: []string{"ssd", "hdd"}}, + }, + MachineImages: []core.MachineImage{ + { + Name: "ubuntu", + Versions: []core.MachineImageVersion{ + { + ExpirableVersion: core.ExpirableVersion{Version: "20.04"}, + CapabilityFlavors: []core.MachineImageFlavor{ + {Capabilities: core.Capabilities{ + "architecture": []string{"amd64"}, + "storage": []string{"ssd"}, + }}, + {Capabilities: core.Capabilities{ + "architecture": []string{"amd64"}, + "storage": []string{"ssd"}, + }}, + }, + }, + }, + }, + }, + MachineTypes: []core.MachineType{ + { + Name: "m5.large", + Capabilities: core.Capabilities{ + "architecture": []string{"amd64"}, + "network": []string{"public"}, + }, + }, + }, + }, + } + oldCloudProfile = cloudProfile.DeepCopy() + + // NamespacedCloudProfile that uses capabilities from parent + namespacedCloudProfile = &gardencorev1beta1.NamespacedCloudProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "namespaced-profile", + Namespace: namespace, + }, + Spec: gardencorev1beta1.NamespacedCloudProfileSpec{ + Parent: gardencorev1beta1.CloudProfileReference{ + Kind: "CloudProfile", + Name: cloudProfileName, + }, + MachineImages: []gardencorev1beta1.MachineImage{ + { + Name: "debian", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "11"}, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{"arm64"}, + "network": []string{"private"}, + "storage": []string{"hdd"}, + }}, + }, + }, + }, + }, + }, + MachineTypes: []gardencorev1beta1.MachineType{ + { + Name: "t3.micro", + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{"arm64"}, + "storage": []string{"ssd"}, + }, + }, + }, + }, + } + + Expect(gardenCoreInformerFactory.Core().V1beta1().NamespacedCloudProfiles().Informer().GetStore().Add(namespacedCloudProfile)).To(Succeed()) + }) + + It("should reject removing capability values still in use by NamespacedCloudProfile machine image flavors and machine types", func() { + // Remove arm64 architecture and private network capabilities that are used by NamespacedCloudProfile + cloudProfile.Spec.MachineCapabilities = []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"amd64"}}, // removed arm64 + {Name: "network", Values: []string{"public"}}, // removed private + {Name: "storage", Values: []string{"ssd"}}, // removed hdd + } + + attrs := admission.NewAttributesRecord(cloudProfile, oldCloudProfile, core.Kind("CloudProfile").WithVersion("version"), "", cloudProfile.Name, core.Resource("cloudprofiles").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, defaultUserInfo) + + err := admissionHandler.Validate(ctx, attrs, nil) + Expect(err).To(HaveOccurred()) + + // Should get multiple errors for different capability values being removed + Expect(err.Error()).To(And( + ContainSubstring("unable to delete MachineCapability \"architecture\" with value \"arm64\" from CloudProfile"), + ContainSubstring("capability value is still in use by NamespacedCloudProfile 'default/namespaced-profile'"), + ContainSubstring("unable to delete MachineCapability \"network\" with value \"private\" from CloudProfile"), + ContainSubstring("unable to delete MachineCapability \"storage\" with value \"hdd\" from CloudProfile"), + )) + }) + + It("should allow removing capability values not used by any NamespacedCloudProfile", func() { + // Create another NamespacedCloudProfile that doesn't use certain capabilities + anotherNamespacedCloudProfile := &gardencorev1beta1.NamespacedCloudProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "another-profile", + Namespace: namespace, + }, + Spec: gardencorev1beta1.NamespacedCloudProfileSpec{ + Parent: gardencorev1beta1.CloudProfileReference{ + Kind: "CloudProfile", + Name: cloudProfileName, + }, + MachineImages: []gardencorev1beta1.MachineImage{ + { + Name: "centos", + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "8"}, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{"amd64"}, // only uses amd64 + "network": []string{"public"}, // only uses public + "storage": []string{"ssd"}, // only uses ssd + }}, + }, + }, + }, + }, + }, + }, + } + + Expect(gardenCoreInformerFactory.Core().V1beta1().NamespacedCloudProfiles().Informer().GetStore().Add(anotherNamespacedCloudProfile)).To(Succeed()) + + // This should succeed since the removed capabilities are not used by the additional NamespacedCloudProfile + // but will still fail due to the first NamespacedCloudProfile + cloudProfile.Spec.MachineCapabilities = []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"amd64"}}, // removed arm64 - still used by first profile + {Name: "network", Values: []string{"public"}}, // removed private - still used by first profile + {Name: "storage", Values: []string{"ssd"}}, // removed hdd - still used by first profile + } + + attrs := admission.NewAttributesRecord(cloudProfile, oldCloudProfile, core.Kind("CloudProfile").WithVersion("version"), "", cloudProfile.Name, core.Resource("cloudprofiles").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, defaultUserInfo) + + err := admissionHandler.Validate(ctx, attrs, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("namespaced-profile")) // should reference the first profile that uses the removed capabilities + Expect(err.Error()).NotTo(ContainSubstring("another-profile")) // should not reference the second profile + + // Now remove the deleted capabilities from the first NamespacedCloudProfile to allow the update + namespacedCloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors[0].Capabilities["architecture"] = []string{"amd64"} + namespacedCloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors[0].Capabilities["network"] = []string{"public"} + namespacedCloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors[0].Capabilities["storage"] = []string{"ssd"} + namespacedCloudProfile.Spec.MachineTypes[0].Capabilities["architecture"] = []string{"amd64"} + namespacedCloudProfile.Spec.MachineTypes[0].Capabilities["storage"] = []string{"ssd"} + + Expect(gardenCoreInformerFactory.Core().V1beta1().NamespacedCloudProfiles().Informer().GetStore().Update(namespacedCloudProfile)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) + }) + + It("should reject completely removing capability definitions still in use", func() { + // Completely remove the storage capability definition + cloudProfile.Spec.MachineCapabilities = []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"amd64", "arm64"}}, + {Name: "network", Values: []string{"public", "private"}}, + // storage capability completely removed + } + + attrs := admission.NewAttributesRecord(cloudProfile, oldCloudProfile, core.Kind("CloudProfile").WithVersion("version"), "", cloudProfile.Name, core.Resource("cloudprofiles").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, defaultUserInfo) + + err := admissionHandler.Validate(ctx, attrs, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(And( + ContainSubstring("unable to delete MachineCapability \"storage\""), + ContainSubstring("capability value is still in use by NamespacedCloudProfile 'default/namespaced-profile'"), + )) + }) + + It("should allow updates when NamespacedCloudProfile has deletion timestamp", func() { + // Mark NamespacedCloudProfile for deletion + deletionTime := metav1.Now() + namespacedCloudProfile.DeletionTimestamp = &deletionTime + + Expect(gardenCoreInformerFactory.Core().V1beta1().NamespacedCloudProfiles().Informer().GetStore().Update(namespacedCloudProfile)).To(Succeed()) + + // Now removing capabilities should succeed since the NamespacedCloudProfile is being deleted + cloudProfile.Spec.MachineCapabilities = []core.CapabilityDefinition{ + // removed network and storage capabilities + {Name: "architecture", Values: []string{"amd64"}}, // removed arm64 + } + + attrs := admission.NewAttributesRecord(cloudProfile, oldCloudProfile, core.Kind("CloudProfile").WithVersion("version"), "", cloudProfile.Name, core.Resource("cloudprofiles").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, defaultUserInfo) + + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) + }) + + It("should allow updates when NamespacedCloudProfile references different parent", func() { + // Make NamespacedCloudProfile reference a different CloudProfile + namespacedCloudProfile.Spec.Parent.Name = "different-parent" + + Expect(gardenCoreInformerFactory.Core().V1beta1().NamespacedCloudProfiles().Informer().GetStore().Update(namespacedCloudProfile)).To(Succeed()) + + // Now removing capabilities should succeed since the NamespacedCloudProfile doesn't reference this CloudProfile + cloudProfile.Spec.MachineCapabilities = []core.CapabilityDefinition{ + {Name: "architecture", Values: []string{"amd64"}}, // removed arm64 + } + + attrs := admission.NewAttributesRecord(cloudProfile, oldCloudProfile, core.Kind("CloudProfile").WithVersion("version"), "", cloudProfile.Name, core.Resource("cloudprofiles").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, defaultUserInfo) + + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) + }) + }) + Context("NamespacedCloudProfile - Extending Kubernetes versions", func() { var ( expirationDateFuture2 metav1.Time diff --git a/plugin/pkg/namespacedcloudprofile/validator/admission.go b/plugin/pkg/namespacedcloudprofile/validator/admission.go index 0b9ef80a6eb..d1b3148f95f 100644 --- a/plugin/pkg/namespacedcloudprofile/validator/admission.go +++ b/plugin/pkg/namespacedcloudprofile/validator/admission.go @@ -12,11 +12,12 @@ import ( "reflect" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/admission" "k8s.io/utils/ptr" - "github.com/gardener/gardener/pkg/api" + gardencoreapi "github.com/gardener/gardener/pkg/api" gardencore "github.com/gardener/gardener/pkg/apis/core" gardencorehelper "github.com/gardener/gardener/pkg/apis/core/helper" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -137,6 +138,10 @@ func (v *ValidateNamespacedCloudProfile) Validate(ctx context.Context, a admissi return apierrors.NewBadRequest("parent CloudProfile could not be found") } + if err := v.simulateTransformationToParentSpecFormat(namespacedCloudProfile, parentCloudProfile, oldNamespacedCloudProfile); err != nil { + return err + } + validationContext := &validationContext{ parentCloudProfile: parentCloudProfile, namespacedCloudProfile: namespacedCloudProfile, @@ -159,6 +164,30 @@ func (v *ValidateNamespacedCloudProfile) Validate(ctx context.Context, a admissi return nil } +// TODO(Roncossek): Remove simulateTransformationToParentSpecFormat once all CloudProfiles have been migrated to use CapabilityFlavors and the Architecture fields are effectively forbidden or have been removed. +func (v *ValidateNamespacedCloudProfile) simulateTransformationToParentSpecFormat(namespacedCloudProfile *gardencore.NamespacedCloudProfile, parentCloudProfile *gardencorev1beta1.CloudProfile, oldNamespacedCloudProfile *gardencore.NamespacedCloudProfile) error { + var namespacedCloudProfileSpecV1beta1 gardencorev1beta1.NamespacedCloudProfileSpec + if err := gardencoreapi.Scheme.Convert(&namespacedCloudProfile.Spec, &namespacedCloudProfileSpecV1beta1, nil); err != nil { + return apierrors.NewInternalError(err) + } + // simulate status transformation if namespaced cloud profile controller would do it + uniformNamespacedCloudProfileSpec := gardenerutils.TransformSpecToParentFormat(namespacedCloudProfileSpecV1beta1, parentCloudProfile.Spec.MachineCapabilities) + if err := gardencoreapi.Scheme.Convert(&uniformNamespacedCloudProfileSpec, &namespacedCloudProfile.Spec, nil); err != nil { + return apierrors.NewInternalError(err) + } + + var oldNamespacedCloudProfileSpecV1beta1 gardencorev1beta1.NamespacedCloudProfileSpec + if err := gardencoreapi.Scheme.Convert(&oldNamespacedCloudProfile.Spec, &oldNamespacedCloudProfileSpecV1beta1, nil); err != nil { + return apierrors.NewInternalError(err) + } + // simulate status transformation if namespaced cloud profile controller would do it + oldUniformNamespacedCloudProfileSpec := gardenerutils.TransformSpecToParentFormat(oldNamespacedCloudProfileSpecV1beta1, parentCloudProfile.Spec.MachineCapabilities) + if err := gardencoreapi.Scheme.Convert(&oldUniformNamespacedCloudProfileSpec, &oldNamespacedCloudProfile.Spec, nil); err != nil { + return apierrors.NewInternalError(err) + } + return nil +} + type validationContext struct { parentCloudProfile *gardencorev1beta1.CloudProfile namespacedCloudProfile *gardencore.NamespacedCloudProfile @@ -235,11 +264,16 @@ func (c *validationContext) validateMachineImageOverrides(ctx context.Context, a oldVersionsMerged = gardenerutils.NewCoreImagesContext(c.oldNamespacedCloudProfile.Status.CloudProfileSpec.MachineImages) } + var parentCloudProfileSpecCore gardencore.CloudProfileSpec + if err := gardencoreapi.Scheme.Convert(&c.parentCloudProfile.Spec, &parentCloudProfileSpecCore, ctx); err != nil { + return apierrors.NewInternalError(err) + } + capabilities := gardencorehelper.CapabilityDefinitionsToCapabilities(parentCloudProfileSpecCore.MachineCapabilities) + for imageIndex, image := range c.namespacedCloudProfile.Spec.MachineImages { imageIndexPath := field.NewPath("spec", "machineImages").Index(imageIndex) - _, isExistingImage := parentImages.GetImage(image.Name) - - if isExistingImage { + _, imageExistsInParent := parentImages.GetImage(image.Name) + if imageExistsInParent { // If in the meantime an image specified only in the NamespacedCloudProfile has been // added to the parent CloudProfile, then ignore already existing fields otherwise invalid for a new override. var imageAlreadyExistsInNamespacedCloudProfile bool @@ -253,11 +287,11 @@ func (c *validationContext) validateMachineImageOverrides(ctx context.Context, a } for imageVersionIndex, imageVersion := range image.Versions { + imageVersionIndexPath := imageIndexPath.Child("versions").Index(imageVersionIndex) if _, isExistingVersion := parentImages.GetImageVersion(image.Name, imageVersion.Version); isExistingVersion { // An image with the specified version is already present in the parent CloudProfile. // Ensure that only the expiration date is overridden. // For new versions added to an existing image, the validation will be done on the simulated merge result. - imageVersionIndexPath := imageIndexPath.Child("versions").Index(imageVersionIndex) // If in the meantime an image version specified only in the NamespacedCloudProfile has been // added to the parent CloudProfile, then ignore already existing fields otherwise invalid for a new override. @@ -291,18 +325,17 @@ func (c *validationContext) validateMachineImageOverrides(ctx context.Context, a allErrs = append(allErrs, field.Invalid(imageVersionIndexPath.Child("expirationDate"), imageVersion.ExpirationDate, fmt.Sprintf("expiration date for version %q is in the past", imageVersion.Version))) } } + } else { + // New image version added to an existing image. + allErrs = append(allErrs, validation.ValidateMachineImageVersion(image, imageVersion, capabilities, sets.Set[string]{}, imageVersionIndexPath)...) } } } else { - var parentCloudProfileSpecCore gardencore.CloudProfileSpec - if err := api.Scheme.Convert(&c.parentCloudProfile.Spec, &parentCloudProfileSpecCore, ctx); err != nil { - allErrs = append(allErrs, field.InternalError(imageIndexPath, err)) + allErrs = append(allErrs, validation.ValidateMachineImage(image, capabilities, sets.Set[string]{}, sets.Set[string]{}, false, imageIndexPath)...) + for index, machineVersion := range image.Versions { + versionsPath := imageIndexPath.Child("versions").Index(index) + allErrs = append(allErrs, validation.ValidateMachineImageAdditionalConfig(machineVersion, versionsPath, image, capabilities)...) } - - // There is no entry for this image in the parent CloudProfile yet. - capabilities := gardencorehelper.CapabilityDefinitionsToCapabilities(parentCloudProfileSpecCore.MachineCapabilities) - allErrs = append(allErrs, validation.ValidateMachineImages([]gardencore.MachineImage{image}, capabilities, imageIndexPath, false)...) - allErrs = append(allErrs, validation.ValidateCloudProfileMachineImages([]gardencore.MachineImage{image}, capabilities, imageIndexPath)...) } } return allErrs.ToAggregate() @@ -335,7 +368,7 @@ func validateNamespacedCloudProfileExtendedMachineImages(machineVersion gardenco func (c *validationContext) validateSimulatedCloudProfileStatusMergeResult() error { namespacedCloudProfile := &gardencorev1beta1.NamespacedCloudProfile{} - if err := api.Scheme.Convert(c.namespacedCloudProfile, namespacedCloudProfile, nil); err != nil { + if err := gardencoreapi.Scheme.Convert(c.namespacedCloudProfile, namespacedCloudProfile, nil); err != nil { return err } errs := ValidateSimulatedNamespacedCloudProfileStatus(c.parentCloudProfile, namespacedCloudProfile) @@ -354,7 +387,7 @@ func ValidateSimulatedNamespacedCloudProfileStatus(originalParentCloudProfile *g namespacedcloudprofile.MergeCloudProfiles(namespacedCloudProfile, parentCloudProfile) coreNamespacedCloudProfile := &gardencore.NamespacedCloudProfile{} - if err := api.Scheme.Convert(namespacedCloudProfile, coreNamespacedCloudProfile, nil); err != nil { + if err := gardencoreapi.Scheme.Convert(namespacedCloudProfile, coreNamespacedCloudProfile, nil); err != nil { return field.ErrorList{{ Type: field.ErrorTypeInternal, Field: "", diff --git a/plugin/pkg/namespacedcloudprofile/validator/admission_test.go b/plugin/pkg/namespacedcloudprofile/validator/admission_test.go index d97b466405e..8ecc93e7475 100644 --- a/plugin/pkg/namespacedcloudprofile/validator/admission_test.go +++ b/plugin/pkg/namespacedcloudprofile/validator/admission_test.go @@ -11,6 +11,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" + "github.com/onsi/gomega/types" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" @@ -126,80 +127,129 @@ var _ = Describe("Admission", func() { }) Describe("Adding machineImage Versions and machineTypes NOT defined in the parent CloudProfile", func() { - BeforeEach(func() { - namespacedCloudProfile.Spec.MachineImages = []gardencore.MachineImage{ - { - Name: "test-image", - Versions: []gardencore.MachineImageVersion{{ - ExpirableVersion: gardencore.ExpirableVersion{Version: "1.0.1"}, - CRI: []gardencore.CRI{{Name: "containerd"}}, - }}}} - }) - - It("should allow adding a machineImage without Capabilities as architecture defaults to amd64", func() { - Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) - attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) - Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) - }) - - It("should reject a machineImage with Capabilities or CapabilityValues not in the parent CloudProfile", func() { - Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) - namespacedCloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = []gardencore.MachineImageFlavor{ - {Capabilities: gardencore.Capabilities{ - // Unsupported CapabilityValue - "capability2": []string{"value3"}, - // Unsupported Capability - "not-in-parent": []string{"value1", "value2"}}}, - } - - attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) - Expect(admissionHandler.Validate(ctx, attrs, nil)).To(MatchError(And( - ContainSubstring(`capability2[0]: Unsupported value: "value3": supported values: "value1", "value2"`), - ContainSubstring(`Unsupported value: "not-in-parent": supported values:`), - ))) - }) - - It("should allow machineTypes and overwrite Architecture if it conflicts with Capabilities.Architecture", func() { - Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) - - namespacedCloudProfile.Spec.MachineTypes = []gardencore.MachineType{{Name: "my-other-machine", - Architecture: ptr.To("amd64"), - Capabilities: gardencore.Capabilities{constants.ArchitectureName: []string{"arm64"}}, - }} - - attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) - - Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) + When("Architecture capability has one supported value", func() { + BeforeEach(func() { + parentCloudProfile.Spec.MachineCapabilities = []gardencorev1beta1.CapabilityDefinition{ + {Name: constants.ArchitectureName, Values: []string{"amd64"}}, + {Name: "capability2", Values: []string{"value1", "value2"}}, + } + namespacedCloudProfile.Spec.MachineImages = []gardencore.MachineImage{ + { + Name: "test-image", + Versions: []gardencore.MachineImageVersion{{ + ExpirableVersion: gardencore.ExpirableVersion{Version: "1.0.1"}, + CRI: []gardencore.CRI{{Name: "containerd"}}, + }}}} + }) + + It("should allow adding a machineImage without Capabilities as architecture defaults from capabilityDefinition", func() { + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) + attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) + }) + + It("should allow machineTypes without Capabilities as architecture defaults to capabilityDefinition", func() { + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) + namespacedCloudProfile.Spec.MachineTypes = []gardencore.MachineType{{Name: "my-other-machine"}} + + attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) + + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) + }) }) - It("should reject unsupported Capabilities or CapabilityValues in machineTypes", func() { - Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) - - namespacedCloudProfile.Spec.MachineTypes = []gardencore.MachineType{{Name: "my-other-machine", - Capabilities: gardencore.Capabilities{constants.ArchitectureName: []string{"arm64"}, - // Unsupported CapabilityValue - "capability2": []string{"value3"}, - // Unsupported Capability - "not-in-parent": []string{"value1", "value2"}}, - }} - - attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) - - Expect(admissionHandler.Validate(ctx, attrs, nil)).To(MatchError(And( - ContainSubstring(`capability2[0]: Unsupported value: "value3": supported values: "value1", "value2"`), - ContainSubstring(`Unsupported value: "not-in-parent": supported values:`), - ))) + When("Architecture capability has multiple supported values", func() { + BeforeEach(func() { + parentCloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{constants.ArchitectureName: []string{"amd64"}}}, + } + namespacedCloudProfile.Spec.MachineImages = []gardencore.MachineImage{ + { + Name: "test-image", + Versions: []gardencore.MachineImageVersion{{ + ExpirableVersion: gardencore.ExpirableVersion{Version: "1.0.1"}, + CRI: []gardencore.CRI{{Name: "containerd"}}, + CapabilityFlavors: []gardencore.MachineImageFlavor{{ + Capabilities: gardencore.Capabilities{constants.ArchitectureName: []string{"amd64"}}, + }}, + }}}} + }) + + It("should allow adding a machineImage with Capabilities", func() { + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) + namespacedCloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = []gardencore.MachineImageFlavor{ + {Capabilities: gardencore.Capabilities{"architecture": []string{"arm64"}}}, + } + attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) + }) + + It("should reject a machineImage with Capabilities or CapabilityValues not in the parent CloudProfile", func() { + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) + namespacedCloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = []gardencore.MachineImageFlavor{ + {Capabilities: gardencore.Capabilities{ + // Unsupported CapabilityValue + "capability2": []string{"value3"}, + // Unsupported Capability + "not-in-parent": []string{"value1", "value2"}}}, + } + + attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(MatchError(And( + ContainSubstring(`capability2[0]: Unsupported value: "value3": supported values: "value1", "value2"`), + ContainSubstring(`Unsupported value: "not-in-parent": supported values:`), + ))) + }) + + It("should reject machineTypes if it conflicts with Capabilities.Architecture", func() { + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) + namespacedCloudProfile.Spec.MachineImages = []gardencore.MachineImage{} + + namespacedCloudProfile.Spec.MachineTypes = []gardencore.MachineType{{Name: "my-other-machine", + Architecture: ptr.To("amd64"), + Capabilities: gardencore.Capabilities{constants.ArchitectureName: []string{"arm64"}}, + }} + + attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) + + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(MatchError(And( + ContainSubstring("status.cloudProfileSpec.machineTypes[0].architecture"), + ContainSubstring("machine type architecture (amd64) conflicts with the capability architecture (arm64)"), + ))) + }) + + It("should reject unsupported Capabilities or CapabilityValues in machineTypes", func() { + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(parentCloudProfile)).To(Succeed()) + namespacedCloudProfile.Spec.MachineTypes = []gardencore.MachineType{{Name: "my-other-machine", + Capabilities: gardencore.Capabilities{ + constants.ArchitectureName: []string{"arm64"}, + // Unsupported CapabilityValue + "capability2": []string{"value3"}, + // Unsupported Capability + "not-in-parent": []string{"value1", "value2"}}, + }} + + attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) + + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(MatchError(And( + ContainSubstring(`capability2[0]: Unsupported value: "value3": supported values: "value1", "value2"`), + ContainSubstring(`Unsupported value: "not-in-parent": supported values:`), + ))) + }) }) }) Describe("Adding machineImages defined in the parent CloudProfile", func() { BeforeEach(func() { - namespacedCloudProfile.Spec.MachineImages = []gardencore.MachineImage{ - { - Name: "test-image", - Versions: []gardencore.MachineImageVersion{{ - ExpirableVersion: gardencore.ExpirableVersion{Version: "1.0.0", ExpirationDate: validExpirationDate}, - }}}} + parentCloudProfile.Spec.MachineImages[0].Versions[0].CapabilityFlavors = []gardencorev1beta1.MachineImageFlavor{{ + Capabilities: gardencorev1beta1.Capabilities{constants.ArchitectureName: []string{"amd64"}}, + }} + namespacedCloudProfile.Spec.MachineImages = []gardencore.MachineImage{{ + Name: "test-image", + Versions: []gardencore.MachineImageVersion{{ + ExpirableVersion: gardencore.ExpirableVersion{Version: "1.0.0", ExpirationDate: validExpirationDate}, + }}, + }} }) It("should allow to add a machineImage without Capabilities", func() { @@ -514,7 +564,20 @@ var _ = Describe("Admission", func() { Expect(admissionHandler.Validate(ctx, attrs, nil)).To(MatchError(ContainSubstring("expiration date for version \"1.0.0\" must be set"))) }) - It("should fail for creating a NamespacedCloudProfile that overrides an existing MachineImage version and specifies classification/cri/arch/flavors/kubeletVersionConstraint/inPlaceUpdates", func() { + DescribeTable("should fail for creating a NamespacedCloudProfile that overrides an existing MachineImage version and specifies classification/cri/arch/flavors/kubeletVersionConstraint/inPlaceUpdates", func(parentUsesCapabilities bool) { + var additionalMatcher types.GomegaMatcher + if parentUsesCapabilities { + parentCloudProfile.Spec.MachineCapabilities = []gardencorev1beta1.CapabilityDefinition{{ + Name: "architecture", + Values: []string{"amd64", "arm64"}, + }} + // capabilityFlavors will be deleted on transform in transformation to parent format + additionalMatcher = PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("spec.machineImages[0].versions[0].capabilityFlavors"), + "Detail": ContainSubstring("must not provide capabilities to an extended machine image in NamespacedCloudProfile"), + })) + } parentCloudProfile.Spec.MachineImages = []gardencorev1beta1.MachineImage{ {Name: "test-image", Versions: []gardencorev1beta1.MachineImageVersion{ {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "1.1.0"}, CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}}, @@ -539,7 +602,7 @@ var _ = Describe("Admission", func() { attrs := admission.NewAttributesRecord(namespacedCloudProfile, nil, gardencorev1beta1.Kind("NamespacedCloudProfile").WithVersion("version"), "", namespacedCloudProfile.Name, gardencorev1beta1.Resource("namespacedcloudprofile").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil) - Expect(admissionHandler.Validate(ctx, attrs, nil)).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + matchers := []types.GomegaMatcher{PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.machineImages[0].versions[0].cri"), "Detail": ContainSubstring("must not provide a cri to an extended machine image in NamespacedCloudProfile"), @@ -547,14 +610,6 @@ var _ = Describe("Admission", func() { "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.machineImages[0].versions[0].classification"), "Detail": ContainSubstring("must not provide a classification to an extended machine image in NamespacedCloudProfile"), - })), PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeForbidden), - "Field": Equal("spec.machineImages[0].versions[0].architectures"), - "Detail": ContainSubstring("must not provide an architecture to an extended machine image in NamespacedCloudProfile"), - })), PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeForbidden), - "Field": Equal("spec.machineImages[0].versions[0].capabilityFlavors"), - "Detail": ContainSubstring("must not provide capabilities to an extended machine image in NamespacedCloudProfile"), })), PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.machineImages[0].versions[0].kubeletVersionConstraint"), @@ -563,8 +618,20 @@ var _ = Describe("Admission", func() { "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.machineImages[0].versions[0].inPlaceUpdates"), "Detail": ContainSubstring("must not provide inPlaceUpdates to an extended machine image in NamespacedCloudProfile"), - })))) - }) + })), PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("spec.machineImages[0].versions[0].architectures"), + "Detail": ContainSubstring("must not provide an architecture to an extended machine image in NamespacedCloudProfile"), + }))} + + if parentUsesCapabilities { + matchers = append(matchers, additionalMatcher) + } + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(ConsistOf(matchers)) + }, + Entry("parent without capabilities", false), + Entry("parent with capabilities", true), + ) It("should fail for updating a NamespacedCloudProfile that specifies an already expired MachineImage version", func() { parentCloudProfile.Spec.MachineImages = []gardencorev1beta1.MachineImage{ diff --git a/plugin/pkg/shoot/validator/admission.go b/plugin/pkg/shoot/validator/admission.go index a1d2c3a71fe..7bd770a22da 100644 --- a/plugin/pkg/shoot/validator/admission.go +++ b/plugin/pkg/shoot/validator/admission.go @@ -1078,7 +1078,7 @@ func (c *validationContext) validateMachineImage(idxPath *field.Path, worker cor isUpdateStrategyInPlace := helper.IsUpdateStrategyInPlace(worker.UpdateStrategy) isMachineImagePresentInCloudprofile, architectureSupported, activeMachineImageVersion, inPlaceUpdateSupported, validMachineImageVersions := validateMachineImagesConstraints(a, c.cloudProfileSpec.MachineImages, isNewWorkerPool, isUpdateStrategyInPlace, worker.Machine, oldWorker.Machine, c.cloudProfileSpec.MachineCapabilities) if !isMachineImagePresentInCloudprofile { - return field.Invalid(idxPath.Child("machine", "image"), worker.Machine.Image, fmt.Sprintf("machine image version is not supported, supported machine image versions are: %+v", validMachineImageVersions)) + return field.NotSupported(idxPath.Child("machine", "image", "version"), worker.Machine.Image.Version, validMachineImageVersions) } if !architectureSupported || !activeMachineImageVersion || (isUpdateStrategyInPlace && !inPlaceUpdateSupported) { diff --git a/plugin/pkg/shoot/validator/admission_test.go b/plugin/pkg/shoot/validator/admission_test.go index 42c64e7c8b3..92975828121 100644 --- a/plugin/pkg/shoot/validator/admission_test.go +++ b/plugin/pkg/shoot/validator/admission_test.go @@ -4135,7 +4135,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) err := admissionHandler.Admit(ctx, attrs, nil) - Expect(err).To(MatchError(ContainSubstring("machine image version is not supported"))) + Expect(err).To(MatchError(ContainSubstring("Unsupported value: \"1.2.baz\": supported values:"))) }) It("should default a machine image version to latest major.minor.patch version", func() { diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml index e1fa4e58cdd..234857b7643 100644 --- a/skaffold-operator.yaml +++ b/skaffold-operator.yaml @@ -1248,6 +1248,8 @@ build: - extensions/pkg/apis/config/v1alpha1 - extensions/pkg/controller - extensions/pkg/controller/cmd + - extensions/pkg/controller/worker + - extensions/pkg/predicate - extensions/pkg/util - extensions/pkg/webhook - extensions/pkg/webhook/certificates @@ -1291,7 +1293,9 @@ build: - pkg/client/kubernetes - pkg/client/kubernetes/cache - pkg/controllerutils + - pkg/controllerutils/mapper - pkg/controllerutils/predicate + - pkg/controllerutils/reconciler - pkg/extensions - pkg/gardenlet/apis/config/v1alpha1 - pkg/healthz @@ -1301,6 +1305,7 @@ build: - pkg/provider-local/admission/mutator - pkg/provider-local/admission/validator - pkg/provider-local/apis/local + - pkg/provider-local/apis/local/helper - pkg/provider-local/apis/local/install - pkg/provider-local/apis/local/v1alpha1 - pkg/provider-local/apis/local/validation @@ -1313,6 +1318,7 @@ build: - pkg/utils/flow - pkg/utils/gardener - pkg/utils/gardener/operator + - pkg/utils/gardener/shootstate - pkg/utils/imagevector - pkg/utils/kubernetes - pkg/utils/kubernetes/health diff --git a/test/integration/controllermanager/namespacedcloudprofile/namespacedcloudprofile_test.go b/test/integration/controllermanager/namespacedcloudprofile/namespacedcloudprofile_test.go index 05ae99ddd34..ca00e11b576 100644 --- a/test/integration/controllermanager/namespacedcloudprofile/namespacedcloudprofile_test.go +++ b/test/integration/controllermanager/namespacedcloudprofile/namespacedcloudprofile_test.go @@ -18,7 +18,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/utils/test" . "github.com/gardener/gardener/pkg/utils/test/matchers" @@ -33,6 +32,7 @@ var _ = DescribeTableSubtree("NamespacedCloudProfile controller tests", func(isC expirationDateFuture metav1.Time imageFlavors []gardencorev1beta1.MachineImageFlavor + capabilities gardencorev1beta1.Capabilities ) BeforeEach(func() { @@ -40,14 +40,12 @@ var _ = DescribeTableSubtree("NamespacedCloudProfile controller tests", func(isC var capabilityDefinitions []gardencorev1beta1.CapabilityDefinition if isCapabilitiesCloudProfile { capabilityDefinitions = []gardencorev1beta1.CapabilityDefinition{ - { - Name: "architecture", - Values: []string{v1beta1constants.ArchitectureAMD64}, - }, + {Name: "architecture", Values: []string{"amd64"}}, } imageFlavors = []gardencorev1beta1.MachineImageFlavor{ - {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{"amd64"}}}, } + capabilities = gardencorev1beta1.Capabilities{"architecture": []string{"amd64"}} } dateNow, _ := time.Parse(time.DateOnly, time.Now().Format(time.DateOnly)) @@ -69,10 +67,7 @@ var _ = DescribeTableSubtree("NamespacedCloudProfile controller tests", func(isC { Name: "some-image", Versions: []gardencorev1beta1.MachineImageVersion{ - { - ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "4.5.6"}, - CapabilityFlavors: imageFlavors, - }, + {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "4.5.6"}}, }, }, }, @@ -103,13 +98,19 @@ var _ = DescribeTableSubtree("NamespacedCloudProfile controller tests", func(isC Name: "some-image", Versions: []gardencorev1beta1.MachineImageVersion{ {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "4.5.6", ExpirationDate: &expirationDateFuture}}, - {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "7.8.9"}, CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}, Architectures: []string{"amd64"}, CapabilityFlavors: imageFlavors}, + {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "7.8.9"}, CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}}, // no capabilities defined, as spec.MachineCapabilities will be used }, }, { Name: "custom-image", Versions: []gardencorev1beta1.MachineImageVersion{ - {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "1.1.2"}, CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}, Architectures: []string{"amd64"}, CapabilityFlavors: imageFlavors}, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "1.1.2"}, + CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}, + Architectures: []string{"amd64"}, + // explicitly define capabilities 1.1.2 is not in parent cloudprofile + CapabilityFlavors: imageFlavors, + }, }, UpdateStrategy: &updateStrategy, }, @@ -142,15 +143,14 @@ var _ = DescribeTableSubtree("NamespacedCloudProfile controller tests", func(isC Name: "some-image", Versions: []gardencorev1beta1.MachineImageVersion{ { - ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "7.8.9"}, - CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}, - Architectures: []string{"amd64"}, - CapabilityFlavors: imageFlavors, + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "7.8.9"}, + CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}, + Architectures: []string{"amd64"}, }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "4.5.6", ExpirationDate: &expirationDateFuture}, - CRI: []gardencorev1beta1.CRI{{Name: "containerd", ContainerRuntimes: nil}}, Architectures: []string{"amd64"}, - CapabilityFlavors: imageFlavors, + CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}, + Architectures: []string{"amd64"}, }, }, UpdateStrategy: &updateStrategy, @@ -183,6 +183,7 @@ var _ = DescribeTableSubtree("NamespacedCloudProfile controller tests", func(isC Memory: resource.MustParse("2Gi"), Usable: &usable, Architecture: &architecture, + Capabilities: capabilities, }}, Regions: []gardencorev1beta1.Region{ {Name: "some-region"}, @@ -541,7 +542,11 @@ var _ = DescribeTableSubtree("NamespacedCloudProfile controller tests", func(isC { Name: "some-image", Versions: []gardencorev1beta1.MachineImageVersion{ - {ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "4.5.6", ExpirationDate: &expirationDatePast}}, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "4.5.6", ExpirationDate: &expirationDatePast}, + Architectures: []string{"amd64"}, + CapabilityFlavors: imageFlavors, + }, }, }, } @@ -556,10 +561,9 @@ var _ = DescribeTableSubtree("NamespacedCloudProfile controller tests", func(isC Name: "some-image", Versions: []gardencorev1beta1.MachineImageVersion{ { - ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "4.5.6"}, - CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}, - Architectures: []string{"amd64"}, - CapabilityFlavors: imageFlavors, + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: "4.5.6"}, + CRI: []gardencorev1beta1.CRI{{Name: "containerd"}}, + Architectures: []string{"amd64"}, }, }, UpdateStrategy: ptr.To(gardencorev1beta1.UpdateStrategyMajor), From 0aa0905616fbfb12a220ee88ad859760554bde76 Mon Sep 17 00:00:00 2001 From: rhubenov Date: Mon, 3 Nov 2025 16:00:21 +0200 Subject: [PATCH 019/176] Switch to using `matches` for config to include the kernel logs (#13328) --- .../components/opentelemetrycollector/config_test.go | 10 +++++----- .../templates/opentelemetry-collector-config.yaml.tpl | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/config_test.go b/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/config_test.go index 7e087c3cceb..4cc55cf9c12 100644 --- a/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/config_test.go +++ b/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/config_test.go @@ -98,11 +98,11 @@ receivers: journald/journal: start_at: beginning storage: file_storage - units: - - kernel - - kubelet.service - - containerd.service - - gardener-node-agent.service + matches: + - _TRANSPORT: kernel + - _SYSTEMD_UNIT: kubelet.service + - _SYSTEMD_UNIT: containerd.service + - _SYSTEMD_UNIT: gardener-node-agent.service operators: - type: move from: body._SYSTEMD_UNIT diff --git a/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/templates/opentelemetry-collector-config.yaml.tpl b/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/templates/opentelemetry-collector-config.yaml.tpl index 2cd0340da65..7a1c7c67eee 100644 --- a/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/templates/opentelemetry-collector-config.yaml.tpl +++ b/pkg/component/extensions/operatingsystemconfig/original/components/opentelemetrycollector/templates/opentelemetry-collector-config.yaml.tpl @@ -14,11 +14,11 @@ receivers: journald/journal: start_at: beginning storage: file_storage - units: - - kernel - - kubelet.service - - containerd.service - - gardener-node-agent.service + matches: + - _TRANSPORT: kernel + - _SYSTEMD_UNIT: kubelet.service + - _SYSTEMD_UNIT: containerd.service + - _SYSTEMD_UNIT: gardener-node-agent.service operators: - type: move from: body._SYSTEMD_UNIT From 7115486100ea6f78435695d7088a46c2481877e0 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Singh Date: Mon, 3 Nov 2025 20:20:11 +0530 Subject: [PATCH 020/176] Fix docs (#13331) --- docs/deployment/setup_gardener.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment/setup_gardener.md b/docs/deployment/setup_gardener.md index 764119fc6eb..21f4b40588a 100644 --- a/docs/deployment/setup_gardener.md +++ b/docs/deployment/setup_gardener.md @@ -281,7 +281,7 @@ NAME K8S VERSION GARDENER VERSION LAST OPERATION RUNTIME VIRTUA crazy-botany 1.31.1 v1.110.0 Succeeded True True True True 1d ``` -In case the automatic DNS record creation is not supported, a record with the external address of the `virtual-garden-istio-ingress` (found in namespace `istio-ingressgateway`) service has to be created manually. +In case the automatic DNS record creation is not supported, a record with the external address of the `istio-ingressgateway` (found in namespace `virtual-garden-istio-ingress`) service has to be created manually. To obtain credentials and interact with the virtual Garden cluster, please follow this [guide](../concepts/operator.md#virtual-garden-kubeconfig). From 803429aafecde13526fcdaeef91bc155dff9f8e8 Mon Sep 17 00:00:00 2001 From: Tobias Schlicht <155947440+tobschli@users.noreply.github.com> Date: Mon, 3 Nov 2025 18:08:12 +0100 Subject: [PATCH 021/176] Support for Kubernetes 1.34 (#12883) * feat: Allow instantiating v1.34 Kubernetes clients * docs: Update supported Kubernetes versions in docs * docs: align docs with usage * feat: Maintain feature gates for v1.34 ./hack/compare-k8s-feature-gates.sh 1.33 1.34 Feature gates added in 1.34 compared to 1.33: ClearingNominatedNodeNameAfterBinding ContainerRestartRules DRAConsumableCapacity DRADeviceBindingConditions DRAExtendedResource DRASchedulerFilterTimeout DetectCacheInconsistency EnvFiles HostnameOverride InPlacePodVerticalScalingExclusiveMemory KubeletPodResourcesListUseActivePods MatchLabelKeysInPodTopologySpreadSelectorMerge NominatedNodeNameForExpectation PodCertificateRequest PreventStaticPodAPIReferences ProbeHostPodSecurityStandards RelaxedServiceNameValidation SchedulerAsyncAPICalls SizeBasedListCostEstimate StructuredAuthenticationConfigurationEgressSelector TokenRequestServiceAccountUIDValidation Feature gates removed in 1.34 compared to 1.33: DevicePluginCDIDevices ElasticIndexedJob LegacySidecarContainers PodDisruptionConditions StatefulSetStartOrdinal Feature gates locked to default true in 1.34 compared to 1.33: AnonymousAuthConfigurableEndpoints APIServerTracing AuthorizeNodeWithSelectors AuthorizeWithSelectors ConsistentListFromCache JobPodReplacementPolicy KubeletCgroupDriverFromCRI KubeletTracing MultiCIDRServiceAllocator NodeSwap OrderedNamespaceDeletion PodLifecycleSleepAction PodLifecycleSleepActionAllowZero ProbeHostPodSecurityStandards RecoverVolumeExpansionFailure RelaxedDNSSearchValidation RelaxedEnvironmentVariableValidation ResilientWatchCacheInitialization SchedulerQueueingHints SeparateTaintEvictionController StreamingCollectionEncodingToJSON StreamingCollectionEncodingToProtobuf StructuredAuthenticationConfiguration WinDSR WinOverlay Feature gates locked to default false in 1.34 compared to 1.33: * feat: Maintain kube-apiserver admission plugins for 1.34 ./hack/compare-k8s-admission-plugins.sh 1.33 1.34 Admission plugins added in 1.34 compared to 1.33: Admission plugins removed in 1.34 compared to 1.33: * feat: Maintain apigroups for 1.34 ./hack/compare-k8s-api-groups.sh 1.33 1.34 Kubernetes API group versions added in 1.34 compared to 1.33: resource.k8s.io/v1 Kubernetes API GVRs added in 1.34 compared to 1.33: admissionregistration.k8s.io/v1beta1/mutatingadmissionpolicies admissionregistration.k8s.io/v1beta1/mutatingadmissionpolicybindings certificates.k8s.io/v1alpha1/podcertificaterequests resource.k8s.io/v1/deviceclasses resource.k8s.io/v1/resourceclaims resource.k8s.io/v1/resourceclaimtemplates resource.k8s.io/v1/resourceslices storage.k8s.io/v1/volumeattributesclasses Kubernetes API group versions removed in 1.34 compared to 1.33: networking.k8s.io/v1alpha1 Kubernetes API GVRs removed in 1.34 compared to 1.33: networking.k8s.io/v1alpha1/ipaddresses networking.k8s.io/v1alpha1/servicecidrs resource.k8s.io/v1alpha3/deviceclasses resource.k8s.io/v1alpha3/resourceclaims resource.k8s.io/v1alpha3/resourceclaimtemplates resource.k8s.io/v1alpha3/resourceslices * feat: Maintain kubernetes-controller-manager controller mappings kube-controller-manager controllers added in 1.34 compared to 1.33: Added Controllers for API Group [certificates/v1alpha1]: podcertificaterequest-cleaner-controller Added Controllers for API Group [resource/v1]: device-taint-eviction-controller resourceclaim-controller Added Controllers for API Group [storage/v1]: volumeattributesclass-protection-controller kube-controller-manager controllers removed in 1.34 compared to 1.33: Removed Controllers for API Group [resource/v1beta1]: device-taint-eviction-controller resourceclaim-controller Removed Controllers for API Group [storage/v1beta1]: volumeattributesclass-protection-controller * feat: Maintain unneeded controllers for workerless Shoots for Kubernetes 1.34 * chore: Update DaemonSet controller comments * chore: Add Kubernetes v1.34.0 to provider-local cloud profile * chore: Bump kindest/node version to 1.34.0 * chore: Change example Shoot to refer CredentialsBinding * Set `trafficDistribution` to `PreferSameZone` for kubernetes versions `>= 1.34` * Adapt charts to new `trafficDistribution` value * Adapt tests to new `trafficDistribution` value * Update TopologyAwareRouting documentation * Address linting findings * Use `PreferClose` for ETCD's clientService trafficDistribution * --- SEPARATOR --- * Align `node-problem-detector` images Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> * Adapt to new labels for metrics https://github.com/kubernetes/kubernetes/pull/130161 * Enable `discovery.k8s.io/v1` API for workerless `Shoot`s Without it, the `kube-apiserver` does not start, because the `EndpointSlice` informers do not sync. * Fix wording / typo Co-authored-by: Marc Vornetran --------- Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Co-authored-by: Marc Vornetran --- README.md | 14 +- .../charts/runtime/templates/service.yaml | 5 +- .../admission-controller/service.yaml | 5 +- .../runtime/templates/apiserver/service.yaml | 5 +- .../components/cloudprofile/cloudprofile.yaml | 1 + docs/development/new-kubernetes-version.md | 2 +- docs/operations/topology_aware_routing.md | 18 +- example/provider-local/shoot.yaml | 2 +- hack/compute-k8s-controllers.sh | 1 + imagevector/containers.yaml | 19 + .../v1alpha1/validation/garden_test.go | 5 +- pkg/component/autoscaling/vpa/vpa_test.go | 23 + pkg/component/etcd/etcd/etcd.go | 6 + pkg/component/etcd/etcd/etcd_test.go | 4 +- .../admission_controller_test.go | 4 +- .../gardener/apiserver/apiserver_test.go | 35 +- .../dashboard/terminal/terminal_test.go | 13 + .../kubernetes/apiserver/deployment.go | 6 +- .../apiserverexposure/service_test.go | 15 + .../controllermanager/controllermanager.go | 4 + .../controllermanager_test.go | 4 + .../templates/service-internal.yaml | 5 +- .../garden-shoot/apiserver-overview.json | 6 +- .../apiserver-storage-details.json | 2 +- pkg/provider-local/node/Dockerfile | 2 +- .../helper/daemon_controller.go | 14 +- pkg/utils/gardener/topology_aware_routing.go | 7 +- .../gardener/topology_aware_routing_test.go | 21 + pkg/utils/kubernetes/controllers.go | 14 +- pkg/utils/validation/apigroups/apigroups.go | 23 +- pkg/utils/validation/features/featuregates.go | 511 +++++++++--------- .../validation/kubernetesversion/version.go | 1 + .../kubernetesversion/version_test.go | 3 +- 33 files changed, 505 insertions(+), 295 deletions(-) diff --git a/README.md b/README.md index fbfe9ecf56f..64d06e2ff6c 100644 --- a/README.md +++ b/README.md @@ -41,13 +41,13 @@ Currently, Gardener is certified for K8s versions up to v1.33, see [the conforma Continuous conformance test results of the latest stable Gardener release are uploaded regularly to the CNCF test grid: -| Provider/K8s | v1.33 | v1.32 | v1.31 | v1.30 | v1.29 | -| ------------ |-----|-----| ------------ | ------------ | ------------ | -| **AWS** | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20AWS) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20AWS) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20AWS) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20AWS) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20AWS) | -| **Azure** | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20Azure) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20Azure) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20Azure) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20Azure) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20Azure) | -| **GCP** | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20GCE) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20GCE) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20GCE) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20GCE) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20GCE) | -| **OpenStack** | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20OpenStack) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20OpenStack) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20OpenStack) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20OpenStack) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20OpenStack) | -| **Alicloud** | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20Alibaba%20Cloud) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20Alibaba%20Cloud) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20Alibaba%20Cloud) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20Alibaba%20Cloud) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20Alibaba%20Cloud) | +| Provider/K8s | v1.34 | v1.33 | v1.32 | v1.31 | v1.30 | v1.29 | +|---------------|-------|-----|-----| ------------ | ------------ | ------------ | +| **AWS** | N/A | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20AWS) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20AWS) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20AWS) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20AWS) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20AWS) | +| **Azure** | N/A | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20Azure) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20Azure) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20Azure) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20Azure) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20Azure) | +| **GCP** | N/A | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20GCE) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20GCE) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20GCE) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20GCE) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20GCE) | +| **OpenStack** | N/A | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20OpenStack) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20OpenStack) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20OpenStack) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20OpenStack) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20OpenStack) | +| **Alicloud** | N/A | [![Gardener v1.33 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.33%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.33%20Alibaba%20Cloud) | [![Gardener v1.32 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.32%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.32%20Alibaba%20Cloud) | [![Gardener v1.31 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.31%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.31%20Alibaba%20Cloud) | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20Alibaba%20Cloud) | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20Alibaba%20Cloud) | Get an overview of the test results at [testgrid](https://testgrid.k8s.io/conformance-gardener). diff --git a/charts/gardener/admission-local/charts/runtime/templates/service.yaml b/charts/gardener/admission-local/charts/runtime/templates/service.yaml index bf2c94637fd..a7c0846bb3d 100644 --- a/charts/gardener/admission-local/charts/runtime/templates/service.yaml +++ b/charts/gardener/admission-local/charts/runtime/templates/service.yaml @@ -22,6 +22,9 @@ spec: - port: 443 protocol: TCP targetPort: {{ .Values.webhookConfig.serverPort }} - {{- if and .Values.service.topologyAwareRouting.enabled (semverCompare ">= 1.31-0" .Capabilities.KubeVersion.Version) }} + {{- if and .Values.service.topologyAwareRouting.enabled (semverCompare ">= 1.31-0" .Capabilities.KubeVersion.Version) (semverCompare "< 1.34-0" .Capabilities.KubeVersion.Version) }} trafficDistribution: PreferClose {{- end }} + {{- if and .Values.service.topologyAwareRouting.enabled (semverCompare ">= 1.34-0" .Capabilities.KubeVersion.Version) }} + trafficDistribution: PreferSameZone + {{- end }} diff --git a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/service.yaml b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/service.yaml index 85f1089dd16..1f243fbc6ca 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/service.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/service.yaml @@ -37,7 +37,10 @@ spec: protocol: TCP port: {{ required ".Values.global.admission.config.server.metrics.port is required" .Values.global.admission.config.server.metrics.port }} targetPort: {{ required ".Values.global.admission.config.server.metrics.port is required" .Values.global.admission.config.server.metrics.port }} - {{- if and .Values.global.admission.service.topologyAwareRouting.enabled (semverCompare ">= 1.31-0" .Capabilities.KubeVersion.Version) }} + {{- if and .Values.global.admission.service.topologyAwareRouting.enabled (semverCompare ">= 1.31-0" .Capabilities.KubeVersion.Version) (semverCompare "< 1.34-0" .Capabilities.KubeVersion.Version) }} trafficDistribution: PreferClose {{- end }} + {{- if and .Values.global.admission.service.topologyAwareRouting.enabled (semverCompare ">= 1.34-0" .Capabilities.KubeVersion.Version) }} + trafficDistribution: PreferSameZone + {{- end }} {{- end }} diff --git a/charts/gardener/controlplane/charts/runtime/templates/apiserver/service.yaml b/charts/gardener/controlplane/charts/runtime/templates/apiserver/service.yaml index 4a07022b61c..235c0d14ab9 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/apiserver/service.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/apiserver/service.yaml @@ -30,7 +30,10 @@ spec: - port: 443 protocol: TCP targetPort: {{ .Values.global.apiserver.securePort | default 8443 }} - {{- if and .Values.global.apiserver.service.topologyAwareRouting.enabled (semverCompare ">= 1.31-0" .Capabilities.KubeVersion.Version) }} + {{- if and .Values.global.admission.service.topologyAwareRouting.enabled (semverCompare ">= 1.31-0" .Capabilities.KubeVersion.Version) (semverCompare "< 1.34-0" .Capabilities.KubeVersion.Version) }} trafficDistribution: PreferClose {{- end }} + {{- if and .Values.global.admission.service.topologyAwareRouting.enabled (semverCompare ">= 1.34-0" .Capabilities.KubeVersion.Version) }} + trafficDistribution: PreferSameZone + {{- end }} {{- end }} diff --git a/dev-setup/gardenconfig/components/cloudprofile/cloudprofile.yaml b/dev-setup/gardenconfig/components/cloudprofile/cloudprofile.yaml index 72aa13a25b0..c4d63fc10a7 100644 --- a/dev-setup/gardenconfig/components/cloudprofile/cloudprofile.yaml +++ b/dev-setup/gardenconfig/components/cloudprofile/cloudprofile.yaml @@ -8,6 +8,7 @@ spec: - name: local kubernetes: versions: + - version: 1.34.0 - version: 1.33.0 - version: 1.32.0 - version: 1.31.1 diff --git a/docs/development/new-kubernetes-version.md b/docs/development/new-kubernetes-version.md index 17ad2121a56..57759ce3fa3 100644 --- a/docs/development/new-kubernetes-version.md +++ b/docs/development/new-kubernetes-version.md @@ -104,7 +104,7 @@ There is a CI/CD job that runs periodically and releases a new `hyperkube` image - The list of supported versions is meanwhile maintained [here](../../pkg/utils/validation/kubernetesversion/version.go) in the `SupportedVersions` variable. - Maintain the Kubernetes feature gates used for validation of `Shoot` resources: - The feature gates are maintained in [this](../../pkg/utils/validation/features/featuregates.go) file. - - To maintain this list for new Kubernetes versions, run `hack/compare-k8s-feature-gates.sh ` (e.g. `hack/compare-k8s-feature-gates.sh v1.33 v1.34`). + - To maintain this list for new Kubernetes versions, run `hack/compare-k8s-feature-gates.sh ` (e.g. `hack/compare-k8s-feature-gates.sh 1.34 1.35`). - It will present 3 lists of feature gates: those added and those removed in `` compared to `` and feature gates that got locked to default in ``. - Add all added feature gates to the map with `` as `AddedInVersion` and no `RemovedInVersion`. - For any removed feature gates, add `` as `RemovedInVersion` to the already existing feature gate in the map. diff --git a/docs/operations/topology_aware_routing.md b/docs/operations/topology_aware_routing.md index 380d9c00c98..599b878a1e7 100644 --- a/docs/operations/topology_aware_routing.md +++ b/docs/operations/topology_aware_routing.md @@ -44,7 +44,7 @@ We reported the drawbacks related to the `TopologyAwareHints` feature in [kubern The `ServiceTrafficDistribution` allows expressing preferences for how traffic should be routed to Service endpoints. For more details, see [upstream documentation](https://kubernetes.io/docs/reference/networking/virtual-ips/#traffic-distribution) of the feature. -The `PreferClose` strategy allows traffic to be routed to Service endpoints in topology-aware and predictable manner. +The `PreferSameZone` strategy allows traffic to be routed to Service endpoints in topology-aware and predictable manner. It is simpler than `service.kubernetes.io/topology-mode: auto` - if there are Service endpoints which reside in the same zone as the client, traffic is routed to one of the endpoints within the same zone as the client. If the client's zone does not have any available Service endpoints, traffic is routed to any available endpoint within the cluster. ## How to make a Service topology-aware @@ -82,9 +82,9 @@ spec: trafficDistribution: PreferClose ``` -### How to make a Service topology-aware in Kubernetes >= 1.32 +### How to make a Service topology-aware in Kubernetes 1.32 to 1.33 -In Kubernetes >= 1.32, `ServiceTrafficDistribution` is being used to make a Service topology-aware. The `.spec.trafficDistribution` field has to be set to `PreferClose`: +In Kubernetes 1.32 to 1.33, `ServiceTrafficDistribution` is being used to make a Service topology-aware. The `.spec.trafficDistribution` field has to be set to `PreferClose`: ```yaml apiVersion: v1 @@ -93,6 +93,18 @@ spec: trafficDistribution: PreferClose ``` +### How to make a Service topology-aware in Kubernetes 1.34 and later + +The value `PreferClose` has been deprecated in favor of `PreferSameZone` and `PreferSameNode`. `PreferSameZone` is an alias for the existing `PreferClose` to clarify its semantics. For more information, read the details in the [Kubernetes deprecation announcement](https://kubernetes.io/blog/2025/08/27/kubernetes-v1-34-release/#preferclose-traffic-distribution-is-deprecated). +In Kubernetes 1.34 and later, `ServiceTrafficDistribution` is still used to make a Service topology-aware. The `.spec.trafficDistribution` field should be set to `PreferSameZone`: + +```yaml +apiVersion: v1 +kind: Service +spec: + trafficDistribution: PreferSameZone +``` + ## Prerequisites for making a Service topology-aware 1. The Pods backing the Service should be spread on most of the available zones. This constraint should be ensured with appropriate scheduling constraints (topology spread constraints, (anti-)affinity). Enabling the feature for a Service with a single backing Pod or Pods all located in the same zone does not lead to a benefit. diff --git a/example/provider-local/shoot.yaml b/example/provider-local/shoot.yaml index 1b44b52eeb7..2b00920dfda 100644 --- a/example/provider-local/shoot.yaml +++ b/example/provider-local/shoot.yaml @@ -9,7 +9,7 @@ metadata: spec: cloudProfile: name: local - secretBindingName: local # dummy, doesn't contain any credentials + credentialsBindingName: local # dummy, doesn't contain any credentials region: local networking: type: calico diff --git a/hack/compute-k8s-controllers.sh b/hack/compute-k8s-controllers.sh index abd7dd9fc73..1d0c9c4bb2c 100755 --- a/hack/compute-k8s-controllers.sh +++ b/hack/compute-k8s-controllers.sh @@ -57,6 +57,7 @@ declare -A path_map=( ["persistentvolume-binder-controller"]="pkg/controller/volume/persistentvolume/pv_controller_base.go" ["persistentvolume-expander-controller"]="pkg/controller/volume/expand/expand_controller.go" ["pod-garbage-collector-controller"]="pkg/controller/podgc/gc_controller.go" + ["podcertificaterequest-cleaner-controller"]="pkg/controller/certificates/cleaner/pcrcleaner.go" ["persistentvolume-protection-controller"]="pkg/controller/volume/pvprotection/pv_protection_controller.go" ["persistentvolumeclaim-protection-controller"]="pkg/controller/volume/pvcprotection/pvc_protection_controller.go" ["replicaset-controller"]="pkg/controller/replicaset/replica_set.go" diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 3690190f464..2de27c9cfdd 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -496,10 +496,29 @@ images: value: - type: 'githubTeam' teamname: 'gardener/gardener-core-networking-maintainers' + - name: node-problem-detector + sourceRepository: github.com/kubernetes/node-problem-detector + repository: registry.k8s.io/node-problem-detector/node-problem-detector + tag: "v1.34.0" + targetVersion: ">= 1.34" + labels: + - name: 'gardener.cloud/cve-categorisation' + value: + network_exposure: 'private' + authentication_enforced: false + user_interaction: 'end-user' + confidentiality_requirement: 'low' + integrity_requirement: 'high' + availability_requirement: 'low' + - name: 'cloud.gardener.cnudie/responsibles' + value: + - type: 'githubTeam' + teamname: 'gardener/mcm-maintainers' - name: node-problem-detector sourceRepository: github.com/kubernetes/node-problem-detector repository: registry.k8s.io/node-problem-detector/node-problem-detector tag: "v0.8.22" + targetVersion: "< 1.34" labels: - name: 'gardener.cloud/cve-categorisation' value: diff --git a/pkg/apis/operator/v1alpha1/validation/garden_test.go b/pkg/apis/operator/v1alpha1/validation/garden_test.go index 8317fa91a4d..db8961eda02 100644 --- a/pkg/apis/operator/v1alpha1/validation/garden_test.go +++ b/pkg/apis/operator/v1alpha1/validation/garden_test.go @@ -810,10 +810,7 @@ var _ = Describe("Validation Tests", func() { }, } - Expect(ValidateGarden(garden, extensions)).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeNotSupported), - "Field": Equal("spec.virtualCluster.kubernetes.version"), - })))) + Expect(ValidateGarden(garden, extensions)).To(BeEmpty()) }) It("should forbid setting rotate-etcd-encryption-key-start annotation when k8s version is >= v1.34", func() { diff --git a/pkg/component/autoscaling/vpa/vpa_test.go b/pkg/component/autoscaling/vpa/vpa_test.go index 96cb5ff06f7..8b78011ccf1 100644 --- a/pkg/component/autoscaling/vpa/vpa_test.go +++ b/pkg/component/autoscaling/vpa/vpa_test.go @@ -2255,6 +2255,29 @@ var _ = Describe("VPA", func() { valuesAdmissionController.TopologyAwareRoutingEnabled = true }) + When("runtime Kubernetes version is >= 1.34", func() { + BeforeEach(func() { + runtimeKubernetesVersion = semver.MustParse("1.34.0") + }) + + It("should successfully deploy with expected vpa-webhook service annotation, label and spec field", func() { + vpa = New(c, namespace, sm, Values{ + ClusterType: component.ClusterTypeShoot, + SecretNameServerCA: secretNameCA, + RuntimeKubernetesVersion: runtimeKubernetesVersion, + AdmissionController: valuesAdmissionController, + Recommender: valuesRecommender, + Updater: valuesUpdater, + }) + Expect(vpa.Deploy(ctx)).To(Succeed()) + + actual := &corev1.Service{} + Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: "vpa-webhook"}, actual)).To(Succeed()) + + Expect(actual.Spec.TrafficDistribution).To(PointTo(Equal(corev1.ServiceTrafficDistributionPreferSameZone))) + }) + }) + When("runtime Kubernetes version is >= 1.32", func() { BeforeEach(func() { runtimeKubernetesVersion = semver.MustParse("1.32.1") diff --git a/pkg/component/etcd/etcd/etcd.go b/pkg/component/etcd/etcd/etcd.go index 9a4140a6b11..301fc83fae9 100644 --- a/pkg/component/etcd/etcd/etcd.go +++ b/pkg/component/etcd/etcd/etcd.go @@ -50,6 +50,7 @@ import ( "github.com/gardener/gardener/pkg/utils/kubernetes/health" secretsutils "github.com/gardener/gardener/pkg/utils/secrets" secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" + versionutils "github.com/gardener/gardener/pkg/utils/version" ) // Class is a string type alias for etcd classes. @@ -260,6 +261,11 @@ func (e *etcd) Deploy(ctx context.Context) error { clientService := &corev1.Service{} gardenerutils.ReconcileTopologyAwareRoutingSettings(clientService, e.values.TopologyAwareRoutingEnabled, e.values.RuntimeKubernetesVersion) + // TODO(tobschli): Remove this once once etcd druid allows PreferSameZone. + if e.values.TopologyAwareRoutingEnabled && versionutils.ConstraintK8sGreaterEqual134.Check(e.values.RuntimeKubernetesVersion) { + clientService.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferClose) + } + ports := []networkingv1.NetworkPolicyPort{ {Port: ptr.To(intstr.FromInt32(etcdconstants.PortEtcdClient)), Protocol: ptr.To(corev1.ProtocolTCP)}, {Port: ptr.To(intstr.FromInt32(etcdconstants.PortBackupRestore)), Protocol: ptr.To(corev1.ProtocolTCP)}, diff --git a/pkg/component/etcd/etcd/etcd_test.go b/pkg/component/etcd/etcd/etcd_test.go index b8b064d7503..2944a4be7a8 100644 --- a/pkg/component/etcd/etcd/etcd_test.go +++ b/pkg/component/etcd/etcd/etcd_test.go @@ -186,7 +186,9 @@ var _ = Describe("Etcd", func() { }, } if topologyAwareRoutingEnabled { - if versionutils.ConstraintK8sGreaterEqual132.Check(runtimeKubernetesVersion) { + if versionutils.ConstraintK8sGreaterEqual134.Check(runtimeKubernetesVersion) { + clientService.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferSameZone) + } else if versionutils.ConstraintK8sGreaterEqual132.Check(runtimeKubernetesVersion) { clientService.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferClose) } else if versionutils.ConstraintK8sEqual131.Check(runtimeKubernetesVersion) { clientService.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferClose) diff --git a/pkg/component/gardener/admissioncontroller/admission_controller_test.go b/pkg/component/gardener/admissioncontroller/admission_controller_test.go index 8852a129a9a..6d7fe49904c 100644 --- a/pkg/component/gardener/admissioncontroller/admission_controller_test.go +++ b/pkg/component/gardener/admissioncontroller/admission_controller_test.go @@ -760,7 +760,9 @@ func service(namespace string, testValues Values) *corev1.Service { } if testValues.TopologyAwareRoutingEnabled { - if versionutils.ConstraintK8sGreaterEqual132.Check(testValues.RuntimeVersion) { + if versionutils.ConstraintK8sGreaterEqual134.Check(testValues.RuntimeVersion) { + svc.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferSameZone) + } else if versionutils.ConstraintK8sGreaterEqual132.Check(testValues.RuntimeVersion) { svc.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferClose) } else if versionutils.ConstraintK8sEqual131.Check(testValues.RuntimeVersion) { svc.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferClose) diff --git a/pkg/component/gardener/apiserver/apiserver_test.go b/pkg/component/gardener/apiserver/apiserver_test.go index 744b8632ca6..36a799c1a9d 100644 --- a/pkg/component/gardener/apiserver/apiserver_test.go +++ b/pkg/component/gardener/apiserver/apiserver_test.go @@ -207,7 +207,6 @@ var _ = Describe("GardenerAPIServer", func() { Labels: map[string]string{ "app": "gardener", "role": "apiserver", - "endpoint-slice-hints.resources.gardener.cloud/consider": "true", }, }, Spec: corev1.ServiceSpec{ @@ -221,7 +220,7 @@ var _ = Describe("GardenerAPIServer", func() { Protocol: corev1.ProtocolTCP, TargetPort: intstr.FromInt32(8443), }}, - TrafficDistribution: ptr.To(corev1.ServiceTrafficDistributionPreferClose), + TrafficDistribution: ptr.To(corev1.ServiceTrafficDistributionPreferSameZone), }, } @@ -1366,6 +1365,23 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook Expect(managedResourceSecretVirtual.Labels["resources.gardener.cloud/garbage-collectable-reference"]).To(Equal("true")) }) + Context("Kubernetes version >= 1.34", func() { + BeforeEach(func() { + values.RuntimeVersion = semver.MustParse("1.34.0") + deployer = New(fakeClient, namespace, fakeSecretManager, values) + }) + + It("should successfully deploy all resources", func() { + expectedRuntimeObjects = append( + expectedRuntimeObjects, + podDisruptionBudget, + serviceRuntime, + ) + + Expect(managedResourceRuntime).To(consistOf(expectedRuntimeObjects...)) + }) + }) + Context("Kubernetes version >= 1.31", func() { BeforeEach(func() { values.RuntimeVersion = semver.MustParse("1.31.0") @@ -1373,6 +1389,9 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook }) It("should successfully deploy all resources", func() { + serviceRuntime.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferClose) + metav1.SetMetaDataLabel(&serviceRuntime.ObjectMeta, "endpoint-slice-hints.resources.gardener.cloud/consider", "true") + expectedRuntimeObjects = append( expectedRuntimeObjects, podDisruptionBudget, @@ -1381,6 +1400,11 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook Expect(managedResourceRuntime).To(consistOf(expectedRuntimeObjects...)) }) + + AfterEach(func() { + delete(serviceRuntime.Labels, "endpoint-slice-hints.resources.gardener.cloud/consider") + serviceRuntime.Spec.TrafficDistribution = nil + }) }) Context("Kubernetes version < 1.31", func() { @@ -1391,6 +1415,7 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook It("should successfully deploy all resources", func() { metav1.SetMetaDataAnnotation(&serviceRuntime.ObjectMeta, "service.kubernetes.io/topology-mode", "auto") + metav1.SetMetaDataLabel(&serviceRuntime.ObjectMeta, "endpoint-slice-hints.resources.gardener.cloud/consider", "true") serviceRuntime.Spec.TrafficDistribution = nil expectedRuntimeObjects = append( @@ -1401,6 +1426,12 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook Expect(managedResourceRuntime).To(consistOf(expectedRuntimeObjects...)) }) + + AfterEach(func() { + delete(serviceRuntime.Annotations, "service.kubernetes.io/topology-mode") + delete(serviceRuntime.Labels, "endpoint-slice-hints.resources.gardener.cloud/consider") + serviceRuntime.Spec.TrafficDistribution = nil + }) }) }) }) diff --git a/pkg/component/gardener/dashboard/terminal/terminal_test.go b/pkg/component/gardener/dashboard/terminal/terminal_test.go index dd9fd9bf7ea..03c577577ba 100644 --- a/pkg/component/gardener/dashboard/terminal/terminal_test.go +++ b/pkg/component/gardener/dashboard/terminal/terminal_test.go @@ -725,6 +725,19 @@ server: topologyAwareRouting = true }) + When("runtime Kubernetes version is >= 1.34", func() { + BeforeEach(func() { + runtimeVersion = semver.MustParse("1.34.0") + }) + + It("should successfully deploy all resources", func() { + service.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferSameZone) + + Expect(managedResourceRuntime).To(consistOf(expectedRuntimeObjects...)) + Expect(managedResourceVirtual).To(consistOf(expectedVirtualObjects...)) + }) + }) + When("runtime Kubernetes version is >= 1.32", func() { BeforeEach(func() { runtimeVersion = semver.MustParse("1.32.1") diff --git a/pkg/component/kubernetes/apiserver/deployment.go b/pkg/component/kubernetes/apiserver/deployment.go index d08493aec55..dc87ecac884 100644 --- a/pkg/component/kubernetes/apiserver/deployment.go +++ b/pkg/component/kubernetes/apiserver/deployment.go @@ -36,6 +36,7 @@ import ( netutils "github.com/gardener/gardener/pkg/utils/net" "github.com/gardener/gardener/pkg/utils/secrets" secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" + versionutils "github.com/gardener/gardener/pkg/utils/version" ) const ( @@ -439,11 +440,14 @@ func (k *kubeAPIServer) computeKubeAPIServerArgs() []string { "apps/v1": false, "autoscaling/v2": false, "batch/v1": false, - "discovery.k8s.io/v1": false, "policy/v1": false, "storage.k8s.io/v1/csinodes": false, } + if versionutils.ConstraintK8sLess134.Check(k.values.Version) { + disableAPIs["discovery.k8s.io/v1"] = false + } + // Allow users to explicitly enable disabled APIs via RuntimeConfig. k.values.RuntimeConfig = utils.MergeStringMaps(disableAPIs, k.values.RuntimeConfig) } diff --git a/pkg/component/kubernetes/apiserverexposure/service_test.go b/pkg/component/kubernetes/apiserverexposure/service_test.go index 2a1fdb1e8de..701fcb6a8ee 100644 --- a/pkg/component/kubernetes/apiserverexposure/service_test.go +++ b/pkg/component/kubernetes/apiserverexposure/service_test.go @@ -191,6 +191,21 @@ var _ = Describe("#Service", func() { values.TopologyAwareRoutingEnabled = true }) + When("runtime Kubernetes version is >= 1.34", func() { + BeforeEach(func() { + values.RuntimeKubernetesVersion = semver.MustParse("1.34.0") + }) + + It("should successfully deploy with expected kube-apiserver service annotation, label and spec field", func() { + Expect(defaultDepWaiter.Deploy(ctx)).To(Succeed()) + + actual := &corev1.Service{} + Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: expectedName}, actual)).To(Succeed()) + + Expect(actual.Spec.TrafficDistribution).To(PointTo(Equal(corev1.ServiceTrafficDistributionPreferSameZone))) + }) + }) + When("runtime Kubernetes version is >= 1.32", func() { BeforeEach(func() { values.RuntimeKubernetesVersion = semver.MustParse("1.32.1") diff --git a/pkg/component/kubernetes/controllermanager/controllermanager.go b/pkg/component/kubernetes/controllermanager/controllermanager.go index 5f3db5c1047..e0e32bf6663 100644 --- a/pkg/component/kubernetes/controllermanager/controllermanager.go +++ b/pkg/component/kubernetes/controllermanager/controllermanager.go @@ -726,6 +726,10 @@ func (k *kubeControllerManager) computeCommand(port int32) []string { if versionutils.ConstraintK8sGreaterEqual133.Check(k.values.TargetVersion) { controllersToDisable.Insert("device-taint-eviction-controller") } + + if versionutils.ConstraintK8sGreaterEqual134.Check(k.values.TargetVersion) { + controllersToDisable.Insert("podcertificaterequest-cleaner-controller") + } } command = append(command, diff --git a/pkg/component/kubernetes/controllermanager/controllermanager_test.go b/pkg/component/kubernetes/controllermanager/controllermanager_test.go index a0fac419b58..5fd8884da22 100644 --- a/pkg/component/kubernetes/controllermanager/controllermanager_test.go +++ b/pkg/component/kubernetes/controllermanager/controllermanager_test.go @@ -1054,6 +1054,10 @@ func commandForKubernetesVersion( controllers = append(controllers, "-device-taint-eviction-controller") } + if versionutils.ConstraintK8sGreaterEqual134.Check(version) { + controllers = append(controllers, "-podcertificaterequest-cleaner-controller") + } + controllers = append(controllers, "-endpoint", "-ephemeral-volume", diff --git a/pkg/component/networking/istio/charts/istio/istio-ingress/templates/service-internal.yaml b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/service-internal.yaml index 4dee8012736..8375df4ce11 100644 --- a/pkg/component/networking/istio/charts/istio/istio-ingress/templates/service-internal.yaml +++ b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/service-internal.yaml @@ -32,6 +32,9 @@ spec: - IPv4 ipFamilyPolicy: PreferDualStack {{- end }} -{{- if semverCompare ">= 1.31-0" .Values.kubernetesVersion }} +{{- if and (semverCompare ">= 1.31-0" .Values.kubernetesVersion) (semverCompare "< 1.34-0" .Values.kubernetesVersion) }} trafficDistribution: PreferClose {{- end }} +{{- if semverCompare ">= 1.34-0" .Values.kubernetesVersion }} + trafficDistribution: PreferSameZone +{{- end }} diff --git a/pkg/component/observability/plutono/dashboards/garden-shoot/apiserver-overview.json b/pkg/component/observability/plutono/dashboards/garden-shoot/apiserver-overview.json index 2e708a4807c..8308e8e7a51 100644 --- a/pkg/component/observability/plutono/dashboards/garden-shoot/apiserver-overview.json +++ b/pkg/component/observability/plutono/dashboards/garden-shoot/apiserver-overview.json @@ -2958,7 +2958,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.90,sum(rate(apiserver_watch_events_sizes_bucket{job=~\"$job\",pod=~\"$pod\"}[$__rate_interval])) by(le,kind))", + "expr": "histogram_quantile(0.90,sum(rate(apiserver_watch_events_sizes_bucket{job=~\"$job\",pod=~\"$pod\"}[$__rate_interval])) by(le,resource))", "interval": "", "legendFormat": "{{kind}}", "refId": "A" @@ -3159,7 +3159,7 @@ "steppedLine": true, "targets": [ { - "expr": "sum(increase(watch_cache_capacity_increase_total{job=~\"$job\",pod=~\"$pod\"}[$__rate_interval])) by (resource)", + "expr": "sum(increase(watch_cache_capacity_increase_total{job=~\"$job\",pod=~\"$pod\"}[$__rate_interval])) by (group,resource)", "interval": "", "legendFormat": "{{resource}}", "refId": "A" @@ -3261,7 +3261,7 @@ "steppedLine": true, "targets": [ { - "expr": "sum(increase(watch_cache_capacity_decrease_total{job=~\"$job\",pod=~\"$pod\"}[$__rate_interval])) by (resource)", + "expr": "sum(increase(watch_cache_capacity_decrease_total{job=~\"$job\",pod=~\"$pod\"}[$__rate_interval])) by (group,resource)", "interval": "", "legendFormat": "{{resource}}", "refId": "A" diff --git a/pkg/component/observability/plutono/dashboards/garden-shoot/apiserver-storage-details.json b/pkg/component/observability/plutono/dashboards/garden-shoot/apiserver-storage-details.json index 3bd2eb14e6e..faa696e5137 100644 --- a/pkg/component/observability/plutono/dashboards/garden-shoot/apiserver-storage-details.json +++ b/pkg/component/observability/plutono/dashboards/garden-shoot/apiserver-storage-details.json @@ -74,7 +74,7 @@ }, "targets": [ { - "expr": "sum(rate(etcd_request_duration_seconds_bucket{job=~\"$apiserver\",type=~\"$type\",operation=~\"$operation\"}[$__rate_interval])) by (le)", + "expr": "sum(rate(etcd_request_duration_seconds_bucket{job=~\"$apiserver\",group=\"$group\",resource=\"$resource\",operation=~\"$operation\"}[$__rate_interval])) by (le)", "format": "heatmap", "interval": "", "intervalFactor": 3, diff --git a/pkg/provider-local/node/Dockerfile b/pkg/provider-local/node/Dockerfile index 78c829576c4..a4d81b1611a 100644 --- a/pkg/provider-local/node/Dockerfile +++ b/pkg/provider-local/node/Dockerfile @@ -1,4 +1,4 @@ -FROM kindest/node:v1.33.4@sha256:25a6018e48dfcaee478f4a59af81157a437f15e6e140bf103f85a2e7cd0cbbf2 +FROM kindest/node:v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a RUN apt-get update -yq && \ apt-get install -yq --no-install-recommends wget apparmor apparmor-utils jq openssh-server sudo logrotate diff --git a/pkg/resourcemanager/controller/node/criticalcomponents/helper/daemon_controller.go b/pkg/resourcemanager/controller/node/criticalcomponents/helper/daemon_controller.go index 0b95c433950..08db215e8aa 100644 --- a/pkg/resourcemanager/controller/node/criticalcomponents/helper/daemon_controller.go +++ b/pkg/resourcemanager/controller/node/criticalcomponents/helper/daemon_controller.go @@ -46,7 +46,8 @@ import ( // Returns true when a daemonset should continue running on a node if a daemonset pod is already // running on that node. // -// Copied from https://github.com/kubernetes/kubernetes/blob/v1.33.0/pkg/controller/daemon/daemon_controller.go#L1299-L1330 +// Copied from https://github.com/kubernetes/kubernetes/blob/v1.34.0/pkg/controller/daemon/daemon_controller.go#L1275-L1306 +// First if block semantically equally changed due to golangci-lint finding QF1001. func NodeShouldRunDaemonPod(node *corev1.Node, ds *appsv1.DaemonSet) (bool, bool) { pod := NewPod(ds, node.Name) @@ -73,7 +74,7 @@ func NodeShouldRunDaemonPod(node *corev1.Node, ds *appsv1.DaemonSet) (bool, bool } // predicates checks if a DaemonSet's pod can run on a node. -// Copied from https://github.com/kubernetes/kubernetes/blob/v1.33.0/pkg/controller/daemon/daemon_controller.go#L1332-L1342 +// Copied from https://github.com/kubernetes/kubernetes/blob/v1.34.0/pkg/controller/daemon/daemon_controller.go#L1308-L1318 func predicates(pod *corev1.Pod, node *corev1.Node, taints []corev1.Taint) (fitsNodeName, fitsNodeAffinity, fitsTaints bool) { fitsNodeName = len(pod.Spec.NodeName) == 0 || pod.Spec.NodeName == node.Name // Ignore parsing errors for backwards compatibility. @@ -86,7 +87,7 @@ func predicates(pod *corev1.Pod, node *corev1.Node, taints []corev1.Taint) (fits } // NewPod creates a new pod -// Copied from https://github.com/kubernetes/kubernetes/blob/v1.33.0/pkg/controller/daemon/daemon_controller.go#L1344-L1354 +// Copied from https://github.com/kubernetes/kubernetes/blob/v1.34.0/pkg/controller/daemon/daemon_controller.go#L1320-L1330 func NewPod(ds *appsv1.DaemonSet, nodeName string) *corev1.Pod { newPod := &corev1.Pod{Spec: ds.Spec.Template.Spec, ObjectMeta: ds.Spec.Template.ObjectMeta} newPod.Namespace = ds.Namespace @@ -99,7 +100,7 @@ func NewPod(ds *appsv1.DaemonSet, nodeName string) *corev1.Pod { } // AddOrUpdateDaemonPodTolerations apply necessary tolerations to DaemonSet Pods, e.g. node.kubernetes.io/not-ready:NoExecute. -// Copied from https://github.com/kubernetes/kubernetes/blob/v1.33.0/pkg/controller/daemon/util/daemonset_util.go#L47-L102 +// Copied from https://github.com/kubernetes/kubernetes/blob/v1.34.0/pkg/controller/daemon/util/daemonset_util.go#L47-L102 func AddOrUpdateDaemonPodTolerations(spec *corev1.PodSpec) { // DaemonSet pods shouldn't be deleted by NodeController in case of node problems. // Add infinite toleration for taint notReady:NoExecute here @@ -158,7 +159,7 @@ func AddOrUpdateDaemonPodTolerations(spec *corev1.PodSpec) { // AddOrUpdateTolerationInPodSpec tries to add a toleration to the toleration list in PodSpec. // Returns true if something was updated, false otherwise. -// Copied from https://github.com/kubernetes/kubernetes/blob/v1.33.0/pkg/apis/core/v1/helper/helpers.go#L261-L287 +// Copied from https://github.com/kubernetes/kubernetes/blob/v1.34.0/pkg/apis/core/v1/helper/helpers.go#L261-L287 func AddOrUpdateTolerationInPodSpec(spec *corev1.PodSpec, toleration *corev1.Toleration) bool { podTolerations := spec.Tolerations @@ -187,7 +188,8 @@ func AddOrUpdateTolerationInPodSpec(spec *corev1.PodSpec, toleration *corev1.Tol // Semantic can do semantic deep equality checks for core objects. // Example: apiequality.Semantic.DeepEqual(aPod, aPodWithNonNilButEmptyMaps) == true -// Copied from https://github.com/kubernetes/kubernetes/blob/v1.33.0/pkg/apis/core/helper/helpers.go#L92-L114 +// Copied from https://github.com/kubernetes/kubernetes/blob/v1.34.0/pkg/apis/core/helper/helpers.go#L92-L114 +// Changed time comparisons changed due to golangci-lint finding QF1009. var Semantic = conversion.EqualitiesOrDie( func(a, b resource.Quantity) bool { // Ignore formatting, only care that numeric value stayed the same. diff --git a/pkg/utils/gardener/topology_aware_routing.go b/pkg/utils/gardener/topology_aware_routing.go index a0baa51c1d0..baa87391787 100644 --- a/pkg/utils/gardener/topology_aware_routing.go +++ b/pkg/utils/gardener/topology_aware_routing.go @@ -26,8 +26,11 @@ func ReconcileTopologyAwareRoutingSettings(service *corev1.Service, topologyAwar if !topologyAwareRoutingEnabled { return } - - if versionutils.ConstraintK8sGreaterEqual132.Check(k8sVersion) { + if versionutils.ConstraintK8sGreaterEqual134.Check(k8sVersion) { + // For Kubernetes >= 1.34, only use the PreferSameZone strategy of the ServiceTrafficDistribution feature. + // PreferClose is deprecated. PreferSameZone is a new alias for PreferClose (https://kubernetes.io/blog/2025/08/27/kubernetes-v1-34-release/#preferclose-traffic-distribution-is-deprecated). + service.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferSameZone) + } else if versionutils.ConstraintK8sGreaterEqual132.Check(k8sVersion) { // For Kubernetes >= 1.32, only use the PreferClose strategy of the ServiceTrafficDistribution feature. service.Spec.TrafficDistribution = ptr.To(corev1.ServiceTrafficDistributionPreferClose) } else if versionutils.ConstraintK8sEqual131.Check(k8sVersion) { diff --git a/pkg/utils/gardener/topology_aware_routing_test.go b/pkg/utils/gardener/topology_aware_routing_test.go index b146f3e19a7..a0a7f3a2e58 100644 --- a/pkg/utils/gardener/topology_aware_routing_test.go +++ b/pkg/utils/gardener/topology_aware_routing_test.go @@ -17,6 +17,27 @@ import ( var _ = Describe("TopologyAwareRouting", func() { Describe("#ReconcileTopologyAwareRoutingSettings", func() { + When("K8s version >= 1.34", func() { + It("should set traffic distribution field when topology-aware routing is enabled", func() { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.kubernetes.io/topology-aware-hints": "auto", + "service.kubernetes.io/topology-mode": "auto", + }, + Labels: map[string]string{"endpoint-slice-hints.resources.gardener.cloud/consider": "true"}, + }, + } + + ReconcileTopologyAwareRoutingSettings(service, true, semver.MustParse("1.34.0")) + + Expect(service.Annotations).NotTo(HaveKey("service.kubernetes.io/topology-mode")) + Expect(service.Annotations).NotTo(HaveKey("service.kubernetes.io/topology-aware-hints")) + Expect(service.Labels).NotTo(HaveKey("endpoint-slice-hints.resources.gardener.cloud/consider")) + Expect(service.Spec.TrafficDistribution).To(PointTo(Equal(corev1.ServiceTrafficDistributionPreferSameZone))) + }) + }) + When("K8s version >= 1.32", func() { It("should set traffic distribution field when topology-aware routing is enabled", func() { service := &corev1.Service{ diff --git a/pkg/utils/kubernetes/controllers.go b/pkg/utils/kubernetes/controllers.go index be6dabe4553..fa0dc4ba10d 100644 --- a/pkg/utils/kubernetes/controllers.go +++ b/pkg/utils/kubernetes/controllers.go @@ -51,6 +51,7 @@ var APIGroupControllerMap = map[string]map[string]versionutils.VersionRange{ }, "certificates/v1alpha1": { "kube-apiserver-serving-clustertrustbundle-publisher-controller": {AddedInVersion: "1.32"}, + "podcertificaterequest-cleaner-controller": {AddedInVersion: "1.34"}, }, "certificates/v1beta1": { "csrsigning": {}, @@ -82,6 +83,10 @@ var APIGroupControllerMap = map[string]map[string]versionutils.VersionRange{ "rbac/v1": { "clusterrole-aggregation": {}, }, + "resource/v1": { + "device-taint-eviction-controller": {AddedInVersion: "1.34"}, + "resourceclaim-controller": {AddedInVersion: "1.34"}, + }, "resource/v1alpha2": { "resource-claim-controller": {RemovedInVersion: "1.31"}, }, @@ -89,14 +94,15 @@ var APIGroupControllerMap = map[string]map[string]versionutils.VersionRange{ "resource-claim-controller": {AddedInVersion: "1.31", RemovedInVersion: "1.32"}, }, "resource/v1beta1": { - "resource-claim-controller": {AddedInVersion: "1.32"}, - "device-taint-eviction-controller": {AddedInVersion: "1.33"}, + "device-taint-eviction-controller": {AddedInVersion: "1.33", RemovedInVersion: "1.34"}, + "resource-claim-controller": {AddedInVersion: "1.32", RemovedInVersion: "1.34"}, }, "storage/v1": { - "selinux-warning-controller": {AddedInVersion: "1.32"}, + "selinux-warning-controller": {AddedInVersion: "1.32"}, + "volumeattributesclass-protection-controller": {AddedInVersion: "1.34"}, }, "storage/v1beta1": { - "volumeattributesclass-protection-controller": {AddedInVersion: "1.32"}, + "volumeattributesclass-protection-controller": {AddedInVersion: "1.32", RemovedInVersion: "1.34"}, }, "storagemigration/v1alpha1": { "storage-version-migrator-controller": {AddedInVersion: "1.30"}, diff --git a/pkg/utils/validation/apigroups/apigroups.go b/pkg/utils/validation/apigroups/apigroups.go index c86c778f342..8488fc7d4ed 100644 --- a/pkg/utils/validation/apigroups/apigroups.go +++ b/pkg/utils/validation/apigroups/apigroups.go @@ -49,7 +49,7 @@ var ( "flowcontrol.apiserver.k8s.io/v1beta3": {}, "internal.apiserver.k8s.io/v1alpha1": {}, "networking.k8s.io/v1": {Required: true, RequiredForWorkerless: true}, - "networking.k8s.io/v1alpha1": {}, + "networking.k8s.io/v1alpha1": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.34"}}, "networking.k8s.io/v1beta1": {}, "node.k8s.io/v1": {}, "node.k8s.io/v1alpha1": {}, @@ -59,6 +59,7 @@ var ( "rbac.authorization.k8s.io/v1": {Required: true, RequiredForWorkerless: true}, "rbac.authorization.k8s.io/v1alpha1": {}, "rbac.authorization.k8s.io/v1beta1": {}, + "resource.k8s.io/v1": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, "resource.k8s.io/v1alpha2": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, "resource.k8s.io/v1alpha3": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, "resource.k8s.io/v1beta1": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, @@ -83,6 +84,8 @@ var ( "admissionregistration.k8s.io/v1alpha1/mutatingadmissionpolicybindings": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, "admissionregistration.k8s.io/v1alpha1/validatingadmissionpolicies": {}, "admissionregistration.k8s.io/v1alpha1/validatingadmissionpolicybindings": {}, + "admissionregistration.k8s.io/v1beta1/mutatingadmissionpolicies": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "admissionregistration.k8s.io/v1beta1/mutatingadmissionpolicybindings": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, "admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations": {}, "admissionregistration.k8s.io/v1beta1/validatingadmissionpolicies": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, "admissionregistration.k8s.io/v1beta1/validatingadmissionpolicybindings": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, @@ -109,6 +112,7 @@ var ( "batch/v1beta1/cronjobs": {}, "certificates.k8s.io/v1/certificatesigningrequests": {Required: true}, "certificates.k8s.io/v1alpha1/clustertrustbundles": {}, + "certificates.k8s.io/v1alpha1/podcertificaterequests": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, "certificates.k8s.io/v1beta1/clustertrustbundles": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, "certificates.k8s.io/v1beta1/certificatesigningrequests": {}, "coordination.k8s.io/v1/leases": {Required: true}, @@ -139,8 +143,8 @@ var ( "networking.k8s.io/v1/ipaddresses": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, "networking.k8s.io/v1/networkpolicies": {}, "networking.k8s.io/v1/servicecidrs": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "networking.k8s.io/v1alpha1/ipaddresses": {}, - "networking.k8s.io/v1alpha1/servicecidrs": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "networking.k8s.io/v1alpha1/ipaddresses": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.34"}}, + "networking.k8s.io/v1alpha1/servicecidrs": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29", RemovedInVersion: "1.34"}}, "networking.k8s.io/v1beta1/ingressclasses": {}, "networking.k8s.io/v1beta1/ingresses": {}, "networking.k8s.io/v1beta1/ipaddresses": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, @@ -162,6 +166,10 @@ var ( "rbac.authorization.k8s.io/v1beta1/clusterroles": {}, "rbac.authorization.k8s.io/v1beta1/rolebindings": {}, "rbac.authorization.k8s.io/v1beta1/roles": {}, + "resource.k8s.io/v1/deviceclasses": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "resource.k8s.io/v1/resourceclaims": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "resource.k8s.io/v1/resourceclaimtemplates": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "resource.k8s.io/v1/resourceslices": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, "resource.k8s.io/v1alpha2/podschedulingcontexts": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, "resource.k8s.io/v1alpha2/resourceclaimparameters": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30", RemovedInVersion: "1.31"}}, "resource.k8s.io/v1alpha2/resourceclaims": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, @@ -169,12 +177,12 @@ var ( "resource.k8s.io/v1alpha2/resourceclasses": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, "resource.k8s.io/v1alpha2/resourceclassparameters": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30", RemovedInVersion: "1.31"}}, "resource.k8s.io/v1alpha2/resourceslices": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30", RemovedInVersion: "1.31"}}, - "resource.k8s.io/v1alpha3/deviceclasses": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "resource.k8s.io/v1alpha3/deviceclasses": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31", RemovedInVersion: "1.34"}}, "resource.k8s.io/v1alpha3/devicetaintrules": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, "resource.k8s.io/v1alpha3/podschedulingcontexts": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31", RemovedInVersion: "1.32"}}, - "resource.k8s.io/v1alpha3/resourceclaims": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "resource.k8s.io/v1alpha3/resourceclaimtemplates": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "resource.k8s.io/v1alpha3/resourceslices": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "resource.k8s.io/v1alpha3/resourceclaims": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31", RemovedInVersion: "1.34"}}, + "resource.k8s.io/v1alpha3/resourceclaimtemplates": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31", RemovedInVersion: "1.34"}}, + "resource.k8s.io/v1alpha3/resourceslices": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31", RemovedInVersion: "1.34"}}, "resource.k8s.io/v1beta1/deviceclasses": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, "resource.k8s.io/v1beta1/resourceclaims": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, "resource.k8s.io/v1beta1/resourceclaimtemplates": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, @@ -191,6 +199,7 @@ var ( "storage.k8s.io/v1/csistoragecapacities": {}, "storage.k8s.io/v1/storageclasses": {}, "storage.k8s.io/v1/volumeattachments": {}, + "storage.k8s.io/v1/volumeattributesclasses": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, "storage.k8s.io/v1alpha1/csistoragecapacities": {}, "storage.k8s.io/v1alpha1/volumeattachments": {}, "storage.k8s.io/v1alpha1/volumeattributesclasses": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, diff --git a/pkg/utils/validation/features/featuregates.go b/pkg/utils/validation/features/featuregates.go index 9238b7dee69..4a2ef75c1db 100644 --- a/pkg/utils/validation/features/featuregates.go +++ b/pkg/utils/validation/features/featuregates.go @@ -27,251 +27,272 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "AllAlpha": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.17"}}, "AllBeta": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.17"}}, - "AdmissionWebhookMatchConditions": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "AggregatedDiscoveryRemoveBetaType": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "AllowDNSOnlyNodeCSR": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "AllowInsecureKubeletCertificateSigningRequests": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "AllowOverwriteTerminationGracePeriodSeconds": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "AllowServiceLBStatusOnNonLB": {LockedValue: false, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "AllowUnsafeMalformedObjectDeletion": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "AllowParsingUserUIDFromCertAuth": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "APIListChunking": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "APIPriorityAndFairness": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "APIResponseCompression": {}, - "APISelfSubjectReview": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "APIServerIdentity": {}, - "APIServerTracing": {}, - "APIServingWithRoutine": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "AggregatedDiscoveryEndpoint": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "AnonymousAuthConfigurableEndpoints": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "AnyVolumeDataSource": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, - "AppArmor": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "AppArmorFields": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30", RemovedInVersion: "1.33"}}, - "AuthorizeNodeWithSelectors": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "AuthorizeWithSelectors": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "BtreeWatchCache": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "CBORServingAndStorage": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "CloudControllerManagerWebhook": {}, - "CloudDualStackNodeIPs": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.32"}}, - "ClusterTrustBundle": {}, - "ClusterTrustBundleProjection": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "ComponentFlagz": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "ComponentSLIs": {}, - "ComponentStatusz": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "ConcurrentWatchObjectDecode": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "ContainerCheckpoint": {}, - "ContainerStopSignals": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "ContextualLogging": {LockedValue: true, LockedToDefaultInVersion: "1.30"}, - "ConsistentHTTPGetHandlers": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.26", RemovedInVersion: "1.31"}}, - "ConsistentListFromCache": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "CoordinatedLeaderElection": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "CPUManager": {LockedValue: true, LockedToDefaultInVersion: "1.26", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "CPUManagerPolicyAlphaOptions": {}, - "CPUManagerPolicyBetaOptions": {}, - "CPUManagerPolicyOptions": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, - "CRDValidationRatcheting": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "CronJobsScheduledAnnotation": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "CrossNamespaceVolumeDataSource": {}, - "CSIMigrationAzureFile": {LockedValue: true, LockedToDefaultInVersion: "1.27", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "CSIMigrationPortworx": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, - "CSIMigrationRBD": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "CSINodeExpandSecret": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "CSIVolumeHealth": {}, - "CustomCPUCFSQuotaPeriod": {}, - "CustomResourceFieldSelectors": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "CustomResourceValidationExpressions": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "DeclarativeValidation": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "DeclarativeValidationTakeover": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "DefaultHostNetworkHostPortsInPodTemplates": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.31"}}, - "DeploymentReplicaSetTerminatingReplicas": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "DevicePluginCDIDevices": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "DisableAllocatorDualWrite": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "DisableCloudProviders": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "DisableCPUQuotaWithExclusiveCPUs": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "DisableKubeletCloudCredentialProviders": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "DisableNodeKubeProxyVersion": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "DRAAdminAccess": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "DRAControlPlaneController": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.26", RemovedInVersion: "1.32"}}, - "DRADeviceTaints": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "DRAPartitionableDevices": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "DRAPrioritizedList": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "DRAResourceClaimDeviceStatus": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "DynamicResourceAllocation": {}, - "EfficientWatchResumption": {LockedValue: true, LockedToDefaultInVersion: "1.24", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "ElasticIndexedJob": {LockedValue: true, LockedToDefaultInVersion: "1.31"}, - "EventedPLEG": {}, - "ExecProbeTimeout": {}, - "ExpandedDNSConfig": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "ExperimentalHostUserNamespaceDefaulting": {LockedValue: false, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "ExternalServiceAccountTokenSigner": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "GitRepoVolumeDriver": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "GracefulNodeShutdown": {}, - "GracefulNodeShutdownBasedOnPodPriority": {}, - "HonorPVReclaimPolicy": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, - "HPAConfigurableTolerance": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "HPAContainerMetrics": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.20", RemovedInVersion: "1.32"}}, - "HPAScaleToZero": {}, - "IPTablesOwnershipCleanup": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "ImageMaximumGCAge": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "ImageVolume": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "InPlacePodVerticalScaling": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.27"}}, - "InPlacePodVerticalScalingAllocatedStatus": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "InPlacePodVerticalScalingExclusiveCPUs": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "InTreePluginAWSUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "InTreePluginAzureDiskUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "InTreePluginAzureFileUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "InTreePluginGCEUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "InTreePluginOpenStackUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "InTreePluginPortworxUnregister": {}, - "InTreePluginRBDUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "InTreePluginvSphereUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "InformerResourceVersion": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "JobBackoffLimitPerIndex": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "JobManagedBy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "JobPodFailurePolicy": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "JobPodReplacementPolicy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "JobReadyPods": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "JobSuccessPolicy": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "KMSv1": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "KMSv2": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{AddedInVersion: "1.25", RemovedInVersion: "1.32"}}, - "KMSv2KDF": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.32"}}, - "KubeletEnsureSecretPulledImages": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "KubeletCgroupDriverFromCRI": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "KubeletCrashLoopBackOffMax": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "KubeletFineGrainedAuthz": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "KubeletInUserNamespace": {}, - "KubeletPodResources": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "KubeletPodResourcesDynamicResources": {}, - "KubeletPodResourcesGet": {}, - "KubeletPodResourcesGetAllocatable": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "KubeletPSI": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "KubeletRegistrationGetOnExistsOnly": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "KubeletSeparateDiskGC": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "KubeletServiceAccountTokenForCredentialProviders": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "KubeletTracing": {}, - "KubeProxyDrainingTerminatingNodes": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.33"}}, - "LegacyServiceAccountTokenCleanUp": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.32"}}, - "LegacyServiceAccountTokenTracking": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{AddedInVersion: "1.26", RemovedInVersion: "1.30"}}, - "LegacySidecarContainers": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "LoadBalancerIPMode": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "ListFromCacheSnapshot": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "LocalStorageCapacityIsolationFSQuotaMonitoring": {}, - "LogarithmicScaleDown": {LockedValue: true, LockedToDefaultInVersion: "1.31"}, - "MatchLabelKeysInPodAffinity": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "MatchLabelKeysInPodTopologySpread": {}, - "MaxUnavailableStatefulSet": {}, - "MemoryManager": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.21"}}, - "MemoryQoS": {}, - "MinDomainsInPodTopologySpread": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.24", RemovedInVersion: "1.32"}}, - "MinimizeIPTablesRestore": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{AddedInVersion: "1.26", RemovedInVersion: "1.30"}}, - "MultiCIDRServiceAllocator": {}, - "MutableCSINodeAllocatableCount": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "MutatingAdmissionPolicy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "NewVolumeManagerReconstruction": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.25", RemovedInVersion: "1.32"}}, - "NFTablesProxyMode": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "NodeInclusionPolicyInPodTopologySpread": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, - "NodeLogQuery": {}, - "NodeOutOfServiceVolumeDetach": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{AddedInVersion: "1.24", RemovedInVersion: "1.32"}}, - "NodeSwap": {}, - "OpenAPIEnums": {}, - "OrderedNamespaceDeletion": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "PDBUnhealthyPodEvictionPolicy": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "PersistentVolumeLastPhaseTransitionTime": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.33"}}, - "PodAndContainerStatsFromCRI": {}, - "PodDeletionCost": {}, - "PodDisruptionConditions": {LockedValue: true, LockedToDefaultInVersion: "1.31"}, - "PodHostIPs": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.32"}}, - "PodIndexLabel": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "PodLevelResources": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "PodLifecycleSleepAction": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "PodLifecycleSleepActionAllowZero": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "PodLogsQuerySplitStreams": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "PodObservedGenerationTracking": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "PodReadyToStartContainersCondition": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "PodSchedulingReadiness": {LockedValue: true, LockedToDefaultInVersion: "1.30"}, - "PodTopologyLabelsAdmission": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "PortForwardWebsockets": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "PreferAlignCpusByUncoreCache": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "PreferSameTrafficDistribution": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "ProcMountType": {}, - "ProxyTerminatingEndpoints": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "QOSReserved": {}, - "ReadWriteOncePod": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "RecoverVolumeExpansionFailure": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "RecursiveReadOnlyMounts": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "ReduceDefaultCrashLoopBackOffDecay": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "RelaxedDNSSearchValidation": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "RelaxedEnvironmentVariableValidation": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "ReloadKubeletServerCertificateFile": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "RemainingItemCount": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "RemoteRequestHeaderUID": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "RemoveSelfLink": {LockedValue: true, LockedToDefaultInVersion: "1.24", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "ResilientWatchCacheInitialization": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "ResourceHealthStatus": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "RetryGenerateName": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "RotateKubeletServerCertificate": {}, - "RuntimeClassInImageCriApi": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "SchedulerAsyncPreemption": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "SchedulerPopFromBackoffQ": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "SchedulerQueueingHints": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "SecurityContextDeny": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, - "SELinuxChangePolicy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "SELinuxMount": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "SELinuxMountReadWriteOncePod": {LockedValue: true, LockedToDefaultInVersion: "1.29"}, - "SeparateCacheWatchRPC": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "SeparateTaintEvictionController": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "ServerSideApply": {LockedValue: true, LockedToDefaultInVersion: "1.22", VersionRange: versionutils.VersionRange{AddedInVersion: "1.14", RemovedInVersion: "1.32"}}, - "ServerSideFieldValidation": {LockedValue: true, LockedToDefaultInVersion: "1.27", VersionRange: versionutils.VersionRange{AddedInVersion: "1.23", RemovedInVersion: "1.32"}}, - "ServiceAccountNodeAudienceRestriction": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "ServiceAccountTokenJTI": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "ServiceAccountTokenNodeBinding": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "ServiceAccountTokenNodeBindingValidation": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "ServiceAccountTokenPodNodeInfo": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "ServiceNodePortStaticSubrange": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, - "ServiceTrafficDistribution": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "SidecarContainers": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "SizeMemoryBackedVolumes": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.20"}}, - "SkipReadOnlyValidationGCE": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.31"}}, - "StableLoadBalancerNodeSet": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.32"}}, - "StatefulSetAutoDeletePVC": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.23"}}, - "StatefulSetStartOrdinal": {LockedValue: true, LockedToDefaultInVersion: "1.31"}, - "StorageCapacityScoring": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "StorageNamespaceIndex": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "StorageVersionAPI": {}, - "StorageVersionHash": {}, - "StorageVersionMigrator": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "StreamingCollectionEncodingToJSON": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "StreamingCollectionEncodingToProtobuf": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "StrictCostEnforcementForVAP": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "StrictCostEnforcementForWebhooks": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "StrictIPCIDRValidation": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, - "StructuredAuthenticationConfiguration": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "StructuredAuthorizationConfiguration": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "SupplementalGroupsPolicy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "SystemdWatchdog": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "TopologyAwareHints": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, - "TopologyManagerPolicyAlphaOptions": {}, - "TopologyManagerPolicyBetaOptions": {}, - "TopologyManagerPolicyOptions": {LockedValue: true, LockedToDefaultInVersion: "1.32"}, - "TranslateStreamCloseWebsocketRequests": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "UnauthenticatedHTTP2DOSMitigation": {}, - "UnknownVersionInteroperabilityProxy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "UserNamespacesPodSecurityStandards": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "UserNamespacesSupport": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, - "ValidatingAdmissionPolicy": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.32"}}, - "VolumeAttributesClass": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, - "VolumeCapacityPriority": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "WatchBookmark": {LockedValue: true, LockedToDefaultInVersion: "1.17", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, - "WatchCacheInitializationPostStartHook": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, - "WatchFromStorageWithoutResourceVersion": {LockedValue: false, LockedToDefaultInVersion: "1.33"}, - "WatchList": {}, - "WatchListClient": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, - "WinDSR": {}, - "WinOverlay": {}, - "WindowsCPUAndMemoryAffinity": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "WindowsGracefulNodeShutdown": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, - "WindowsHostNetwork": {}, - "ZeroLimitedNominalConcurrencyShares": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29", RemovedInVersion: "1.32"}}, + "AdmissionWebhookMatchConditions": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "AggregatedDiscoveryRemoveBetaType": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "AllowDNSOnlyNodeCSR": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "AllowInsecureKubeletCertificateSigningRequests": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "AllowOverwriteTerminationGracePeriodSeconds": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "AllowServiceLBStatusOnNonLB": {LockedValue: false, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "AllowUnsafeMalformedObjectDeletion": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "AllowParsingUserUIDFromCertAuth": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "APIListChunking": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "APIPriorityAndFairness": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "APIResponseCompression": {}, + "APISelfSubjectReview": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "APIServerIdentity": {}, + "APIServerTracing": {LockedValue: true, LockedToDefaultInVersion: "1.34"}, + "APIServingWithRoutine": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "AggregatedDiscoveryEndpoint": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "AnonymousAuthConfigurableEndpoints": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "AnyVolumeDataSource": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, + "AppArmor": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "AppArmorFields": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30", RemovedInVersion: "1.33"}}, + "AuthorizeNodeWithSelectors": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "AuthorizeWithSelectors": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "BtreeWatchCache": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "CBORServingAndStorage": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "ClearingNominatedNodeNameAfterBinding": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "CloudControllerManagerWebhook": {}, + "CloudDualStackNodeIPs": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.32"}}, + "ClusterTrustBundle": {}, + "ClusterTrustBundleProjection": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "ComponentFlagz": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "ComponentSLIs": {}, + "ComponentStatusz": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "ConcurrentWatchObjectDecode": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "ContainerCheckpoint": {}, + "ContainerRestartRules": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "ContainerStopSignals": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "ContextualLogging": {LockedValue: true, LockedToDefaultInVersion: "1.30"}, + "ConsistentHTTPGetHandlers": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.26", RemovedInVersion: "1.31"}}, + "ConsistentListFromCache": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "CoordinatedLeaderElection": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "CPUManager": {LockedValue: true, LockedToDefaultInVersion: "1.26", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "CPUManagerPolicyAlphaOptions": {}, + "CPUManagerPolicyBetaOptions": {}, + "CPUManagerPolicyOptions": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, + "CRDValidationRatcheting": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "CronJobsScheduledAnnotation": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "CrossNamespaceVolumeDataSource": {}, + "CSIMigrationAzureFile": {LockedValue: true, LockedToDefaultInVersion: "1.27", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "CSIMigrationPortworx": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, + "CSIMigrationRBD": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "CSINodeExpandSecret": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "CSIVolumeHealth": {}, + "CustomCPUCFSQuotaPeriod": {}, + "CustomResourceFieldSelectors": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "CustomResourceValidationExpressions": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "DeclarativeValidation": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "DeclarativeValidationTakeover": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "DefaultHostNetworkHostPortsInPodTemplates": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.31"}}, + "DeploymentReplicaSetTerminatingReplicas": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "DetectCacheInconsistency": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "DevicePluginCDIDevices": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.34"}}, + "DisableAllocatorDualWrite": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "DisableCloudProviders": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "DisableCPUQuotaWithExclusiveCPUs": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "DisableKubeletCloudCredentialProviders": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "DisableNodeKubeProxyVersion": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "DRAAdminAccess": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "DRAConsumableCapacity": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "DRAControlPlaneController": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.26", RemovedInVersion: "1.32"}}, + "DRADeviceBindingConditions": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "DRADeviceTaints": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "DRAExtendedResource": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "DRAPartitionableDevices": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "DRAPrioritizedList": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "DRAResourceClaimDeviceStatus": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "DRASchedulerFilterTimeout": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "DynamicResourceAllocation": {}, + "EfficientWatchResumption": {LockedValue: true, LockedToDefaultInVersion: "1.24", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "ElasticIndexedJob": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.34"}}, + "EnvFiles": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "EventedPLEG": {}, + "ExecProbeTimeout": {}, + "ExpandedDNSConfig": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "ExperimentalHostUserNamespaceDefaulting": {LockedValue: false, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "ExternalServiceAccountTokenSigner": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "GitRepoVolumeDriver": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "GracefulNodeShutdown": {}, + "GracefulNodeShutdownBasedOnPodPriority": {}, + "HonorPVReclaimPolicy": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, + "HostnameOverride": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "HPAConfigurableTolerance": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "HPAContainerMetrics": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.20", RemovedInVersion: "1.32"}}, + "HPAScaleToZero": {}, + "IPTablesOwnershipCleanup": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "ImageMaximumGCAge": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "ImageVolume": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "InPlacePodVerticalScaling": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.27"}}, + "InPlacePodVerticalScalingAllocatedStatus": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "InPlacePodVerticalScalingExclusiveCPUs": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "InPlacePodVerticalScalingExclusiveMemory": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "InTreePluginAWSUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "InTreePluginAzureDiskUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "InTreePluginAzureFileUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "InTreePluginGCEUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "InTreePluginOpenStackUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "InTreePluginPortworxUnregister": {}, + "InTreePluginRBDUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "InTreePluginvSphereUnregister": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "InformerResourceVersion": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "JobBackoffLimitPerIndex": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "JobManagedBy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "JobPodFailurePolicy": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "JobPodReplacementPolicy": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "JobReadyPods": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "JobSuccessPolicy": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "KMSv1": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "KMSv2": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{AddedInVersion: "1.25", RemovedInVersion: "1.32"}}, + "KMSv2KDF": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.32"}}, + "KubeletEnsureSecretPulledImages": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "KubeletCgroupDriverFromCRI": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "KubeletCrashLoopBackOffMax": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "KubeletFineGrainedAuthz": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "KubeletInUserNamespace": {}, + "KubeletPodResources": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "KubeletPodResourcesDynamicResources": {}, + "KubeletPodResourcesListUseActivePods": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "KubeletPodResourcesGet": {}, + "KubeletPodResourcesGetAllocatable": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "KubeletPSI": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "KubeletRegistrationGetOnExistsOnly": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "KubeletSeparateDiskGC": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "KubeletServiceAccountTokenForCredentialProviders": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "KubeletTracing": {LockedValue: true, LockedToDefaultInVersion: "1.34"}, + "KubeProxyDrainingTerminatingNodes": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.33"}}, + "LegacyServiceAccountTokenCleanUp": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.32"}}, + "LegacyServiceAccountTokenTracking": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{AddedInVersion: "1.26", RemovedInVersion: "1.30"}}, + "LegacySidecarContainers": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33", RemovedInVersion: "1.34"}}, + "LoadBalancerIPMode": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "ListFromCacheSnapshot": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "LocalStorageCapacityIsolationFSQuotaMonitoring": {}, + "LogarithmicScaleDown": {LockedValue: true, LockedToDefaultInVersion: "1.31"}, + "MatchLabelKeysInPodAffinity": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "MatchLabelKeysInPodTopologySpread": {}, + "MatchLabelKeysInPodTopologySpreadSelectorMerge": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "MaxUnavailableStatefulSet": {}, + "MemoryManager": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.21"}}, + "MemoryQoS": {}, + "MinDomainsInPodTopologySpread": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.24", RemovedInVersion: "1.32"}}, + "MinimizeIPTablesRestore": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{AddedInVersion: "1.26", RemovedInVersion: "1.30"}}, + "MultiCIDRServiceAllocator": {LockedValue: true, LockedToDefaultInVersion: "1.34"}, + "MutableCSINodeAllocatableCount": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "MutatingAdmissionPolicy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "NewVolumeManagerReconstruction": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.25", RemovedInVersion: "1.32"}}, + "NFTablesProxyMode": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "NodeInclusionPolicyInPodTopologySpread": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, + "NodeLogQuery": {}, + "NodeOutOfServiceVolumeDetach": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{AddedInVersion: "1.24", RemovedInVersion: "1.32"}}, + "NodeSwap": {LockedValue: true, LockedToDefaultInVersion: "1.34"}, + "NominatedNodeNameForExpectation": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "OpenAPIEnums": {}, + "OrderedNamespaceDeletion": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "PDBUnhealthyPodEvictionPolicy": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "PersistentVolumeLastPhaseTransitionTime": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.33"}}, + "PodAndContainerStatsFromCRI": {}, + "PodCertificateRequest": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "PodDeletionCost": {}, + "PodDisruptionConditions": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.34"}}, + "PodHostIPs": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.32"}}, + "PodIndexLabel": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "PodLevelResources": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "PodLifecycleSleepAction": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "PodLifecycleSleepActionAllowZero": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "PodLogsQuerySplitStreams": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "PodObservedGenerationTracking": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "PodReadyToStartContainersCondition": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "PodSchedulingReadiness": {LockedValue: true, LockedToDefaultInVersion: "1.30"}, + "PodTopologyLabelsAdmission": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "PortForwardWebsockets": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "PreferAlignCpusByUncoreCache": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "PreferSameTrafficDistribution": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "PreventStaticPodAPIReferences": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "ProbeHostPodSecurityStandards": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "ProcMountType": {}, + "ProxyTerminatingEndpoints": {LockedValue: true, LockedToDefaultInVersion: "1.28", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "QOSReserved": {}, + "ReadWriteOncePod": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "RecoverVolumeExpansionFailure": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "RecursiveReadOnlyMounts": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "ReduceDefaultCrashLoopBackOffDecay": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "RelaxedDNSSearchValidation": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "RelaxedEnvironmentVariableValidation": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "RelaxedServiceNameValidation": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "ReloadKubeletServerCertificateFile": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "RemainingItemCount": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "RemoteRequestHeaderUID": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "RemoveSelfLink": {LockedValue: true, LockedToDefaultInVersion: "1.24", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "ResilientWatchCacheInitialization": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "ResourceHealthStatus": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "RetryGenerateName": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "RotateKubeletServerCertificate": {}, + "RuntimeClassInImageCriApi": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "SchedulerAsyncAPICalls": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "SchedulerAsyncPreemption": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "SchedulerPopFromBackoffQ": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "SchedulerQueueingHints": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "SecurityContextDeny": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.30"}}, + "SELinuxChangePolicy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "SELinuxMount": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "SELinuxMountReadWriteOncePod": {LockedValue: true, LockedToDefaultInVersion: "1.29"}, + "SeparateCacheWatchRPC": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "SeparateTaintEvictionController": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "ServerSideApply": {LockedValue: true, LockedToDefaultInVersion: "1.22", VersionRange: versionutils.VersionRange{AddedInVersion: "1.14", RemovedInVersion: "1.32"}}, + "ServerSideFieldValidation": {LockedValue: true, LockedToDefaultInVersion: "1.27", VersionRange: versionutils.VersionRange{AddedInVersion: "1.23", RemovedInVersion: "1.32"}}, + "ServiceAccountNodeAudienceRestriction": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "ServiceAccountTokenJTI": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "ServiceAccountTokenNodeBinding": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "ServiceAccountTokenNodeBindingValidation": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "ServiceAccountTokenPodNodeInfo": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "ServiceNodePortStaticSubrange": {LockedValue: true, LockedToDefaultInVersion: "1.29", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.31"}}, + "ServiceTrafficDistribution": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "SidecarContainers": {LockedValue: true, LockedToDefaultInVersion: "1.33", VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "SizeBasedListCostEstimate": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "SizeMemoryBackedVolumes": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.20"}}, + "SkipReadOnlyValidationGCE": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28", RemovedInVersion: "1.31"}}, + "StableLoadBalancerNodeSet": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.32"}}, + "StatefulSetAutoDeletePVC": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.23"}}, + "StatefulSetStartOrdinal": {LockedValue: true, LockedToDefaultInVersion: "1.31", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.34"}}, + "StorageCapacityScoring": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "StorageNamespaceIndex": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "StorageVersionAPI": {}, + "StorageVersionHash": {}, + "StorageVersionMigrator": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "StreamingCollectionEncodingToJSON": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "StreamingCollectionEncodingToProtobuf": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "StrictCostEnforcementForVAP": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "StrictCostEnforcementForWebhooks": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "StrictIPCIDRValidation": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.33"}}, + "StructuredAuthenticationConfiguration": {LockedValue: true, LockedToDefaultInVersion: "1.34", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "StructuredAuthenticationConfigurationEgressSelector": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "StructuredAuthorizationConfiguration": {LockedValue: true, LockedToDefaultInVersion: "1.32", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "SupplementalGroupsPolicy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "SystemdWatchdog": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "TokenRequestServiceAccountUIDValidation": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.34"}}, + "TopologyAwareHints": {LockedValue: true, LockedToDefaultInVersion: "1.33"}, + "TopologyManagerPolicyAlphaOptions": {}, + "TopologyManagerPolicyBetaOptions": {}, + "TopologyManagerPolicyOptions": {LockedValue: true, LockedToDefaultInVersion: "1.32"}, + "TranslateStreamCloseWebsocketRequests": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "UnauthenticatedHTTP2DOSMitigation": {}, + "UnknownVersionInteroperabilityProxy": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "UserNamespacesPodSecurityStandards": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "UserNamespacesSupport": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.28"}}, + "ValidatingAdmissionPolicy": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.32"}}, + "VolumeAttributesClass": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.29"}}, + "VolumeCapacityPriority": {VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "WatchBookmark": {LockedValue: true, LockedToDefaultInVersion: "1.17", VersionRange: versionutils.VersionRange{RemovedInVersion: "1.33"}}, + "WatchCacheInitializationPostStartHook": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.31"}}, + "WatchFromStorageWithoutResourceVersion": {LockedValue: false, LockedToDefaultInVersion: "1.33"}, + "WatchList": {}, + "WatchListClient": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.30"}}, + "WinDSR": {LockedValue: true, LockedToDefaultInVersion: "1.34"}, + "WinOverlay": {LockedValue: true, LockedToDefaultInVersion: "1.34"}, + "WindowsCPUAndMemoryAffinity": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "WindowsGracefulNodeShutdown": {VersionRange: versionutils.VersionRange{AddedInVersion: "1.32"}}, + "WindowsHostNetwork": {}, + "ZeroLimitedNominalConcurrencyShares": {LockedValue: true, LockedToDefaultInVersion: "1.30", VersionRange: versionutils.VersionRange{AddedInVersion: "1.29", RemovedInVersion: "1.32"}}, } // IsFeatureGateSupported returns true if the given feature gate is supported for the given Kubernetes version. diff --git a/pkg/utils/validation/kubernetesversion/version.go b/pkg/utils/validation/kubernetesversion/version.go index 2bffe094614..87390240128 100644 --- a/pkg/utils/validation/kubernetesversion/version.go +++ b/pkg/utils/validation/kubernetesversion/version.go @@ -18,6 +18,7 @@ var SupportedVersions = []string{ "1.31", "1.32", "1.33", + "1.34", } // CheckIfSupported checks if the provided version is part of the supported Kubernetes versions list. diff --git a/pkg/utils/validation/kubernetesversion/version_test.go b/pkg/utils/validation/kubernetesversion/version_test.go index 7a40321fcfb..6900c41e16b 100644 --- a/pkg/utils/validation/kubernetesversion/version_test.go +++ b/pkg/utils/validation/kubernetesversion/version_test.go @@ -24,6 +24,7 @@ var _ = Describe("Version", func() { Entry("1.31", "1.31", Succeed()), Entry("1.32", "1.32", Succeed()), Entry("1.33", "1.33", Succeed()), - Entry("1.34", "1.34", MatchError(ContainSubstring("unsupported kubernetes version"))), + Entry("1.34", "1.34", Succeed()), + Entry("1.35", "1.35", MatchError(ContainSubstring("unsupported kubernetes version"))), ) }) From cacc4343cb209d2445430ebd5c8825c508ba7090 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 3 Nov 2025 18:08:31 +0100 Subject: [PATCH 022/176] Update module github.com/gardener/cert-management to v0.19.0 (#13327) --- go.mod | 21 +++++++++++---------- go.sum | 46 ++++++++++++++++++++++++---------------------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 0f376e35c27..b578bb677c1 100644 --- a/go.mod +++ b/go.mod @@ -13,12 +13,12 @@ require ( github.com/distribution/distribution/v3 v3.0.0 github.com/docker/cli v28.5.1+incompatible github.com/fluent/fluent-operator/v3 v3.3.0 - github.com/gardener/cert-management v0.18.0 + github.com/gardener/cert-management v0.19.0 github.com/gardener/dependency-watchdog v1.6.0 github.com/gardener/etcd-druid/api v0.33.0 github.com/gardener/machine-controller-manager v0.60.2 github.com/gardener/terminal-controller-manager v0.34.0 - github.com/go-jose/go-jose/v4 v4.1.1 + github.com/go-jose/go-jose/v4 v4.1.3 github.com/go-logr/logr v1.4.3 github.com/go-test/deep v1.1.0 github.com/gogo/protobuf v1.3.2 @@ -137,10 +137,11 @@ require ( github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/errors v0.22.0 // indirect - github.com/go-openapi/jsonpointer v0.21.2 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/errors v0.22.3 // indirect + github.com/go-openapi/jsonpointer v0.22.1 // indirect + github.com/go-openapi/jsonreference v0.21.2 // indirect github.com/go-openapi/swag v0.23.1 // indirect + github.com/go-openapi/swag/jsonname v0.25.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gobuffalo/flect v1.0.3 // indirect @@ -166,7 +167,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect + github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/labstack/echo/v4 v4.13.4 // indirect @@ -175,7 +176,7 @@ require ( github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/miekg/dns v1.1.67 // indirect + github.com/miekg/dns v1.1.68 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -229,7 +230,7 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect go.etcd.io/etcd/client/v3 v3.6.4 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/collector/featuregate v1.37.0 // indirect go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 // indirect go.opentelemetry.io/contrib/exporters/autoexport v0.57.0 // indirect @@ -267,8 +268,8 @@ require ( golang.org/x/term v0.36.0 // indirect google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect - google.golang.org/grpc v1.75.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect + google.golang.org/grpc v1.76.0 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/go.sum b/go.sum index 4004cecb47d..a33960d6efe 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyT cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/compute v1.28.0 h1:OPtBxMcheSS+DWfci803qvPly3d4w7Eu5ztKBcFfzwk= -cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= -cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= @@ -236,8 +236,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/gardener/cert-management v0.18.0 h1:s2YhkN8z7lXe9En52GCeqQ9be10uEbLtH/FFAh6BVgQ= -github.com/gardener/cert-management v0.18.0/go.mod h1:9+JT+EBJB2OIX65EG+P1p/DZ/UJ3W8WR0h40ZjKbw+Q= +github.com/gardener/cert-management v0.19.0 h1:BNumdw748Pg9798NzxHmmpKuXFRLHSPuvcPKQHOiFcw= +github.com/gardener/cert-management v0.19.0/go.mod h1:u5OKwiDyUdCuW9vhDV92ozCVkynXUBrYCMHr4rVNiCY= github.com/gardener/dependency-watchdog v1.6.0 h1:ARCIbcNmhjefmV7ex8ADReeD2MPsEawwT/MoZKTQV/M= github.com/gardener/dependency-watchdog v1.6.0/go.mod h1:NXkna7bW5O+IGxLAX0KdEaW8yFREDfSHSccuoY+YZu0= github.com/gardener/etcd-druid/api v0.33.0 h1:YwgsYYldaLig2laJMAAMX/dg9/XsQx/LPz8+iL52V6w= @@ -254,8 +254,8 @@ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6O github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= -github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -268,18 +268,18 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= -github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= +github.com/go-openapi/errors v0.22.3 h1:k6Hxa5Jg1TUyZnOwV2Lh81j8ayNw5VVYLvKrp4zFKFs= +github.com/go-openapi/errors v0.22.3/go.mod h1:+WvbaBBULWCOna//9B9TbLNGSFOfF8lY9dw4hGiEiKQ= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.21.2 h1:AqQaNADVwq/VnkCmQg6ogE+M3FOsKTytwges0JdwVuA= -github.com/go-openapi/jsonpointer v0.21.2/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= +github.com/go-openapi/jsonpointer v0.22.1 h1:sHYI1He3b9NqJ4wXLoJDKmUmHkWy/L7rtEo92JUxBNk= +github.com/go-openapi/jsonpointer v0.22.1/go.mod h1:pQT9OsLkfz1yWoMgYFy4x3U5GY5nUlsOn1qSBH5MkCM= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/jsonreference v0.21.2 h1:Wxjda4M/BBQllegefXrY/9aq1fxBA8sI5M/lFU6tSWU= +github.com/go-openapi/jsonreference v0.21.2/go.mod h1:pp3PEjIsJ9CZDGCNOyXIQxsNuroxm8FAJ/+quA0yKzQ= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= @@ -287,6 +287,8 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-openapi/swag/jsonname v0.25.1 h1:Sgx+qbwa4ej6AomWC6pEfXrA6uP2RkaNjA9BR8a1RJU= +github.com/go-openapi/swag/jsonname v0.25.1/go.mod h1:71Tekow6UOLBD3wS7XhdT98g5J5GR13NOTQ9/6Q11Zo= github.com/go-resty/resty/v2 v2.15.3 h1:bqff+hcqAflpiF591hhJzNdkRsFhlB96CYfBwSFvql8= github.com/go-resty/resty/v2 v2.15.3/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -461,8 +463,8 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 h1:9Nu54bhS/H/Kgo2/7xNSUuC5G28VR8ljfrLKU2G4IjU= +github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12/go.mod h1:TBzl5BIHNXfS9+C35ZyJaklL7mLDbgUkcgXzSLa8Tk0= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -511,8 +513,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= -github.com/miekg/dns v1.1.67 h1:kg0EHj0G4bfT5/oOys6HhZw4vmMlnoZ+gDu8tJ/AlI0= -github.com/miekg/dns v1.1.67/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= +github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA= +github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -768,8 +770,8 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/collector/featuregate v1.37.0 h1:CjsHzjktiqq/dxid4Xkhuf3yD6oB/c7yRBWhokBJqpE= go.opentelemetry.io/collector/featuregate v1.37.0/go.mod h1:Y/KsHbvREENKvvN9RlpiWk/IGBK+CATBYzIIpU7nccc= go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 h1:UW0+QyeyBVhn+COBec3nGhfnFe5lwB0ic1JBVjzhk0w= @@ -1019,8 +1021,8 @@ google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eY google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:hL97c3SYopEHblzpxRL4lSs523++l8DYxGM1FQiYmb4= google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1029,8 +1031,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 0ac4022d6a12ffece3400c68c217955ef4925942 Mon Sep 17 00:00:00 2001 From: Georgi Baltiev <82998942+georgibaltiev@users.noreply.github.com> Date: Tue, 4 Nov 2025 08:24:11 +0200 Subject: [PATCH 023/176] Improve input validations for the `SecretBinding` and `CredentialsBinding` resources (#13258) * Validate an object reference's name and namespace fields against the DNS 1123 convention. * fix: reuse existing validation functions and refactor tests with additional values * add: else block --- pkg/apis/core/validation/secretbinding.go | 11 ++++ .../core/validation/secretbinding_test.go | 31 ++++++++++- pkg/apis/core/validation/utils.go | 9 ++++ pkg/apis/core/validation/utils_test.go | 53 ++++++++++++++----- 4 files changed, 89 insertions(+), 15 deletions(-) diff --git a/pkg/apis/core/validation/secretbinding.go b/pkg/apis/core/validation/secretbinding.go index 92b39e8f2ef..fb321fe2a24 100644 --- a/pkg/apis/core/validation/secretbinding.go +++ b/pkg/apis/core/validation/secretbinding.go @@ -7,6 +7,7 @@ package validation import ( corev1 "k8s.io/api/core/v1" apivalidation "k8s.io/apimachinery/pkg/api/validation" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" "github.com/gardener/gardener/pkg/apis/core" @@ -69,6 +70,16 @@ func validateSecretReferenceOptionalNamespace(ref corev1.SecretReference, fldPat if len(ref.Name) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("name"), "must provide a name")) + } else { + for _, err := range validation.IsDNS1123Subdomain(ref.Name) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), ref.Name, err)) + } + } + + if len(ref.Namespace) > 0 { + for _, err := range apivalidation.ValidateNamespaceName(ref.Namespace, false) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("namespace"), ref.Namespace, err)) + } } return allErrs diff --git a/pkg/apis/core/validation/secretbinding_test.go b/pkg/apis/core/validation/secretbinding_test.go index 80887c43a21..b1873396f98 100644 --- a/pkg/apis/core/validation/secretbinding_test.go +++ b/pkg/apis/core/validation/secretbinding_test.go @@ -76,8 +76,9 @@ var _ = Describe("SecretBinding Validation Tests", func() { Entry("should forbid SecretBinding with '_' in the name (not a DNS-1123 subdomain)", metav1.ObjectMeta{Name: "binding_test", Namespace: "garden"}, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), - "Field": Equal("metadata.name"), + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("metadata.name"), + "BadValue": Equal("binding_test"), }))), ), ) @@ -104,6 +105,32 @@ var _ = Describe("SecretBinding Validation Tests", func() { )) }) + It("should forbid a SecretBinding with invalid secretRef name and namespace", func() { + secretBinding.ObjectMeta = metav1.ObjectMeta{ + Name: "name", + Namespace: "namespace", + } + secretBinding.SecretRef = corev1.SecretReference{ + Name: "name-non-valid-@", + Namespace: "-namespace-non-valid123", + } + secretBinding.Provider = &core.SecretBindingProvider{} + + errorList := ValidateSecretBinding(secretBinding) + Expect(errorList).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("secretRef.name"), + "BadValue": Equal("name-non-valid-@"), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("secretRef.namespace"), + "BadValue": Equal("-namespace-non-valid123"), + })), + )) + }) + It("should forbid empty stated Quota names", func() { secretBinding.Quotas = []corev1.ObjectReference{ {}, diff --git a/pkg/apis/core/validation/utils.go b/pkg/apis/core/validation/utils.go index 2adeccc76d8..dc1c35e4c56 100644 --- a/pkg/apis/core/validation/utils.go +++ b/pkg/apis/core/validation/utils.go @@ -599,10 +599,19 @@ func ValidateObjectReferenceNameAndNamespace(ref corev1.ObjectReference, fldPath if len(ref.Name) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("name"), "must provide a name")) + } else { + for _, err := range validation.IsDNS1123Subdomain(ref.Name) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), ref.Name, err)) + } } if requireNamespace && len(ref.Namespace) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("namespace"), "must provide a namespace")) } + if len(ref.Namespace) > 0 { + for _, err := range apivalidation.ValidateNamespaceName(ref.Namespace, false) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("namespace"), ref.Namespace, err)) + } + } return allErrs } diff --git a/pkg/apis/core/validation/utils_test.go b/pkg/apis/core/validation/utils_test.go index a0d745cb7cb..6b10a6a8b72 100644 --- a/pkg/apis/core/validation/utils_test.go +++ b/pkg/apis/core/validation/utils_test.go @@ -160,6 +160,28 @@ var _ = Describe("Utils tests", func() { })), )) }) + + It("should deny a non-DNS1123 name", func() { + ref := corev1.ObjectReference{Name: "-name-"} + Expect(ValidateObjectReferenceNameAndNamespace(ref, fldPath, false)).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal(fldPath.Child("name").String()), + "BadValue": Equal("-name-"), + })), + )) + }) + + It("should deny an invalid namespace", func() { + ref := corev1.ObjectReference{Name: "name", Namespace: "namespace-123-@"} + Expect(ValidateObjectReferenceNameAndNamespace(ref, fldPath, true)).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal(fldPath.Child("namespace").String()), + "BadValue": Equal("namespace-123-@"), + })), + )) + }) }) DescribeTable("#ValidateCredentialsRef", @@ -180,8 +202,9 @@ var _ = Describe("Utils tests", func() { corev1.ObjectReference{APIVersion: "v1", Kind: "Secret", Name: "Foo", Namespace: "bar"}, ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), - "Field": Equal("credentialsRef.name"), + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("credentialsRef.name"), + "BadValue": Equal("Foo"), })), ), ), @@ -189,8 +212,9 @@ var _ = Describe("Utils tests", func() { corev1.ObjectReference{APIVersion: "security.gardener.cloud/v1alpha1", Kind: "WorkloadIdentity", Name: "foo", Namespace: "bar?"}, ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), - "Field": Equal("credentialsRef.namespace"), + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("credentialsRef.namespace"), + "BadValue": Equal("bar?"), })), ), ), @@ -253,9 +277,10 @@ var _ = Describe("Utils tests", func() { Expect(validationResult). To(ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), - "Field": Equal("spec.machineImages[0].name"), - "Detail": ContainSubstring("machine image name must be a qualified name"), + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.machineImages[0].name"), + "BadValue": Equal(name), + "Detail": ContainSubstring("machine image name must be a qualified name"), })), )) } else { @@ -304,9 +329,10 @@ var _ = Describe("Utils tests", func() { Expect(validationResult). To(ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), - "Field": Equal("spec.machineTypes[0].name"), - "Detail": ContainSubstring("machine type name must be a qualified name"), + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.machineTypes[0].name"), + "BadValue": Equal(name), + "Detail": ContainSubstring("machine type name must be a qualified name"), })), )) } else { @@ -330,9 +356,10 @@ var _ = Describe("Utils tests", func() { Expect(validationResult). To(ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeInvalid), - "Field": Equal("spec.volumeTypes[0].name"), - "Detail": ContainSubstring("volume type name must be a qualified name"), + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.volumeTypes[0].name"), + "BadValue": Equal(name), + "Detail": ContainSubstring("volume type name must be a qualified name"), })), )) } else { From 341416d59fc06977fcf0f69ea326b33bd114286b Mon Sep 17 00:00:00 2001 From: Sebastian Stauch Date: Tue, 4 Nov 2025 10:18:11 +0100 Subject: [PATCH 024/176] If overlay gets disabled delay network deployment until the control plane is fully operational (#13332) * when the overlay gets disabled wait with network deployment until controlplane is ready * don't suppress error --- .../shoot/shoot/reconciler_reconcile.go | 71 ++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go index 5e0afacff4c..97eac950f3d 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go @@ -6,11 +6,14 @@ package shoot import ( "context" + "encoding/json" "fmt" + "strconv" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" @@ -155,6 +158,11 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope return v1beta1helper.NewWrappedLastErrors(v1beta1helper.FormatLastErrDescription(err), err) } + isOverlayDisablementInProgress, err := isOverlayDisablementInProgress(ctx, o.Shoot.GetInfo(), botanist.SeedClientSet.Client(), botanist.Shoot.ControlPlaneNamespace) + if err != nil { + return v1beta1helper.NewWrappedLastErrors(v1beta1helper.FormatLastErrDescription(err), err) + } + var ( g = flow.NewGraph(fmt.Sprintf("Shoot cluster %s", utils.IifString(isRestoring, "restoration", "reconciliation"))) @@ -609,7 +617,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope Name: "Deploying shoot network plugin", Fn: flow.TaskFn(botanist.DeployNetwork).RetryUntilTimeout(defaultInterval, defaultTimeout), SkipIf: o.Shoot.IsWorkerless, - Dependencies: flow.NewTaskIDs(deployReferencedResources, waitUntilGardenerResourceManagerReady, waitUntilOperatingSystemConfigReady, deployKubeScheduler, waitUntilShootNamespacesReady), + Dependencies: flow.NewTaskIDs(deployReferencedResources, waitUntilGardenerResourceManagerReady, waitUntilOperatingSystemConfigReady, deployKubeScheduler, waitUntilShootNamespacesReady).InsertIf(isOverlayDisablementInProgress, waitUntilControlPlaneReady), }) waitUntilNetworkIsReady = g.Add(flow.Task{ Name: "Waiting until shoot network plugin has been reconciled", @@ -1082,3 +1090,64 @@ func shootHasPendingInPlaceUpdateWorkers(shoot *gardencorev1beta1.Shoot) bool { return shoot.Status.InPlaceUpdates != nil && shoot.Status.InPlaceUpdates.PendingWorkerUpdates != nil && (len(shoot.Status.InPlaceUpdates.PendingWorkerUpdates.AutoInPlaceUpdate) > 0 || len(shoot.Status.InPlaceUpdates.PendingWorkerUpdates.ManualInPlaceUpdate) > 0) } + +func isOverlayDisablementInProgress(ctx context.Context, shoot *gardencorev1beta1.Shoot, seedClient client.Client, shootControlPlaneNamespace string) (bool, error) { + shootOverlayEnabled, err := getOverlayEnabledFromShootSpec(shoot) + if err != nil { + return false, fmt.Errorf("failed to get overlay enabled status from shoot spec: %w", err) + } + + networkObj := &extensionsv1alpha1.Network{} + if err = seedClient.Get(ctx, client.ObjectKey{Name: shoot.Name, Namespace: shootControlPlaneNamespace}, networkObj); err != nil { + // If network doesn't exist yet, no switch in progress + if apierrors.IsNotFound(err) { + return false, nil + } + + return false, fmt.Errorf("failed to get network object: %w", err) + } + + networkOverlayEnabled, err := getOverlayEnabledFromProviderConfig(networkObj.Spec.ProviderConfig) + if err != nil { + return false, fmt.Errorf("failed to get overlay enabled status from provider config: %w", err) + } + + // If settings differ, overlay switch is in progress - need to wait for control plane + return !shootOverlayEnabled && networkOverlayEnabled, nil +} + +func getOverlayEnabledFromShootSpec(shoot *gardencorev1beta1.Shoot) (bool, error) { + if shoot.Spec.Networking == nil || shoot.Spec.Networking.ProviderConfig == nil { + return true, nil + } + + return getOverlayEnabledFromProviderConfig(shoot.Spec.Networking.ProviderConfig) +} + +func getOverlayEnabledFromProviderConfig(providerConfig *runtime.RawExtension) (bool, error) { + if providerConfig == nil { + return true, nil + } + + var config map[string]interface{} + if err := json.Unmarshal(providerConfig.Raw, &config); err != nil { + return false, fmt.Errorf("failed to unmarshal provider config: %w", err) + } + + if overlay, exists := config["overlay"]; exists { + if overlayMap, ok := overlay.(map[string]interface{}); ok { + if enabled, exists := overlayMap["enabled"]; exists { + if enabledStr, ok := enabled.(string); ok { + enabledBool, err := strconv.ParseBool(enabledStr) + if err != nil { + return false, fmt.Errorf("failed to get overlay enabled status from provider config: %w", err) + } + + return enabledBool, nil + } + } + } + } + + return true, nil +} From 7efe7d8fc9a12ed13608c4f1a035029cb97fdc7c Mon Sep 17 00:00:00 2001 From: Marin Atanasov Nikolov Date: Tue, 4 Nov 2025 12:37:11 +0200 Subject: [PATCH 025/176] docs: Add link to GEP-34 and fix file name (#13336) * docs: typo fix in GEP file name * docs: add link to GEP-34 --- docs/README.md | 1 + ...34-observability2.0-opentelemetry-operator-and-collectors.md} | 0 2 files changed, 1 insertion(+) rename docs/proposals/{34-observability2.0-opentelemtry-operator-and-collectors.md => 34-observability2.0-opentelemetry-operator-and-collectors.md} (100%) diff --git a/docs/README.md b/docs/README.md index f1e13e09bc3..ce088f725dd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -159,6 +159,7 @@ * [GEP-31: In-Place Node Updates of Shoot Clusters](proposals/31-inplace-node-update.md) * [GEP-32: Cloud Profile Version Classification Lifecycles](proposals/32-version-classification-lifecycles.md) * [GEP-33: Machine Image Capabilities](proposals/33-machine-image-capabilities.md) +* [GEP-34: Introducing OpenTelemetry Operator and Collectors in Shoot Control Planes](proposals/34-observability2.0-opentelemetry-operator-and-collectors.md) ## Development diff --git a/docs/proposals/34-observability2.0-opentelemtry-operator-and-collectors.md b/docs/proposals/34-observability2.0-opentelemetry-operator-and-collectors.md similarity index 100% rename from docs/proposals/34-observability2.0-opentelemtry-operator-and-collectors.md rename to docs/proposals/34-observability2.0-opentelemetry-operator-and-collectors.md From 616d40532042b9d1083eec1bd830d2a9824ea2b0 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Tue, 4 Nov 2025 13:19:07 +0100 Subject: [PATCH 026/176] Update registry.k8s.io/ingress-nginx/controller-chroot Docker tag to v1.14.0 (#13319) * Update registry.k8s.io/ingress-nginx/controller-chroot Docker tag to v1.14.0 * chore: Add targetVersion constraints for nginx-ingress-controller image According to: https://github.com/kubernetes/ingress-nginx#supported-versions-table --------- Co-authored-by: Marc Vornetran --- imagevector/containers.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 2de27c9cfdd..c1fccbb350f 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -83,10 +83,25 @@ images: sourceRepository: github.com/gardener/dependency-watchdog repository: europe-docker.pkg.dev/gardener-project/releases/gardener/dependency-watchdog tag: "v1.6.0" + - name: nginx-ingress-controller + sourceRepository: github.com/kubernetes/ingress-nginx + repository: registry.k8s.io/ingress-nginx/controller-chroot + tag: "v1.14.0" + targetVersion: ">= 1.30" + labels: + - name: 'gardener.cloud/cve-categorisation' + value: + network_exposure: 'public' + authentication_enforced: true + user_interaction: 'end-user' + confidentiality_requirement: 'high' + integrity_requirement: 'high' + availability_requirement: 'low' - name: nginx-ingress-controller sourceRepository: github.com/kubernetes/ingress-nginx repository: registry.k8s.io/ingress-nginx/controller-chroot tag: "v1.13.4" + targetVersion: "1.29.x" labels: - name: 'gardener.cloud/cve-categorisation' value: From f0408765fc72c469725d53ad4d6fb1b27724a5ea Mon Sep 17 00:00:00 2001 From: Johannes Scheerer Date: Tue, 4 Nov 2025 13:19:16 +0100 Subject: [PATCH 027/176] Fix timing issue in bootstrap token test (#13337) --- pkg/utils/kubernetes/bootstraptoken/bootstraptoken_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/utils/kubernetes/bootstraptoken/bootstraptoken_test.go b/pkg/utils/kubernetes/bootstraptoken/bootstraptoken_test.go index fee62ea0e82..62b140a8822 100644 --- a/pkg/utils/kubernetes/bootstraptoken/bootstraptoken_test.go +++ b/pkg/utils/kubernetes/bootstraptoken/bootstraptoken_test.go @@ -16,6 +16,7 @@ import ( fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" . "github.com/gardener/gardener/pkg/utils/kubernetes/bootstraptoken" + "github.com/gardener/gardener/pkg/utils/test" ) var _ = Describe("bootstraptoken", func() { @@ -30,6 +31,7 @@ var _ = Describe("bootstraptoken", func() { BeforeEach(func() { fakeClient = fakeclient.NewClientBuilder().Build() + DeferCleanup(test.WithVar(&Now, func() metav1.Time { return metav1.NewTime(time.Date(2050, 5, 5, 5, 5, 5, 0, time.UTC)) })) }) Describe("#ComputeBootstrapToken", func() { @@ -44,7 +46,7 @@ var _ = Describe("bootstraptoken", func() { HaveKeyWithValue("token-id", Equal([]byte(tokenID))), HaveKeyWithValue("token-secret", HaveLen(16)), HaveKeyWithValue("description", Equal([]byte(description))), - HaveKeyWithValue("expiration", Equal([]byte(metav1.Now().Add(validity).Format(time.RFC3339)))), + HaveKeyWithValue("expiration", Equal([]byte(Now().Add(validity).Format(time.RFC3339)))), HaveKeyWithValue("usage-bootstrap-authentication", Equal([]byte("true"))), HaveKeyWithValue("usage-bootstrap-signing", Equal([]byte("true"))), )) @@ -75,7 +77,7 @@ var _ = Describe("bootstraptoken", func() { HaveKeyWithValue("token-id", Equal([]byte(tokenID))), HaveKeyWithValue("token-secret", Equal([]byte(tokenSecret))), HaveKeyWithValue("description", Equal([]byte(description))), - HaveKeyWithValue("expiration", Equal([]byte(metav1.Now().Add(validity).Format(time.RFC3339)))), + HaveKeyWithValue("expiration", Equal([]byte(Now().Add(validity).Format(time.RFC3339)))), HaveKeyWithValue("usage-bootstrap-authentication", Equal([]byte("true"))), HaveKeyWithValue("usage-bootstrap-signing", Equal([]byte("true"))), )) From e8741ee1574c38b220deba0cd1d20021b68b081c Mon Sep 17 00:00:00 2001 From: Ashwani Kumar Date: Tue, 4 Nov 2025 20:11:06 +0530 Subject: [PATCH 028/176] Remove ambiguity in CP health reporting when DWD prober scales down certain deployments (#13338) * Remove ambiguity in CP health reporting when DWD prober scales down * Addressed review comments --- pkg/gardenlet/controller/shoot/care/health.go | 4 ++-- pkg/gardenlet/controller/shoot/care/health_test.go | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/gardenlet/controller/shoot/care/health.go b/pkg/gardenlet/controller/shoot/care/health.go index 42fd14a571e..48583f486d8 100644 --- a/pkg/gardenlet/controller/shoot/care/health.go +++ b/pkg/gardenlet/controller/shoot/care/health.go @@ -418,7 +418,7 @@ func (h *Health) checkControlPlane( if scaledDownDeploymentNames, err := CheckIfDependencyWatchdogProberScaledDownControllers(ctx, h.seedClient.Client(), h.shoot.ControlPlaneNamespace); err != nil { return ptr.To(v1beta1helper.FailedCondition(h.clock, h.shoot.GetInfo().Status.LastOperation, h.conditionThresholds, condition, "ControllersScaledDownCheckError", err.Error())), nil } else if len(scaledDownDeploymentNames) > 0 { - return ptr.To(v1beta1helper.FailedCondition(h.clock, h.shoot.GetInfo().Status.LastOperation, h.conditionThresholds, condition, "ControllersScaledDown", fmt.Sprintf("The following deployments have been scaled down to 0 replicas (perhaps by dependency-watchdog-prober): %s", strings.Join(scaledDownDeploymentNames, ", ")))), nil + return ptr.To(v1beta1helper.FailedCondition(h.clock, h.shoot.GetInfo().Status.LastOperation, h.conditionThresholds, condition, "ControllersScaledDown", fmt.Sprintf("The following deployments have been scaled down to 0 replicas by dependency-watchdog-prober: %s", strings.Join(scaledDownDeploymentNames, ", ")))), nil } } @@ -457,7 +457,7 @@ func CheckIfDependencyWatchdogProberScaledDownControllers(ctx context.Context, s return nil, fmt.Errorf("failed reading Deployment %s for scale-down check: %w", deployment.Name, err) } - if ptr.Deref(deployment.Spec.Replicas, 0) == 0 { + if _, ok := deployment.Annotations["dependency-watchdog.gardener.cloud/meltdown-protection-active"]; ok { scaledDownDeploymentNames = append(scaledDownDeploymentNames, deployment.Name) } } diff --git a/pkg/gardenlet/controller/shoot/care/health_test.go b/pkg/gardenlet/controller/shoot/care/health_test.go index a91e6cb236b..587466bf4aa 100644 --- a/pkg/gardenlet/controller/shoot/care/health_test.go +++ b/pkg/gardenlet/controller/shoot/care/health_test.go @@ -869,17 +869,23 @@ var _ = Describe("health check", func() { Expect(scaledDownDeploymentNames).To(BeEmpty()) }) - It("should report names because some relevant deployment have replicas == 0", func() { + It("should report names because some relevant deployment have replicas == 0 and meltdown annotation is set", func() { deploymentKCM.Spec.Replicas = nil deploymentMCM.Spec.Replicas = ptr.To[int32](0) + // Set meltdown annotation only on deploymentMCM + if deploymentMCM.Annotations == nil { + deploymentMCM.Annotations = map[string]string{} + } + deploymentMCM.Annotations["dependency-watchdog.gardener.cloud/meltdown-protection-active"] = "" + Expect(fakeClient.Create(ctx, deploymentCA)).To(Succeed()) Expect(fakeClient.Create(ctx, deploymentKCM)).To(Succeed()) Expect(fakeClient.Create(ctx, deploymentMCM)).To(Succeed()) scaledDownDeploymentNames, err := CheckIfDependencyWatchdogProberScaledDownControllers(ctx, fakeClient, controlPlaneNamespace) Expect(err).NotTo(HaveOccurred()) - Expect(scaledDownDeploymentNames).To(HaveExactElements(deploymentKCM.Name, deploymentMCM.Name)) + Expect(scaledDownDeploymentNames).To(HaveExactElements(deploymentMCM.Name)) }) }) From f34fc0bfb2d272d02d2afbb0dc24d25c74a2a33f Mon Sep 17 00:00:00 2001 From: Viktor Date: Tue, 4 Nov 2025 18:36:08 +0200 Subject: [PATCH 029/176] Revert "support server block imports for node-local-dns (#13160)" (#13344) This reverts commit 0acc41c6ef96a5e93976651df430e9bce3b9d870. --- docs/usage/networking/custom-dns-config.md | 6 +- docs/usage/networking/node-local-dns.md | 2 +- imagevector/containers.go | 2 - imagevector/containers.yaml | 17 ----- pkg/component/networking/coredns/coredns.go | 5 +- .../networking/nodelocaldns/nodelocaldns.go | 19 +++--- .../nodelocaldns/nodelocaldns_test.go | 53 +--------------- .../networking/nodelocaldns/resources.go | 63 ++----------------- .../operation/botanist/nodelocaldns.go | 20 +++--- .../controller/dnsrecord/actuator.go | 3 +- .../controller/dnsrecord/actuator_test.go | 5 +- skaffold-gardenadm.yaml | 2 - skaffold-operator.yaml | 2 - skaffold.yaml | 2 - 14 files changed, 28 insertions(+), 173 deletions(-) diff --git a/docs/usage/networking/custom-dns-config.md b/docs/usage/networking/custom-dns-config.md index 61cf672ad9d..706f162f69c 100644 --- a/docs/usage/networking/custom-dns-config.md +++ b/docs/usage/networking/custom-dns-config.md @@ -102,9 +102,9 @@ This should bring the cluster DNS back to functioning state. ## Node Local DNS -Starting with Gardener v1.131, custom DNS configurations are fully supported in NodeLocalDNS. In this version, the `coredns-custom` `ConfigMap` is mounted into the NodeLocalDNS pod, allowing custom override and server configurations to be imported into the DNS server. The server configuration is read by a sidecar container and a new configuration file with the correct bind statement and port mappings is generated and imported into NodeLocalDNS pods. Prior to Gardener v1.131, custom DNS configurations might not function as expected with NodeLocalDNS. -With NodeLocalDNS, ordinary DNS queries targeting upstream DNS servers (i.e., non-Kubernetes domains) are sent directly to the upstream DNS server, bypassing CoreDNS. Therefore, configurations for non-Kubernetes entities, such as the `istio.server` block in the [custom DNS configuration](custom-dns-config.md) example, may not have any effect when NodeLocalDNS is enabled on landscapes with Gardener prior to v1.131. -If you require custom DNS configurations for non-Kubernetes domains, you need to disable forwarding to upstream DNS with Gardener v1.130 and below. This can be done by setting the `disableForwardToUpstreamDNS` option in the Shoot resource to `true`: +Starting with Gardener v1.128, custom DNS configurations are fully supported in NodeLocalDNS. In this version, the `coredns-custom` `ConfigMap` is mounted into the NodeLocalDNS pod, allowing custom override and server configurations to be imported into the DNS server. Prior to Gardener v1.128, custom DNS configurations might not function as expected with NodeLocalDNS. +With NodeLocalDNS, ordinary DNS queries targeting upstream DNS servers (i.e., non-Kubernetes domains) are sent directly to the upstream DNS server, bypassing CoreDNS. Therefore, configurations for non-Kubernetes entities, such as the `istio.server` block in the [custom DNS configuration](custom-dns-config.md) example, may not have any effect when NodeLocalDNS is enabled on landscapes with Gardener prior to v1.128. +If you require custom DNS configurations for non-Kubernetes domains, you need to disable forwarding to upstream DNS with Gardener v1.127 and below. This can be done by setting the `disableForwardToUpstreamDNS` option in the Shoot resource to `true`: ```yaml ... spec: diff --git a/docs/usage/networking/node-local-dns.md b/docs/usage/networking/node-local-dns.md index 4585a04d729..ca95fa01af6 100644 --- a/docs/usage/networking/node-local-dns.md +++ b/docs/usage/networking/node-local-dns.md @@ -48,5 +48,5 @@ For more information about `node-local-dns`, please refer to the [KEP](https://g ## Known Issues -Custom DNS configuration may not work as expected in conjunction with `NodeLocalDNS` prior to gardener v1.131. +Custom DNS configuration may not work as expected in conjunction with `NodeLocalDNS` prior to gardener v1.128. Please refer to [Custom DNS Configuration](custom-dns-config.md#node-local-dns). diff --git a/imagevector/containers.go b/imagevector/containers.go index 0ea93ce1eab..c655a64e28e 100644 --- a/imagevector/containers.go +++ b/imagevector/containers.go @@ -25,8 +25,6 @@ const ( ContainerImageNameConfigmapReloader = "configmap-reloader" // ContainerImageNameCoredns is a constant for an image in the image vector with name 'coredns'. ContainerImageNameCoredns = "coredns" - // ContainerImageNameCorednsConfigAdapter is a constant for an image in the image vector with name 'coredns-config-adapter'. - ContainerImageNameCorednsConfigAdapter = "coredns-config-adapter" // ContainerImageNameCortex is a constant for an image in the image vector with name 'cortex'. ContainerImageNameCortex = "cortex" // ContainerImageNameDependencyWatchdog is a constant for an image in the image vector with name 'dependency-watchdog'. diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index c1fccbb350f..585af4b3f6a 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -494,23 +494,6 @@ images: value: - type: 'githubTeam' teamname: 'gardener/gardener-core-networking-maintainers' - - name: coredns-config-adapter - sourceRepository: github.com/gardener/coredns-config-adapter - repository: europe-docker.pkg.dev/gardener-project/releases/gardener/coredns-config-adapter - tag: "v0.3.0" - labels: - - name: 'gardener.cloud/cve-categorisation' - value: - network_exposure: 'private' - authentication_enforced: false - user_interaction: 'end-user' - confidentiality_requirement: 'low' - integrity_requirement: 'high' - availability_requirement: 'high' - - name: 'cloud.gardener.cnudie/responsibles' - value: - - type: 'githubTeam' - teamname: 'gardener/gardener-core-networking-maintainers' - name: node-problem-detector sourceRepository: github.com/kubernetes/node-problem-detector repository: registry.k8s.io/node-problem-detector/node-problem-detector diff --git a/pkg/component/networking/coredns/coredns.go b/pkg/component/networking/coredns/coredns.go index e5db3bcaa4a..68175e44986 100644 --- a/pkg/component/networking/coredns/coredns.go +++ b/pkg/component/networking/coredns/coredns.go @@ -55,9 +55,6 @@ const ( containerName = "coredns" serviceName = "kube-dns" // this is due to legacy reasons - // CustomConfigMapName is the name of the custom CoreDNS ConfigMap. - CustomConfigMapName = "coredns-custom" - portNameMetrics = "metrics" portMetrics = 9153 @@ -322,7 +319,7 @@ import custom/*.server configMapCustom = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: CustomConfigMapName, + Name: "coredns-custom", Namespace: metav1.NamespaceSystem, Annotations: map[string]string{resourcesv1alpha1.Ignore: "true"}, }, diff --git a/pkg/component/networking/nodelocaldns/nodelocaldns.go b/pkg/component/networking/nodelocaldns/nodelocaldns.go index 52891283d63..571c70e10b5 100644 --- a/pkg/component/networking/nodelocaldns/nodelocaldns.go +++ b/pkg/component/networking/nodelocaldns/nodelocaldns.go @@ -64,8 +64,6 @@ const ( metricsPortName = "metrics" errorMetricsPortName = "errormetrics" - sideCarName = "coredns-config-adapter" - domain = gardencorev1beta1.DefaultDomain serviceName = "kube-dns-upstream" livenessProbePort = 8099 @@ -76,14 +74,13 @@ const ( daemonSetPollInterval = 5 * time.Second - volumeMountNameCleanUp = "cleanup-script" - volumeMountPathCleanUp = "/scripts" - volumeMountNameXtablesLock = "xtables-lock" - volumeMountPathXtablesLock = "/run/xtables.lock" - volumeMountPathCustomConfig = "/etc/custom" - volumeMountNameCustomConfig = "custom-config-volume" - volumeMountNameGeneratedConfig = "generated-config" - volumeMountPathGeneratedConfig = "/etc/generated-config" + volumeMountNameCleanUp = "cleanup-script" + volumeMountPathCleanUp = "/scripts" + volumeMountNameXtablesLock = "xtables-lock" + volumeMountPathXtablesLock = "/run/xtables.lock" + volumeMountPathCustomConfig = "/etc/custom" + volumeMountNameCustomConfig = "custom-config-volume" + customConfigMapName = "coredns-custom" ) var ( @@ -106,8 +103,6 @@ type Values struct { Image string // AlpineImage is the container image used for the cleanup DaemonSet. AlpineImage string - // CorednsConfigAdapterImage is the container image used for the coredns config adapter sidecar. - CorednsConfigAdapterImage string // VPAEnabled marks whether VerticalPodAutoscaler is enabled for the shoot. VPAEnabled bool // Config is the node local configuration for the shoot spec diff --git a/pkg/component/networking/nodelocaldns/nodelocaldns_test.go b/pkg/component/networking/nodelocaldns/nodelocaldns_test.go index 2a0ff6eeed3..3518128667c 100644 --- a/pkg/component/networking/nodelocaldns/nodelocaldns_test.go +++ b/pkg/component/networking/nodelocaldns/nodelocaldns_test.go @@ -31,7 +31,6 @@ import ( resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component" - "github.com/gardener/gardener/pkg/component/networking/coredns" . "github.com/gardener/gardener/pkg/component/networking/nodelocaldns" "github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references" "github.com/gardener/gardener/pkg/utils" @@ -351,7 +350,6 @@ data: cache 30 reload } - import generated-config/custom-server-block.server immutable: true kind: ConfigMap metadata: @@ -453,41 +451,6 @@ status: Type: corev1.SeccompProfileTypeRuntimeDefault, }, }, - InitContainers: []corev1.Container{ - { - Name: "coredns-config-adapter", - Image: values.CorednsConfigAdapterImage, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("5m"), - corev1.ResourceMemory: resource.MustParse("10Mi"), - }, - }, - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: ptr.To(false), - RunAsNonRoot: ptr.To(true), - RunAsUser: ptr.To[int64](65532), - RunAsGroup: ptr.To[int64](65532), - }, - Args: []string{ - "-inputDir=/etc/custom", - "-outputDir=/etc/generated-config", - "-bind=bind " + bindIP(values), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "custom-config-volume", - MountPath: "/etc/custom", - ReadOnly: true, - }, - { - MountPath: "/etc/generated-config", - Name: "generated-config", - }, - }, - RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), - }, - }, Containers: []corev1.Container{ { Name: "node-cache", @@ -569,10 +532,6 @@ status: MountPath: "/etc/custom", ReadOnly: true, }, - { - MountPath: "/etc/generated-config", - Name: "generated-config", - }, }, }, }, @@ -618,19 +577,13 @@ status: VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: coredns.CustomConfigMapName, + Name: "coredns-custom", }, DefaultMode: ptr.To[int32](420), Optional: ptr.To(true), }, }, }, - { - Name: "generated-config", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, }, }, }, @@ -648,8 +601,6 @@ spec: containerPolicies: - containerName: '*' controlledValues: RequestsOnly - - containerName: coredns-config-adapter - mode: "Off" targetRef: apiVersion: apps/v1 kind: DaemonSet @@ -769,7 +720,6 @@ ip6.arpa:53 { cache 30 reload } -import generated-config/custom-server-block.server `, } configMapHash = utils.ComputeConfigMapChecksum(configMapData)[:8] @@ -1044,7 +994,6 @@ ip6.arpa:53 { cache 30 reload } -import generated-config/custom-server-block.server `, } configMapHash = utils.ComputeConfigMapChecksum(configMapData)[:8] diff --git a/pkg/component/networking/nodelocaldns/resources.go b/pkg/component/networking/nodelocaldns/resources.go index b184a9ef61b..78fdf7ecd4d 100644 --- a/pkg/component/networking/nodelocaldns/resources.go +++ b/pkg/component/networking/nodelocaldns/resources.go @@ -21,7 +21,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/gardener/gardener/pkg/component/networking/coredns" nodelocaldnsconstants "github.com/gardener/gardener/pkg/component/networking/nodelocaldns/constants" "github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" @@ -101,7 +100,6 @@ ip6.arpa:53 { cache 30 reload } -import generated-config/custom-server-block.server `, }, } @@ -210,41 +208,6 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc v1beta1constants.LabelNodeLocalDNS: "true", v1beta1constants.LabelWorkerPool: worker.Name, }, - InitContainers: []corev1.Container{ - { - Name: sideCarName, - Image: n.values.CorednsConfigAdapterImage, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("5m"), - corev1.ResourceMemory: resource.MustParse("10Mi"), - }, - }, - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: ptr.To(false), - RunAsNonRoot: ptr.To(true), - RunAsUser: ptr.To[int64](65532), - RunAsGroup: ptr.To[int64](65532), - }, - Args: []string{ - "-inputDir=" + volumeMountPathCustomConfig, - "-outputDir=" + volumeMountPathGeneratedConfig, - "-bind=bind " + n.bindIP(), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: volumeMountNameCustomConfig, - MountPath: volumeMountPathCustomConfig, - ReadOnly: true, - }, - { - MountPath: volumeMountPathGeneratedConfig, - Name: volumeMountNameGeneratedConfig, - }, - }, - RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), - }, - }, Containers: []corev1.Container{ { Name: containerName, @@ -326,10 +289,6 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc MountPath: volumeMountPathCustomConfig, ReadOnly: true, }, - { - MountPath: volumeMountPathGeneratedConfig, - Name: volumeMountNameGeneratedConfig, - }, }, }, }, @@ -375,19 +334,13 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: coredns.CustomConfigMapName, + Name: customConfigMapName, }, DefaultMode: ptr.To[int32](420), Optional: ptr.To(true), }, }, }, - { - Name: volumeMountNameGeneratedConfig, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, }, }, }, @@ -413,16 +366,10 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc UpdateMode: &vpaUpdateMode, }, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ - ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ - { - ContainerName: vpaautoscalingv1.DefaultContainerResourcePolicy, - ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), - }, - { - ContainerName: sideCarName, - Mode: ptr.To(vpaautoscalingv1.ContainerScalingModeOff), - }, - }, + ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{{ + ContainerName: vpaautoscalingv1.DefaultContainerResourcePolicy, + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + }}, }, }, } diff --git a/pkg/gardenlet/operation/botanist/nodelocaldns.go b/pkg/gardenlet/operation/botanist/nodelocaldns.go index 4691d8715e3..f1ab2b75b73 100644 --- a/pkg/gardenlet/operation/botanist/nodelocaldns.go +++ b/pkg/gardenlet/operation/botanist/nodelocaldns.go @@ -35,23 +35,17 @@ func (b *Botanist) DefaultNodeLocalDNS() (nodelocaldns.Interface, error) { return nil, err } - imageCorednsConfigAdapter, err := imagevector.Containers().FindImage(imagevector.ContainerImageNameCorednsConfigAdapter) - if err != nil { - return nil, err - } - return nodelocaldns.New( b.SeedClientSet.Client(), b.Shoot.ControlPlaneNamespace, nodelocaldns.Values{ - Image: image.String(), - AlpineImage: imageAlpine.String(), - CorednsConfigAdapterImage: imageCorednsConfigAdapter.String(), - VPAEnabled: b.Shoot.WantsVerticalPodAutoscaler, - Config: v1beta1helper.GetNodeLocalDNS(b.Shoot.GetInfo().Spec.SystemComponents), - Workers: b.Shoot.GetInfo().Spec.Provider.Workers, - KubeProxyConfig: b.Shoot.GetInfo().Spec.Kubernetes.KubeProxy, - Log: b.Logger, + Image: image.String(), + AlpineImage: imageAlpine.String(), + VPAEnabled: b.Shoot.WantsVerticalPodAutoscaler, + Config: v1beta1helper.GetNodeLocalDNS(b.Shoot.GetInfo().Spec.SystemComponents), + Workers: b.Shoot.GetInfo().Spec.Provider.Workers, + KubeProxyConfig: b.Shoot.GetInfo().Spec.Kubernetes.KubeProxy, + Log: b.Logger, }, ), nil } diff --git a/pkg/provider-local/controller/dnsrecord/actuator.go b/pkg/provider-local/controller/dnsrecord/actuator.go index 905d40f3207..a12611e09d7 100644 --- a/pkg/provider-local/controller/dnsrecord/actuator.go +++ b/pkg/provider-local/controller/dnsrecord/actuator.go @@ -21,7 +21,6 @@ import ( v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" - "github.com/gardener/gardener/pkg/component/networking/coredns" ) // Actuator implements the DNSRecord actuator for the local DNS provider. @@ -86,7 +85,7 @@ func (a *Actuator) Restore(ctx context.Context, log logr.Logger, dnsRecord *exte } func (a *Actuator) patchCoreDNSConfigMap(ctx context.Context, mutate func(configMap *corev1.ConfigMap)) error { - configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: coredns.CustomConfigMapName, Namespace: "gardener-extension-provider-local-coredns"}} + configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "coredns-custom", Namespace: "gardener-extension-provider-local-coredns"}} _, err := controllerutil.CreateOrPatch(ctx, a.Client, configMap, func() error { mutate(configMap) return nil diff --git a/pkg/provider-local/controller/dnsrecord/actuator_test.go b/pkg/provider-local/controller/dnsrecord/actuator_test.go index fb6edac999f..fe3d9303184 100644 --- a/pkg/provider-local/controller/dnsrecord/actuator_test.go +++ b/pkg/provider-local/controller/dnsrecord/actuator_test.go @@ -22,7 +22,6 @@ import ( "github.com/gardener/gardener/extensions/pkg/controller/dnsrecord" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" - "github.com/gardener/gardener/pkg/component/networking/coredns" "github.com/gardener/gardener/pkg/logger" . "github.com/gardener/gardener/pkg/provider-local/controller/dnsrecord" ) @@ -104,14 +103,14 @@ var _ = Describe("Actuator", func() { } emptyConfigMap = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: coredns.CustomConfigMapName, + Name: "coredns-custom", Namespace: extensionNamespace.Name, }, Data: map[string]string{"test": "data"}, } configMapWithRule = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: coredns.CustomConfigMapName, + Name: "coredns-custom", Namespace: extensionNamespace.Name, }, Data: map[string]string{ diff --git a/skaffold-gardenadm.yaml b/skaffold-gardenadm.yaml index 63fecc64b89..fff45bdb73d 100644 --- a/skaffold-gardenadm.yaml +++ b/skaffold-gardenadm.yaml @@ -1132,8 +1132,6 @@ build: - pkg/component/kubernetes/proxy - pkg/component/kubernetes/proxy/resources/cleanup.sh - pkg/component/kubernetes/proxy/resources/conntrack-fix.sh - - pkg/component/networking/coredns - - pkg/component/networking/coredns/constants - pkg/component/nodemanagement/machinecontrollermanager - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml index 234857b7643..7f5dfe7f84f 100644 --- a/skaffold-operator.yaml +++ b/skaffold-operator.yaml @@ -1146,8 +1146,6 @@ build: - pkg/component/kubernetes/proxy - pkg/component/kubernetes/proxy/resources/cleanup.sh - pkg/component/kubernetes/proxy/resources/conntrack-fix.sh - - pkg/component/networking/coredns - - pkg/component/networking/coredns/constants - pkg/component/nodemanagement/machinecontrollermanager - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml diff --git a/skaffold.yaml b/skaffold.yaml index 1d8c51412d5..7bde9200a08 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -740,8 +740,6 @@ build: - pkg/component/kubernetes/proxy - pkg/component/kubernetes/proxy/resources/cleanup.sh - pkg/component/kubernetes/proxy/resources/conntrack-fix.sh - - pkg/component/networking/coredns - - pkg/component/networking/coredns/constants - pkg/component/nodemanagement/machinecontrollermanager - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml From 27ae697aa03e1da3bde7041aecbe8c332e948e7f Mon Sep 17 00:00:00 2001 From: Sebastian Stauch Date: Tue, 4 Nov 2025 17:36:18 +0100 Subject: [PATCH 030/176] Revert "If overlay gets disabled delay network deployment until the control plane is fully operational (#13332)" (#13346) This reverts commit 341416d59fc06977fcf0f69ea326b33bd114286b. --- .../shoot/shoot/reconciler_reconcile.go | 71 +------------------ 1 file changed, 1 insertion(+), 70 deletions(-) diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go index 97eac950f3d..5e0afacff4c 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go @@ -6,14 +6,11 @@ package shoot import ( "context" - "encoding/json" "fmt" - "strconv" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" @@ -158,11 +155,6 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope return v1beta1helper.NewWrappedLastErrors(v1beta1helper.FormatLastErrDescription(err), err) } - isOverlayDisablementInProgress, err := isOverlayDisablementInProgress(ctx, o.Shoot.GetInfo(), botanist.SeedClientSet.Client(), botanist.Shoot.ControlPlaneNamespace) - if err != nil { - return v1beta1helper.NewWrappedLastErrors(v1beta1helper.FormatLastErrDescription(err), err) - } - var ( g = flow.NewGraph(fmt.Sprintf("Shoot cluster %s", utils.IifString(isRestoring, "restoration", "reconciliation"))) @@ -617,7 +609,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope Name: "Deploying shoot network plugin", Fn: flow.TaskFn(botanist.DeployNetwork).RetryUntilTimeout(defaultInterval, defaultTimeout), SkipIf: o.Shoot.IsWorkerless, - Dependencies: flow.NewTaskIDs(deployReferencedResources, waitUntilGardenerResourceManagerReady, waitUntilOperatingSystemConfigReady, deployKubeScheduler, waitUntilShootNamespacesReady).InsertIf(isOverlayDisablementInProgress, waitUntilControlPlaneReady), + Dependencies: flow.NewTaskIDs(deployReferencedResources, waitUntilGardenerResourceManagerReady, waitUntilOperatingSystemConfigReady, deployKubeScheduler, waitUntilShootNamespacesReady), }) waitUntilNetworkIsReady = g.Add(flow.Task{ Name: "Waiting until shoot network plugin has been reconciled", @@ -1090,64 +1082,3 @@ func shootHasPendingInPlaceUpdateWorkers(shoot *gardencorev1beta1.Shoot) bool { return shoot.Status.InPlaceUpdates != nil && shoot.Status.InPlaceUpdates.PendingWorkerUpdates != nil && (len(shoot.Status.InPlaceUpdates.PendingWorkerUpdates.AutoInPlaceUpdate) > 0 || len(shoot.Status.InPlaceUpdates.PendingWorkerUpdates.ManualInPlaceUpdate) > 0) } - -func isOverlayDisablementInProgress(ctx context.Context, shoot *gardencorev1beta1.Shoot, seedClient client.Client, shootControlPlaneNamespace string) (bool, error) { - shootOverlayEnabled, err := getOverlayEnabledFromShootSpec(shoot) - if err != nil { - return false, fmt.Errorf("failed to get overlay enabled status from shoot spec: %w", err) - } - - networkObj := &extensionsv1alpha1.Network{} - if err = seedClient.Get(ctx, client.ObjectKey{Name: shoot.Name, Namespace: shootControlPlaneNamespace}, networkObj); err != nil { - // If network doesn't exist yet, no switch in progress - if apierrors.IsNotFound(err) { - return false, nil - } - - return false, fmt.Errorf("failed to get network object: %w", err) - } - - networkOverlayEnabled, err := getOverlayEnabledFromProviderConfig(networkObj.Spec.ProviderConfig) - if err != nil { - return false, fmt.Errorf("failed to get overlay enabled status from provider config: %w", err) - } - - // If settings differ, overlay switch is in progress - need to wait for control plane - return !shootOverlayEnabled && networkOverlayEnabled, nil -} - -func getOverlayEnabledFromShootSpec(shoot *gardencorev1beta1.Shoot) (bool, error) { - if shoot.Spec.Networking == nil || shoot.Spec.Networking.ProviderConfig == nil { - return true, nil - } - - return getOverlayEnabledFromProviderConfig(shoot.Spec.Networking.ProviderConfig) -} - -func getOverlayEnabledFromProviderConfig(providerConfig *runtime.RawExtension) (bool, error) { - if providerConfig == nil { - return true, nil - } - - var config map[string]interface{} - if err := json.Unmarshal(providerConfig.Raw, &config); err != nil { - return false, fmt.Errorf("failed to unmarshal provider config: %w", err) - } - - if overlay, exists := config["overlay"]; exists { - if overlayMap, ok := overlay.(map[string]interface{}); ok { - if enabled, exists := overlayMap["enabled"]; exists { - if enabledStr, ok := enabled.(string); ok { - enabledBool, err := strconv.ParseBool(enabledStr) - if err != nil { - return false, fmt.Errorf("failed to get overlay enabled status from provider config: %w", err) - } - - return enabledBool, nil - } - } - } - } - - return true, nil -} From 56564deef7903feb9c927d46b62140abb4369655 Mon Sep 17 00:00:00 2001 From: Axel Siebenborn Date: Tue, 4 Nov 2025 22:09:06 +0100 Subject: [PATCH 031/176] Generate an ULA as services range for workerless IPv6 shoots. (#13224) --- plugin/pkg/shoot/mutator/admission.go | 66 ++++++++++++++++---- plugin/pkg/shoot/mutator/admission_test.go | 40 +++++++++++- plugin/pkg/shoot/validator/admission.go | 7 ++- plugin/pkg/shoot/validator/admission_test.go | 35 +++++++++++ 4 files changed, 133 insertions(+), 15 deletions(-) diff --git a/plugin/pkg/shoot/mutator/admission.go b/plugin/pkg/shoot/mutator/admission.go index db3e660ebe7..db7cd9a89df 100644 --- a/plugin/pkg/shoot/mutator/admission.go +++ b/plugin/pkg/shoot/mutator/admission.go @@ -6,6 +6,7 @@ package mutator import ( "context" + "crypto/rand" "errors" "fmt" "io" @@ -229,18 +230,44 @@ func addDeploymentTasks(shoot *core.Shoot, tasks ...string) { } func (c *mutationContext) defaultShootNetworks(workerless bool) { - if c.seed != nil { - if c.shoot.Spec.Networking.Pods == nil && !workerless && - c.seed.Spec.Networks.ShootDefaults != nil && c.seed.Spec.Networks.ShootDefaults.Pods != nil && - cidrMatchesIPFamily(*c.seed.Spec.Networks.ShootDefaults.Pods, c.shoot.Spec.Networking.IPFamilies) { - c.shoot.Spec.Networking.Pods = c.seed.Spec.Networks.ShootDefaults.Pods - } + if c.seed == nil { + return + } - if c.shoot.Spec.Networking.Services == nil && - c.seed.Spec.Networks.ShootDefaults != nil && c.seed.Spec.Networks.ShootDefaults.Services != nil && - cidrMatchesIPFamily(*c.seed.Spec.Networks.ShootDefaults.Services, c.shoot.Spec.Networking.IPFamilies) { - c.shoot.Spec.Networking.Services = c.seed.Spec.Networks.ShootDefaults.Services - } + c.defaultPodsNetwork(workerless) + c.defaultServicesNetwork(workerless) +} + +func (c *mutationContext) defaultPodsNetwork(workerless bool) { + if c.shoot.Spec.Networking.Pods != nil || workerless { + return + } + + defaults := c.seed.Spec.Networks.ShootDefaults + if defaults == nil || defaults.Pods == nil { + return + } + + if cidrMatchesIPFamily(*defaults.Pods, c.shoot.Spec.Networking.IPFamilies) { + c.shoot.Spec.Networking.Pods = defaults.Pods + } +} + +func (c *mutationContext) defaultServicesNetwork(workerless bool) { + if c.shoot.Spec.Networking.Services != nil { + return + } + + defaults := c.seed.Spec.Networks.ShootDefaults + if defaults != nil && defaults.Services != nil && + cidrMatchesIPFamily(*defaults.Services, c.shoot.Spec.Networking.IPFamilies) { + c.shoot.Spec.Networking.Services = defaults.Services + return + } + + if workerless && slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv6) { + ulaCIDR := generateULAServicesCIDR() + c.shoot.Spec.Networking.Services = &ulaCIDR } } @@ -248,3 +275,20 @@ func cidrMatchesIPFamily(cidr string, ipfamilies []core.IPFamily) bool { ip, _, _ := net.ParseCIDR(cidr) return ip != nil && (ip.To4() != nil && slices.Contains(ipfamilies, core.IPFamilyIPv4) || ip.To4() == nil && slices.Contains(ipfamilies, core.IPFamilyIPv6)) } + +// generateULAServicesCIDR generates a /112 ULA (Unique Local Address) CIDR for IPv6 services. +func generateULAServicesCIDR() string { + // Generate a random 40-bit Global ID (5 bytes) for the ULA + // ULA format: fd + 40-bit Global ID + 16-bit Subnet-ID + 64-bit Interface ID + // For services, we use a /112 which leaves 16 bits for service IPs + + var globalID [5]byte + if _, err := rand.Read(globalID[:]); err != nil { + // Fallback to a deterministic value if random generation fails + return "fd00:10:2::/112" + } + + // Format as fd + 5 bytes of Global ID + 2 bytes of Subnet ID (using 0000 for services) + return fmt.Sprintf("fd%02x:%02x%02x:%02x%02x::/112", + globalID[0], globalID[1], globalID[2], globalID[3], globalID[4]) +} diff --git a/plugin/pkg/shoot/mutator/admission_test.go b/plugin/pkg/shoot/mutator/admission_test.go index d99d88e1513..a62389db0b7 100644 --- a/plugin/pkg/shoot/mutator/admission_test.go +++ b/plugin/pkg/shoot/mutator/admission_test.go @@ -6,6 +6,7 @@ package mutator_test import ( "context" + "net" "time" . "github.com/onsi/ginkgo/v2" @@ -422,7 +423,7 @@ var _ = Describe("mutator", func() { Expect(shoot.Spec.Networking.Services).To(BeNil()) }) - It("should not default shoot pod network if shoot is workerless", func() { + It("should skip pod network defaulting but default service network for workerless shoots", func() { shoot.Spec.Provider.Workers = nil Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) @@ -435,6 +436,43 @@ var _ = Describe("mutator", func() { Expect(shoot.Spec.Networking.Services).To(Equal(&servicesCIDR)) }) + It("should generate ULA services CIDR for workerless IPv6 shoots without seed defaults", func() { + shoot.Spec.Provider.Workers = nil + shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6} + Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Networking.Pods).To(BeNil()) + + Expect(shoot.Spec.Networking.Services).NotTo(BeNil()) + servicesCIDR := *shoot.Spec.Networking.Services + _, ipNet, parseErr := net.ParseCIDR(servicesCIDR) + Expect(parseErr).NotTo(HaveOccurred(), "Generated services CIDR should be valid") + Expect(ipNet.IP.To16()).NotTo(BeNil(), "Generated services CIDR should be IPv6") + Expect(ipNet.IP[0]).To(Equal(byte(0xfd)), "Generated services CIDR should be in ULA range (fd00::/8)") + Expect(shoot.Spec.Networking.Services).To(Equal(&servicesCIDR)) + }) + + It("should use seed defaults for workerless IPv6 shoots when available", func() { + servicesCIDR := "2001:db8:10::/112" + seed.Spec.Networks.ShootDefaults.Services = ptr.To(servicesCIDR) + shoot.Spec.Provider.Workers = nil + shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6} + Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Networking.Pods).To(BeNil()) + + Expect(shoot.Spec.Networking.Services).NotTo(BeNil()) + Expect(shoot.Spec.Networking.Services).To(Equal(&servicesCIDR)) + }) + It("should not default shoot networks if shoot IP family does not match seed .spec.networks.shootDefaults", func() { shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6} diff --git a/plugin/pkg/shoot/validator/admission.go b/plugin/pkg/shoot/validator/admission.go index 7bd770a22da..c35b0f309df 100644 --- a/plugin/pkg/shoot/validator/admission.go +++ b/plugin/pkg/shoot/validator/admission.go @@ -874,9 +874,10 @@ func (c *validationContext) validateShootNetworks(a admission.Attributes, worker allErrs = append(allErrs, field.Required(path.Child("pods"), "pods is required")) } - if c.shoot.Spec.Networking.Services == nil && - slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv4) { - allErrs = append(allErrs, field.Required(path.Child("services"), "services is required")) + if c.shoot.Spec.Networking.Services == nil { + if slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv4) || (workerless && slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv6)) { + allErrs = append(allErrs, field.Required(path.Child("services"), "services is required")) + } } if slices.Contains(c.shoot.Spec.Networking.IPFamilies, core.IPFamilyIPv4) { diff --git a/plugin/pkg/shoot/validator/admission_test.go b/plugin/pkg/shoot/validator/admission_test.go index 92975828121..01b6eea6a2a 100644 --- a/plugin/pkg/shoot/validator/admission_test.go +++ b/plugin/pkg/shoot/validator/admission_test.go @@ -2322,6 +2322,41 @@ var _ = Describe("validator", func() { }) + It("should reject because shoot services network is nil (workerless Shoot, Ipv6)", func() { + shoot.Spec.Provider.Workers = nil + shoot.Spec.Networking.Services = nil + shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6} + + Expect(coreInformerFactory.Core().V1beta1().Projects().Informer().GetStore().Add(&project)).To(Succeed()) + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) + Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) + Expect(coreInformerFactory.Core().V1beta1().SecretBindings().Informer().GetStore().Add(&secretBinding)).To(Succeed()) + Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("services is required"))) + + }) + + It("should not reject because shoot services network is nil for IPv6", func() { + shoot.Spec.Networking.Services = nil + shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6} + + Expect(coreInformerFactory.Core().V1beta1().Projects().Informer().GetStore().Add(&project)).To(Succeed()) + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) + Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) + Expect(coreInformerFactory.Core().V1beta1().SecretBindings().Informer().GetStore().Add(&secretBinding)).To(Succeed()) + Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + }) + It("should reject because the shoot node and the seed node networks intersect (HA control plane)", func() { shoot.Spec.Networking.Nodes = &seedNodesCIDR shoot.Spec.ControlPlane = &core.ControlPlane{HighAvailability: &core.HighAvailability{FailureTolerance: core.FailureTolerance{Type: core.FailureToleranceTypeZone}}} From c0fe68f8b19b94895fe7016f05f6f09efa4e37c3 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Wed, 5 Nov 2025 09:53:17 +0200 Subject: [PATCH 032/176] Fetch the referenced `CloudProfile` / `NamespacedCloudProfile` (part 4) (#13222) * Fetch the referenced CloudProfile / NamespacedCloudProfile * Fix integration tests --- plugin/pkg/shoot/mutator/admission.go | 44 ++++++++++++++---- plugin/pkg/shoot/mutator/admission_test.go | 46 +++++++++++++++++-- .../seed/seed_suite_test.go | 2 +- .../credentialsbinding_suite_test.go | 2 +- .../exposureclass/exposureclass_suite_test.go | 2 +- .../managedseedset_suite_test.go | 2 +- .../project/activity/activity_suite_test.go | 2 +- .../project/project/project_suite_test.go | 2 +- .../project/stale/stale_suite_test.go | 2 +- .../secretbinding/secretbinding_suite_test.go | 2 +- .../seed/lifecycle/lifecycle_suite_test.go | 2 +- .../shoot/conditions/conditions_suite_test.go | 2 +- .../hibernation/hibernation_suite_test.go | 2 +- .../maintenance/maintenance_suite_test.go | 2 +- .../shoot/quota/quota_suite_test.go | 2 +- .../shoot/reference/reference_suite_test.go | 2 +- .../shoot/retry/retry_suite_test.go | 2 +- .../statuslabel/statuslabel_suite_test.go | 2 +- .../backupentry/backupentry_suite_test.go | 2 +- .../gardenlet/bastion/bastion_suite_test.go | 2 +- .../managedseed/managedseed_suite_test.go | 2 +- .../gardenlet/shoot/care/care_suite_test.go | 2 +- 22 files changed, 96 insertions(+), 34 deletions(-) diff --git a/plugin/pkg/shoot/mutator/admission.go b/plugin/pkg/shoot/mutator/admission.go index db7cd9a89df..33cefdda30b 100644 --- a/plugin/pkg/shoot/mutator/admission.go +++ b/plugin/pkg/shoot/mutator/admission.go @@ -29,6 +29,7 @@ import ( gardencoreinformers "github.com/gardener/gardener/pkg/client/core/informers/externalversions" gardencorev1beta1listers "github.com/gardener/gardener/pkg/client/core/listers/core/v1beta1" "github.com/gardener/gardener/pkg/controllerutils" + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" plugin "github.com/gardener/gardener/plugin/pkg" ) @@ -43,8 +44,10 @@ func Register(plugins *admission.Plugins) { type MutateShoot struct { *admission.Handler - seedLister gardencorev1beta1listers.SeedLister - readyFunc admission.ReadyFunc + cloudProfileLister gardencorev1beta1listers.CloudProfileLister + namespacedCloudProfileLister gardencorev1beta1listers.NamespacedCloudProfileLister + seedLister gardencorev1beta1listers.SeedLister + readyFunc admission.ReadyFunc } var ( @@ -68,17 +71,31 @@ func (m *MutateShoot) AssignReadyFunc(f admission.ReadyFunc) { // SetCoreInformerFactory gets Lister from SharedInformerFactory. func (m *MutateShoot) SetCoreInformerFactory(f gardencoreinformers.SharedInformerFactory) { + cloudProfileInformer := f.Core().V1beta1().CloudProfiles() + m.cloudProfileLister = cloudProfileInformer.Lister() + + namespacedCloudProfileInformer := f.Core().V1beta1().NamespacedCloudProfiles() + m.namespacedCloudProfileLister = namespacedCloudProfileInformer.Lister() + seedInformer := f.Core().V1beta1().Seeds() m.seedLister = seedInformer.Lister() readyFuncs = append( readyFuncs, + cloudProfileInformer.Informer().HasSynced, + namespacedCloudProfileInformer.Informer().HasSynced, seedInformer.Informer().HasSynced, ) } // ValidateInitialization checks whether the plugin was correctly initialized. func (m *MutateShoot) ValidateInitialization() error { + if m.cloudProfileLister == nil { + return errors.New("missing cloudProfile lister") + } + if m.namespacedCloudProfileLister == nil { + return errors.New("missing namespacedCloudProfile lister") + } if m.seedLister == nil { return errors.New("missing seed lister") } @@ -126,9 +143,16 @@ func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admissi } } + cloudProfileSpec, err := gardenerutils.GetCloudProfileSpec(m.cloudProfileLister, m.namespacedCloudProfileLister, shoot) + if err != nil { + return apierrors.NewInternalError(fmt.Errorf("could not find referenced cloud profile: %w", err)) + } + if cloudProfileSpec == nil { + return nil + } + var seed *gardencorev1beta1.Seed if shoot.Spec.SeedName != nil { - var err error seed, err = m.seedLister.Get(*shoot.Spec.SeedName) if err != nil { return apierrors.NewInternalError(fmt.Errorf("could not find referenced seed %q: %w", *shoot.Spec.SeedName, err)) @@ -136,9 +160,10 @@ func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admissi } mutationContext := &mutationContext{ - seed: seed, - shoot: shoot, - oldShoot: oldShoot, + cloudProfileSpec: cloudProfileSpec, + seed: seed, + shoot: shoot, + oldShoot: oldShoot, } if a.GetOperation() == admission.Create { @@ -152,9 +177,10 @@ func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admissi } type mutationContext struct { - seed *gardencorev1beta1.Seed - shoot *core.Shoot - oldShoot *core.Shoot + cloudProfileSpec *gardencorev1beta1.CloudProfileSpec + seed *gardencorev1beta1.Seed + shoot *core.Shoot + oldShoot *core.Shoot } func addCreatedByAnnotation(shoot *core.Shoot, userName string) { diff --git a/plugin/pkg/shoot/mutator/admission_test.go b/plugin/pkg/shoot/mutator/admission_test.go index a62389db0b7..95377e51a8a 100644 --- a/plugin/pkg/shoot/mutator/admission_test.go +++ b/plugin/pkg/shoot/mutator/admission_test.go @@ -51,12 +51,12 @@ var _ = Describe("mutator", func() { }) Describe("#ValidateInitialization", func() { - It("should return error if no SeedLister is set", func() { + It("should return error if a lister is missing", func() { admissionHandler, err := New() Expect(err).NotTo(HaveOccurred()) err = admissionHandler.ValidateInitialization() - Expect(err).To(MatchError("missing seed lister")) + Expect(err).To(MatchError("missing cloudProfile lister")) }) It("should not return error if all listers are set", func() { @@ -73,9 +73,10 @@ var _ = Describe("mutator", func() { var ( ctx context.Context - userInfo = &user.DefaultInfo{Name: "foo"} - seed gardencorev1beta1.Seed - shoot core.Shoot + userInfo = &user.DefaultInfo{Name: "foo"} + cloudProfile gardencorev1beta1.CloudProfile + seed gardencorev1beta1.Seed + shoot core.Shoot coreInformerFactory gardencoreinformers.SharedInformerFactory @@ -85,6 +86,11 @@ var _ = Describe("mutator", func() { BeforeEach(func() { ctx = context.Background() + cloudProfile = gardencorev1beta1.CloudProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "profile", + }, + } seed = gardencorev1beta1.Seed{ ObjectMeta: metav1.ObjectMeta{ Name: "seed", @@ -96,6 +102,7 @@ var _ = Describe("mutator", func() { Namespace: "garden-my-project", }, Spec: core.ShootSpec{ + CloudProfileName: ptr.To("profile"), Provider: core.Provider{ Workers: []core.Worker{ { @@ -152,7 +159,26 @@ var _ = Describe("mutator", func() { }) Context("reference checks", func() { + It("should reject because the referenced cloud profile was not found", func() { + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(BeInternalServerError()) + }) + + It("should exit early if CloudProfile is not set", func() { + shoot.Spec.CloudProfileName = nil + shoot.Spec.CloudProfile = nil + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + + err := admissionHandler.Admit(ctx, attrs, nil) + Expect(err).NotTo(HaveOccurred()) + }) + It("should reject because the referenced seed was not found", func() { + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) + shoot.Spec.SeedName = ptr.To("seed") attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) @@ -162,6 +188,10 @@ var _ = Describe("mutator", func() { }) Context("created-by annotation", func() { + BeforeEach(func() { + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) + }) + It("should add the created-by annotation on shoot creation", func() { Expect(shoot.Annotations).NotTo(HaveKeyWithValue(v1beta1constants.GardenCreatedBy, userInfo.Name)) @@ -189,6 +219,8 @@ var _ = Describe("mutator", func() { BeforeEach(func() { oldShoot = shoot.DeepCopy() + + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) }) It("should add deploy tasks because shoot is being created", func() { @@ -333,6 +365,8 @@ var _ = Describe("mutator", func() { BeforeEach(func() { shoot.Spec.Maintenance = &core.Maintenance{} oldShoot = shoot.DeepCopy() + + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) }) DescribeTable("confine spec roll-out checks", @@ -397,6 +431,8 @@ var _ = Describe("mutator", func() { Services: nil, IPFamilies: []core.IPFamily{core.IPFamilyIPv4}, } + + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) }) It("should not default shoot networks if shoot .spec.seedName is nil", func() { diff --git a/test/integration/controllermanager/controllerregistration/seed/seed_suite_test.go b/test/integration/controllermanager/controllerregistration/seed/seed_suite_test.go index 3d81fb9c500..8f054463d49 100644 --- a/test/integration/controllermanager/controllerregistration/seed/seed_suite_test.go +++ b/test/integration/controllermanager/controllerregistration/seed/seed_suite_test.go @@ -74,7 +74,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ControllerRegistrationResources"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ControllerRegistrationResources,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/credentialsbinding/credentialsbinding_suite_test.go b/test/integration/controllermanager/credentialsbinding/credentialsbinding_suite_test.go index cff25bce225..ac14e0ddf51 100644 --- a/test/integration/controllermanager/credentialsbinding/credentialsbinding_suite_test.go +++ b/test/integration/controllermanager/credentialsbinding/credentialsbinding_suite_test.go @@ -59,7 +59,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/exposureclass/exposureclass_suite_test.go b/test/integration/controllermanager/exposureclass/exposureclass_suite_test.go index e156cad9b3b..7779ef9cf10 100644 --- a/test/integration/controllermanager/exposureclass/exposureclass_suite_test.go +++ b/test/integration/controllermanager/exposureclass/exposureclass_suite_test.go @@ -64,7 +64,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootExposureClass,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootExposureClass,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/managedseedset/managedseedset_suite_test.go b/test/integration/controllermanager/managedseedset/managedseedset_suite_test.go index f3faecdb2aa..284840261e6 100644 --- a/test/integration/controllermanager/managedseedset/managedseedset_suite_test.go +++ b/test/integration/controllermanager/managedseedset/managedseedset_suite_test.go @@ -62,7 +62,7 @@ var _ = BeforeSuite(func() { testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ Args: []string{ - "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,SeedValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootValidator", + "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,SeedValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootValidator,ShootMutator", }, }, } diff --git a/test/integration/controllermanager/project/activity/activity_suite_test.go b/test/integration/controllermanager/project/activity/activity_suite_test.go index 3c0d052eb83..d27a014a293 100644 --- a/test/integration/controllermanager/project/activity/activity_suite_test.go +++ b/test/integration/controllermanager/project/activity/activity_suite_test.go @@ -61,7 +61,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ExtensionLabels,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ExtensionLabels,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/project/project/project_suite_test.go b/test/integration/controllermanager/project/project/project_suite_test.go index 23114840f17..3a26cebeddc 100644 --- a/test/integration/controllermanager/project/project/project_suite_test.go +++ b/test/integration/controllermanager/project/project/project_suite_test.go @@ -69,7 +69,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/project/stale/stale_suite_test.go b/test/integration/controllermanager/project/stale/stale_suite_test.go index 7ac03cfb3fd..1b604f6c01b 100644 --- a/test/integration/controllermanager/project/stale/stale_suite_test.go +++ b/test/integration/controllermanager/project/stale/stale_suite_test.go @@ -69,7 +69,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ExtensionLabels,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ExtensionLabels,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/secretbinding/secretbinding_suite_test.go b/test/integration/controllermanager/secretbinding/secretbinding_suite_test.go index a4537a38a66..c967a14cfdb 100644 --- a/test/integration/controllermanager/secretbinding/secretbinding_suite_test.go +++ b/test/integration/controllermanager/secretbinding/secretbinding_suite_test.go @@ -59,7 +59,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/seed/lifecycle/lifecycle_suite_test.go b/test/integration/controllermanager/seed/lifecycle/lifecycle_suite_test.go index 44abd21ecb0..f354fbdfde2 100644 --- a/test/integration/controllermanager/seed/lifecycle/lifecycle_suite_test.go +++ b/test/integration/controllermanager/seed/lifecycle/lifecycle_suite_test.go @@ -69,7 +69,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,SeedValidator,SeedMutator,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,SeedValidator,SeedMutator,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/shoot/conditions/conditions_suite_test.go b/test/integration/controllermanager/shoot/conditions/conditions_suite_test.go index 55aef39fc5e..da0e3fef7bd 100644 --- a/test/integration/controllermanager/shoot/conditions/conditions_suite_test.go +++ b/test/integration/controllermanager/shoot/conditions/conditions_suite_test.go @@ -59,7 +59,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/shoot/hibernation/hibernation_suite_test.go b/test/integration/controllermanager/shoot/hibernation/hibernation_suite_test.go index fd1de2fd55e..b69dacfd989 100644 --- a/test/integration/controllermanager/shoot/hibernation/hibernation_suite_test.go +++ b/test/integration/controllermanager/shoot/hibernation/hibernation_suite_test.go @@ -59,7 +59,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/shoot/maintenance/maintenance_suite_test.go b/test/integration/controllermanager/shoot/maintenance/maintenance_suite_test.go index f603ef6680f..ae49aa7e44d 100644 --- a/test/integration/controllermanager/shoot/maintenance/maintenance_suite_test.go +++ b/test/integration/controllermanager/shoot/maintenance/maintenance_suite_test.go @@ -63,7 +63,7 @@ var _ = BeforeSuite(func() { testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ Args: []string{ - "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction", + "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootMutator", }, }, } diff --git a/test/integration/controllermanager/shoot/quota/quota_suite_test.go b/test/integration/controllermanager/shoot/quota/quota_suite_test.go index a0b6aabf2e7..5d5d13e78ba 100644 --- a/test/integration/controllermanager/shoot/quota/quota_suite_test.go +++ b/test/integration/controllermanager/shoot/quota/quota_suite_test.go @@ -62,7 +62,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/shoot/reference/reference_suite_test.go b/test/integration/controllermanager/shoot/reference/reference_suite_test.go index a94add97447..b675e1446ab 100644 --- a/test/integration/controllermanager/shoot/reference/reference_suite_test.go +++ b/test/integration/controllermanager/shoot/reference/reference_suite_test.go @@ -57,7 +57,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/shoot/retry/retry_suite_test.go b/test/integration/controllermanager/shoot/retry/retry_suite_test.go index 22cd7e1b00c..66b4f442683 100644 --- a/test/integration/controllermanager/shoot/retry/retry_suite_test.go +++ b/test/integration/controllermanager/shoot/retry/retry_suite_test.go @@ -57,7 +57,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootMutator"}, }, } diff --git a/test/integration/controllermanager/shoot/statuslabel/statuslabel_suite_test.go b/test/integration/controllermanager/shoot/statuslabel/statuslabel_suite_test.go index ca48d66d728..3ebc629fdbb 100644 --- a/test/integration/controllermanager/shoot/statuslabel/statuslabel_suite_test.go +++ b/test/integration/controllermanager/shoot/statuslabel/statuslabel_suite_test.go @@ -57,7 +57,7 @@ var _ = BeforeSuite(func() { By("Start test environment") testEnv = &gardenerenvtest.GardenerTestEnvironment{ GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootQuotaValidator,ShootValidator,ShootTolerationRestriction,ShootDNS,ShootMutator"}, }, } diff --git a/test/integration/gardenlet/backupentry/backupentry_suite_test.go b/test/integration/gardenlet/backupentry/backupentry_suite_test.go index c2baa2a2354..af8996181d7 100644 --- a/test/integration/gardenlet/backupentry/backupentry_suite_test.go +++ b/test/integration/gardenlet/backupentry/backupentry_suite_test.go @@ -87,7 +87,7 @@ var _ = BeforeSuite(func() { ErrorIfCRDPathMissing: true, }, GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,Bastion,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,Bastion,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ShootMutator"}, }, } diff --git a/test/integration/gardenlet/bastion/bastion_suite_test.go b/test/integration/gardenlet/bastion/bastion_suite_test.go index c83bd80a6ef..fc4521385c3 100644 --- a/test/integration/gardenlet/bastion/bastion_suite_test.go +++ b/test/integration/gardenlet/bastion/bastion_suite_test.go @@ -87,7 +87,7 @@ var _ = BeforeSuite(func() { ErrorIfCRDPathMissing: true, }, GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,Bastion,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,SeedValidator"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,Bastion,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,SeedValidator,ShootMutator"}, }, } diff --git a/test/integration/gardenlet/managedseed/managedseed_suite_test.go b/test/integration/gardenlet/managedseed/managedseed_suite_test.go index c30b33b6841..375395233cf 100644 --- a/test/integration/gardenlet/managedseed/managedseed_suite_test.go +++ b/test/integration/gardenlet/managedseed/managedseed_suite_test.go @@ -84,7 +84,7 @@ var _ = BeforeSuite(func() { Environment: &envtest.Environment{}, GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ Args: []string{ - "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,SeedValidator,ShootQuotaValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootDNS,ShootValidator,SeedValidator", + "--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,SeedValidator,ShootQuotaValidator,ShootTolerationRestriction,ManagedSeedShoot,ManagedSeed,ShootManagedSeed,ShootDNS,ShootValidator,SeedValidator,ShootMutator", }, }, } diff --git a/test/integration/gardenlet/shoot/care/care_suite_test.go b/test/integration/gardenlet/shoot/care/care_suite_test.go index a92c6e65ea6..cd2228a4202 100644 --- a/test/integration/gardenlet/shoot/care/care_suite_test.go +++ b/test/integration/gardenlet/shoot/care/care_suite_test.go @@ -118,7 +118,7 @@ var _ = BeforeSuite(func() { ErrorIfCRDPathMissing: true, }, GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ - Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator"}, + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ShootMutator"}, }, } From 50281d280afc360c0d55f4cac632a0bbfccf4994 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 5 Nov 2025 10:52:17 +0100 Subject: [PATCH 033/176] Update quay.io/prometheus/node-exporter Docker tag to v1.10.2 (#13266) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 585af4b3f6a..a58dc41993f 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -317,7 +317,7 @@ images: - name: node-exporter sourceRepository: github.com/prometheus/node_exporter repository: quay.io/prometheus/node-exporter - tag: v1.9.1 + tag: v1.10.2 labels: - name: gardener.cloud/cve-categorisation value: From ab10b0ae988a42ab9e4e5c5c62b2e932d107ff00 Mon Sep 17 00:00:00 2001 From: Vladimir Nachev Date: Wed, 5 Nov 2025 11:52:24 +0200 Subject: [PATCH 034/176] [GEP-26] Fix backup migration from WorkloadIdentity to Secret (#13282) * Fix backup migration from WorkloadIdentity to Secret When the WorkloadIdentity relevant annotations and labels are left on the backup bucket/entry secret in the seed, other controllers decides that indeed WorkloadIdentity credentials are being used, however they fail to reconcile because data fields like 'config' are not set. Therefore, the backupbucket and backupentry controllers now cleanup all unknown annotations and labels. Data fields are already being cleaned up. * Change backupentry genericactuator to unset the annotations and labels --- .../backupentry/genericactuator/actuator.go | 4 ++++ .../genericactuator/actuator_test.go | 19 +++++++++++++++++++ .../controller/backupbucket/reconciler.go | 3 ++- .../backupbucket/reconciler_test.go | 16 ++++++++++++++++ .../controller/backupentry/reconciler.go | 3 ++- .../controller/backupentry/reconciler_test.go | 17 +++++++++++++++++ 6 files changed, 60 insertions(+), 2 deletions(-) diff --git a/extensions/pkg/controller/backupentry/genericactuator/actuator.go b/extensions/pkg/controller/backupentry/genericactuator/actuator.go index e743b25ff82..afef0369e15 100644 --- a/extensions/pkg/controller/backupentry/genericactuator/actuator.go +++ b/extensions/pkg/controller/backupentry/genericactuator/actuator.go @@ -132,6 +132,10 @@ func (a *actuator) deployEtcdBackupSecret(ctx context.Context, log logr.Logger, if token, ok := etcdSecret.Data[securityv1alpha1constants.DataKeyToken]; ok { etcdSecretData[securityv1alpha1constants.DataKeyToken] = token } + } else { + // Unset all annotations and labels when static credentials are used. + etcdSecret.Annotations = map[string]string{} + etcdSecret.Labels = nil } metav1.SetMetaDataAnnotation(&etcdSecret.ObjectMeta, AnnotationKeyCreatedByBackupEntry, be.Name) diff --git a/extensions/pkg/controller/backupentry/genericactuator/actuator_test.go b/extensions/pkg/controller/backupentry/genericactuator/actuator_test.go index d5eb8c584f3..0823728f456 100644 --- a/extensions/pkg/controller/backupentry/genericactuator/actuator_test.go +++ b/extensions/pkg/controller/backupentry/genericactuator/actuator_test.go @@ -153,6 +153,25 @@ var _ = Describe("Actuator", func() { Expect(actual.Data).To(Equal(etcdBackupSecretData)) }) + It("should remove all unknown annotations and labels from the etcd-backup secret", func() { + existingEtcdBackupSecret := etcdBackupSecret.DeepCopy() + testKeyValue := map[string]string{"key1": "value1", "key2:": "value2"} + existingEtcdBackupSecret.Annotations = testKeyValue + existingEtcdBackupSecret.Labels = testKeyValue + + fakeClient = fakeclient.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(seedNamespace, backupEntrySecret, existingEtcdBackupSecret).Build() + mgr.EXPECT().GetClient().Return(fakeClient) + backupEntryDelegate.EXPECT().GetETCDSecretData(ctx, gomock.AssignableToTypeOf(logr.Logger{}), backupEntry, backupProviderSecretData).Return(etcdBackupSecretData, nil) + + a = genericactuator.NewActuator(mgr, backupEntryDelegate) + Expect(a.Reconcile(ctx, log, backupEntry)).To(Succeed()) + + actual := &corev1.Secret{} + Expect(fakeClient.Get(ctx, etcdBackupSecretKey, actual)).To(Succeed()) + Expect(actual.Annotations).To(Equal(map[string]string{"backup.gardener.cloud/created-by": backupEntry.Name})) + Expect(actual.Labels).To(BeEmpty()) + }) + Context("#WorkloadIdentity", func() { var existingEtcdBackupSecret *corev1.Secret diff --git a/pkg/gardenlet/controller/backupbucket/reconciler.go b/pkg/gardenlet/controller/backupbucket/reconciler.go index 2c1c0ca2d97..cdf508ffff4 100644 --- a/pkg/gardenlet/controller/backupbucket/reconciler.go +++ b/pkg/gardenlet/controller/backupbucket/reconciler.go @@ -374,7 +374,8 @@ func (r *Reconciler) reconcileBackupBucketExtensionSecret(ctx context.Context, e switch credentials := backupCredentials.(type) { case *corev1.Secret: _, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.SeedClient, extensionSecret, func() error { - metav1.SetMetaDataAnnotation(&extensionSecret.ObjectMeta, v1beta1constants.GardenerTimestamp, now) + extensionSecret.Annotations = map[string]string{v1beta1constants.GardenerTimestamp: now} + extensionSecret.Labels = nil extensionSecret.Data = credentials.Data return nil }) diff --git a/pkg/gardenlet/controller/backupbucket/reconciler_test.go b/pkg/gardenlet/controller/backupbucket/reconciler_test.go index 20c1d1c0ae0..5820a50bd0a 100644 --- a/pkg/gardenlet/controller/backupbucket/reconciler_test.go +++ b/pkg/gardenlet/controller/backupbucket/reconciler_test.go @@ -316,6 +316,22 @@ var _ = Describe("Controller", func() { Expect(seedClient.Get(ctx, client.ObjectKeyFromObject(extensionBackupBucket), extensionBackupBucket)).To(Succeed()) Expect(extensionBackupBucket.Annotations).To(HaveKey(v1beta1constants.GardenerOperation)) }) + + It("should clean the unknown annotations and labels from the extension secret on reconciliation", func() { + testKeyValue := map[string]string{"key1": "value1", "key2:": "value2"} + extensionSecret.Annotations = testKeyValue // Do not set the timestamp annotation to trigger reconciliation + extensionSecret.Labels = testKeyValue + Expect(seedClient.Create(ctx, extensionSecret)).To(Succeed()) + Expect(seedClient.Create(ctx, extensionBackupBucket)).To(Succeed()) + + result, err := reconciler.Reconcile(ctx, request) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + Expect(seedClient.Get(ctx, client.ObjectKeyFromObject(extensionSecret), extensionSecret)).To(Succeed()) + Expect(extensionSecret.Annotations).To(Equal(map[string]string{v1beta1constants.GardenerTimestamp: fakeClock.Now().UTC().Format(time.RFC3339Nano)})) + Expect(extensionSecret.Labels).To(BeEmpty()) + }) }) Describe("#WorkloadIdentity Credentials", func() { diff --git a/pkg/gardenlet/controller/backupentry/reconciler.go b/pkg/gardenlet/controller/backupentry/reconciler.go index 2f7849e4eb5..0773b0137e0 100644 --- a/pkg/gardenlet/controller/backupentry/reconciler.go +++ b/pkg/gardenlet/controller/backupentry/reconciler.go @@ -667,7 +667,8 @@ func (r *Reconciler) reconcileBackupEntryExtensionSecret(ctx context.Context, ex switch credentials := backupCredentials.(type) { case *corev1.Secret: _, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.SeedClient, extensionSecret, func() error { - metav1.SetMetaDataAnnotation(&extensionSecret.ObjectMeta, v1beta1constants.GardenerTimestamp, now) + extensionSecret.Annotations = map[string]string{v1beta1constants.GardenerTimestamp: now} + extensionSecret.Labels = nil extensionSecret.Data = credentials.Data return nil }) diff --git a/pkg/gardenlet/controller/backupentry/reconciler_test.go b/pkg/gardenlet/controller/backupentry/reconciler_test.go index 2d3ece7887e..59e50190e2b 100644 --- a/pkg/gardenlet/controller/backupentry/reconciler_test.go +++ b/pkg/gardenlet/controller/backupentry/reconciler_test.go @@ -350,6 +350,23 @@ var _ = Describe("Controller", func() { "generatedSecret2": []byte("generatedValue2"), })) }) + + It("should clean the unknown annotations and labels from the extension secret on reconciliation", func() { + testKeyValues := map[string]string{"key1": "value1", "key2:": "value2"} + extensionSecret.Annotations = testKeyValues // Do not set the timestamp annotation to trigger reconciliation + extensionSecret.Labels = testKeyValues + Expect(seedClient.Create(ctx, extensionSecret)).To(Succeed()) + Expect(seedClient.Create(ctx, extensionBackupEntry)).To(Succeed()) + + result, err := reconciler.Reconcile(ctx, request) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + Expect(seedClient.Get(ctx, client.ObjectKeyFromObject(extensionSecret), extensionSecret)).To(Succeed()) + Expect(extensionSecret.Annotations).To(Equal(map[string]string{v1beta1constants.GardenerTimestamp: fakeClock.Now().UTC().Format(time.RFC3339Nano)})) + Expect(extensionSecret.Labels).To(BeEmpty()) + + }) }) Describe("#WorkloadIdentity Credentials", func() { From 883f8d383e9ba2548b8f50cc5836e979e446f4df Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 5 Nov 2025 13:07:18 +0100 Subject: [PATCH 035/176] Update opentelemetry-operator to v0.138.0 (#13323) --- go.mod | 2 +- go.sum | 4 ++-- imagevector/containers.yaml | 2 +- ...crd-opentelemetry.io_instrumentations.yaml | 7 ++++++ ...ntelemetry.io_opentelemetrycollectors.yaml | 24 +++++++++++++++++++ ...crd-opentelemetry.io_targetallocators.yaml | 2 ++ 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index b578bb677c1..7d0539e82c7 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/onsi/ginkgo/v2 v2.27.1 github.com/onsi/gomega v1.38.2 - github.com/open-telemetry/opentelemetry-operator v0.137.0 + github.com/open-telemetry/opentelemetry-operator v0.138.0 github.com/opencontainers/image-spec v1.1.1 github.com/pelletier/go-toml v1.9.5 github.com/perses/perses-operator v0.2.0 diff --git a/go.sum b/go.sum index a33960d6efe..4fe25342b26 100644 --- a/go.sum +++ b/go.sum @@ -576,8 +576,8 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= -github.com/open-telemetry/opentelemetry-operator v0.137.0 h1:RpaTlHGXVRtclYe0kao01SDwvftAfNYV00BuaeJJGxw= -github.com/open-telemetry/opentelemetry-operator v0.137.0/go.mod h1:RuM1oKvL0W9gNONH1mpV/1g08jGu7LugSl0BOkhuQhk= +github.com/open-telemetry/opentelemetry-operator v0.138.0 h1:PWNLD3i+sGpPLV2HpTylgGXuJB7iZp4cpoGDtw2XTBo= +github.com/open-telemetry/opentelemetry-operator v0.138.0/go.mod h1:RuM1oKvL0W9gNONH1mpV/1g08jGu7LugSl0BOkhuQhk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index a58dc41993f..37774cab9e0 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -188,7 +188,7 @@ images: - name: opentelemetry-operator sourceRepository: github.com/open-telemetry/opentelemetry-operator repository: europe-docker.pkg.dev/gardener-project/releases/3rd/opentelemetry-operator/opentelemetry-operator - tag: "v0.137.0" + tag: "v0.138.0" labels: - name: 'gardener.cloud/cve-categorisation' value: diff --git a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_instrumentations.yaml b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_instrumentations.yaml index 4ce815cc83d..656a939dad4 100644 --- a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_instrumentations.yaml +++ b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_instrumentations.yaml @@ -672,6 +672,7 @@ spec: description: |- VolumeSizeLimit defines size limit for volume used for auto-instrumentation. The default size is 200Mi. + Deprecated: use spec..volume.size instead. This field will be inactive in a future release. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -1133,6 +1134,7 @@ spec: description: |- VolumeSizeLimit defines size limit for volume used for auto-instrumentation. The default size is 200Mi. + Deprecated: use spec..volume.size instead. This field will be inactive in a future release. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -1789,6 +1791,7 @@ spec: description: |- VolumeSizeLimit defines size limit for volume used for auto-instrumentation. The default size is 200Mi. + Deprecated: use spec..volume.size instead. This field will be inactive in a future release. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -2264,6 +2267,7 @@ spec: description: |- VolumeSizeLimit defines size limit for volume used for auto-instrumentation. The default size is 200Mi. + Deprecated: use spec..volume.size instead. This field will be inactive in a future release. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -2879,6 +2883,7 @@ spec: description: |- VolumeSizeLimit defines size limit for volume used for auto-instrumentation. The default size is 200Mi. + Deprecated: use spec..volume.size instead. This field will be inactive in a future release. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -3328,6 +3333,7 @@ spec: description: |- VolumeSizeLimit defines size limit for volume used for auto-instrumentation. The default size is 200Mi. + Deprecated: use spec..volume.size instead. This field will be inactive in a future release. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -3795,6 +3801,7 @@ spec: description: |- VolumeSizeLimit defines size limit for volume used for auto-instrumentation. The default size is 200Mi. + Deprecated: use spec..volume.size instead. This field will be inactive in a future release. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object diff --git a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opentelemetrycollectors.yaml b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opentelemetrycollectors.yaml index 1e4a435e3ab..b03164db519 100644 --- a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opentelemetrycollectors.yaml +++ b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_opentelemetrycollectors.yaml @@ -2678,6 +2678,7 @@ spec: description: MaxReplicas sets an upper bound to the autoscaling feature. If MaxReplicas is set autoscaling is enabled. format: int32 + minimum: 1 type: integer metrics: description: |- @@ -2809,17 +2810,20 @@ spec: feature. Set this if you are using autoscaling. It must be at least 1 format: int32 + minimum: 1 type: integer targetCPUUtilization: description: |- TargetCPUUtilization sets the target average CPU used across all replicas. If average CPU exceeds this value, the HPA will scale up. Defaults to 90 percent. format: int32 + minimum: 1 type: integer targetMemoryUtilization: description: TargetMemoryUtilization sets the target average memory utilization across all replicas format: int32 + minimum: 1 type: integer type: object config: @@ -5103,6 +5107,7 @@ spec: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 + minimum: 1 type: integer initialDelaySeconds: description: |- @@ -5110,18 +5115,21 @@ spec: Defaults to 0 seconds. Minimum value is 0. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 + minimum: 0 type: integer periodSeconds: description: |- How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 + minimum: 1 type: integer successThreshold: description: |- Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 + minimum: 1 type: integer terminationGracePeriodSeconds: description: |- @@ -5136,6 +5144,7 @@ spec: This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 + minimum: 1 type: integer timeoutSeconds: description: |- @@ -5143,6 +5152,7 @@ spec: Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 + minimum: 1 type: integer type: object managementState: @@ -5571,6 +5581,8 @@ spec: description: Allows defining which port to bind to the host in the Container. format: int32 + maximum: 65535 + minimum: 0 type: integer name: description: |- @@ -5638,6 +5650,7 @@ spec: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 + minimum: 1 type: integer initialDelaySeconds: description: |- @@ -5645,18 +5658,21 @@ spec: Defaults to 0 seconds. Minimum value is 0. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 + minimum: 0 type: integer periodSeconds: description: |- How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 + minimum: 1 type: integer successThreshold: description: |- Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 + minimum: 1 type: integer terminationGracePeriodSeconds: description: |- @@ -5671,6 +5687,7 @@ spec: This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 + minimum: 1 type: integer timeoutSeconds: description: |- @@ -5678,6 +5695,7 @@ spec: Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 + minimum: 1 type: integer type: object replicas: @@ -5969,6 +5987,7 @@ spec: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 + minimum: 1 type: integer initialDelaySeconds: description: |- @@ -5976,18 +5995,21 @@ spec: Defaults to 0 seconds. Minimum value is 0. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 + minimum: 0 type: integer periodSeconds: description: |- How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 + minimum: 1 type: integer successThreshold: description: |- Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 + minimum: 1 type: integer terminationGracePeriodSeconds: description: |- @@ -6002,6 +6024,7 @@ spec: This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 + minimum: 1 type: integer timeoutSeconds: description: |- @@ -6009,6 +6032,7 @@ spec: Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 + minimum: 1 type: integer type: object targetAllocator: diff --git a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_targetallocators.yaml b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_targetallocators.yaml index ef04fe2003e..a27df582f9b 100644 --- a/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_targetallocators.yaml +++ b/pkg/component/observability/opentelemetry/operator/assets/crd-opentelemetry.io_targetallocators.yaml @@ -4915,6 +4915,8 @@ spec: description: Allows defining which port to bind to the host in the Container. format: int32 + maximum: 65535 + minimum: 0 type: integer name: description: |- From fe1f44cd8c09e1a2fb0117560c9acc3a9cfa47e5 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 5 Nov 2025 13:07:25 +0100 Subject: [PATCH 036/176] Update dependency gardener/vpn2 to v0.44.0 (#13339) --- imagevector/containers.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 37774cab9e0..eb2bd901913 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -183,7 +183,7 @@ images: - name: vpn-server sourceRepository: github.com/gardener/vpn2 repository: europe-docker.pkg.dev/gardener-project/releases/gardener/vpn-server - tag: "0.43.0" + tag: "0.44.0" # OpenTelemetry - name: opentelemetry-operator sourceRepository: github.com/open-telemetry/opentelemetry-operator @@ -458,7 +458,7 @@ images: - name: vpn-client sourceRepository: github.com/gardener/vpn2 repository: europe-docker.pkg.dev/gardener-project/releases/gardener/vpn-client - tag: "0.43.0" + tag: "0.44.0" - name: coredns sourceRepository: github.com/coredns/coredns repository: registry.k8s.io/coredns/coredns From 602133e5bc2614ace13cd608beb9dc387910b0f7 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Singh Date: Wed, 5 Nov 2025 17:37:32 +0530 Subject: [PATCH 037/176] Update backup secret example links in Gardener setup docs (#13340) --- docs/deployment/setup_gardener.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/deployment/setup_gardener.md b/docs/deployment/setup_gardener.md index 21f4b40588a..ecb4dbda331 100644 --- a/docs/deployment/setup_gardener.md +++ b/docs/deployment/setup_gardener.md @@ -288,7 +288,12 @@ To obtain credentials and interact with the virtual Garden cluster, please follo Reference documentation and examples: - [Garden configuration example](../../example/operator/20-garden.yaml) -- [Backup secret examples](https://github.com/gardener/etcd-backup-restore/tree/master/example/storage-provider-secrets) +- Backup secret examples + - [Alicloud](https://github.com/gardener/gardener-extension-provider-alicloud/blob/master/example/30-etcd-backup-secret.yaml) + - [AWS](https://github.com/gardener/gardener-extension-provider-aws/blob/master/example/30-etcd-backup-secret.yaml) + - [Azure](https://github.com/gardener/gardener-extension-provider-azure/blob/master/example/30-etcd-backup-secret.yaml) + - [GCP](https://github.com/gardener/gardener-extension-provider-gcp/blob/master/example/30-etcd-backup-secret.yaml) + - [OpenStack](https://github.com/gardener/gardener-extension-provider-openstack/blob/master/example/30-etcd-backup-secret.yaml) - [Configuration options](../concepts/operator.md#garden-resources) - [Accessing the virtual Garden](../concepts/operator.md#virtual-garden-kubeconfig) From 66c5e058c9db2efda4a96a45482c5258884d67ee Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 5 Nov 2025 13:07:38 +0100 Subject: [PATCH 038/176] Update dependency golangci/golangci-lint to v2.6.1 (#13343) --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 876511c8741..2217843c385 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -54,7 +54,7 @@ TYPOS := $(TOOLS_BIN_DIR)/typos # default tool versions # renovate: datasource=github-releases depName=golangci/golangci-lint -GOLANGCI_LINT_VERSION ?= v2.6.0 +GOLANGCI_LINT_VERSION ?= v2.6.1 # renovate: datasource=github-releases depName=securego/gosec GOSEC_VERSION ?= v2.22.10 # renovate: datasource=github-releases depName=joelanford/go-apidiff From fd3a5cf00e23e51dbadae6cf0f18ff4f7473bfe8 Mon Sep 17 00:00:00 2001 From: Marin Atanasov Nikolov Date: Wed, 5 Nov 2025 14:07:45 +0200 Subject: [PATCH 039/176] docs/extensions: fix link to known extension implementations (#13359) --- docs/extensions/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/extensions/overview.md b/docs/extensions/overview.md index f808d6b4cb3..c97114e2469 100644 --- a/docs/extensions/overview.md +++ b/docs/extensions/overview.md @@ -185,4 +185,4 @@ We are truly looking forward to your feedback! ## Known Extensions -We track all extensions of Gardener in the [known Gardener Extensions List](../README.md#known-extension-implementations) repo. +We track all extensions of Gardener in the [known Gardener Extensions List](../../extensions/README.md#known-extension-implementations) repo. From be542101dcf8575c04627e659b11501b94342487 Mon Sep 17 00:00:00 2001 From: Indrit Fejza Date: Wed, 5 Nov 2025 14:49:19 +0100 Subject: [PATCH 040/176] add new panel for node load average to the node details dashboard (#13280) * add new panel for node load average * revert unneeded changes * do now change ips * use fill 0 * add Node to title to be in line with the other panels --- .../owners/worker/node-details-dashboard.json | 126 ++++++++++++++++-- 1 file changed, 115 insertions(+), 11 deletions(-) diff --git a/pkg/component/observability/plutono/dashboards/shoot/owners/worker/node-details-dashboard.json b/pkg/component/observability/plutono/dashboards/shoot/owners/worker/node-details-dashboard.json index c4350029860..2d24b414ffa 100644 --- a/pkg/component/observability/plutono/dashboards/shoot/owners/worker/node-details-dashboard.json +++ b/pkg/component/observability/plutono/dashboards/shoot/owners/worker/node-details-dashboard.json @@ -16,8 +16,7 @@ "editable": true, "gnetId": 2115, "graphTooltip": 0, - "id": 32, - "iteration": 1622621007137, + "iteration": 1761637646458, "links": [], "panels": [ { @@ -107,7 +106,7 @@ "showThresholdMarkers": true, "text": {} }, - "pluginVersion": "7.5.16", + "pluginVersion": "7.5.43", "targets": [ { "expr": "sum(rate(node_cpu_seconds_total{mode=~\"system|user|irq|softirq\", node=~\"$Node\"}[$__rate_interval])) / sum(kube_node_status_capacity{resource=\"cpu\", unit=\"core\", node=~\"$Node\"}) * 100", @@ -195,7 +194,7 @@ "showThresholdMarkers": true, "text": {} }, - "pluginVersion": "7.5.16", + "pluginVersion": "7.5.43", "targets": [ { "expr": "sum((node_memory_MemTotal_bytes{node=~\"$Node\"} - node_memory_MemAvailable_bytes{node=~\"$Node\"}) / node_memory_MemTotal_bytes{node=~\"$Node\"}) * 100", @@ -253,7 +252,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.16", + "pluginVersion": "7.5.43", "pointradius": 2, "points": false, "renderer": "flot", @@ -383,7 +382,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.16", + "pluginVersion": "7.5.43", "pointradius": 2, "points": false, "renderer": "flot", @@ -517,7 +516,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.16", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", @@ -633,7 +632,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.16", + "pluginVersion": "7.5.43", "pointradius": 2, "points": false, "renderer": "flot", @@ -707,6 +706,111 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Load average figure gives the number of jobs in the run queue (state R) or waiting for disk I/O (state D) averaged over 1 minute. See https://linux.die.net/man/5/proc", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 84, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.43", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "node_load1{node=\"$Node\"}", + "interval": "", + "legendFormat": "Load Average (1min)", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(kube_node_status_capacity{resource=\"cpu\", unit=\"core\", node=~\"$Node\"}) ", + "hide": false, + "interval": "", + "legendFormat": "CPU count", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "System Load Average ($Node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "collapsed": false, "datasource": null, @@ -759,7 +863,7 @@ "alertThreshold": false }, "percentage": false, - "pluginVersion": "7.5.16", + "pluginVersion": "7.5.43", "pointradius": 2, "points": false, "renderer": "flot", @@ -892,7 +996,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.16", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", @@ -1016,7 +1120,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.16", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", From a0d0a160768615a56ec71bfe18eb4dc2b1eeac1a Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 5 Nov 2025 14:49:26 +0100 Subject: [PATCH 041/176] Update module sigs.k8s.io/controller-runtime to v0.22.4 (#13333) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7d0539e82c7..96718ac8ab4 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( k8s.io/metrics v0.34.1 k8s.io/pod-security-admission v0.34.1 k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 - sigs.k8s.io/controller-runtime v0.22.3 + sigs.k8s.io/controller-runtime v0.22.4 sigs.k8s.io/controller-tools v0.19.0 sigs.k8s.io/yaml v1.6.0 ) diff --git a/go.sum b/go.sum index 4fe25342b26..bd38ba64b76 100644 --- a/go.sum +++ b/go.sum @@ -1153,8 +1153,8 @@ oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.22.3 h1:I7mfqz/a/WdmDCEnXmSPm8/b/yRTy6JsKKENTijTq8Y= -sigs.k8s.io/controller-runtime v0.22.3/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= +sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= +sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= sigs.k8s.io/controller-tools v0.19.0 h1:OU7jrPPiZusryu6YK0jYSjPqg8Vhf8cAzluP9XGI5uk= sigs.k8s.io/controller-tools v0.19.0/go.mod h1:y5HY/iNDFkmFla2CfQoVb2AQXMsBk4ad84iR1PLANB0= sigs.k8s.io/gateway-api v1.3.0 h1:q6okN+/UKDATola4JY7zXzx40WO4VISk7i9DIfOvr9M= From 7372984bbac610633357dca087481e82e433ffaa Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Wed, 5 Nov 2025 14:49:33 +0100 Subject: [PATCH 042/176] [GEP-28] Unify `gardenadm` node directories (#13357) --- pkg/gardenadm/cmd/options_manifests.go | 4 +++- test/e2e/gardenadm/unmanagedinfra/gardenadm.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/gardenadm/cmd/options_manifests.go b/pkg/gardenadm/cmd/options_manifests.go index 870cac8fb36..8a66ba23112 100644 --- a/pkg/gardenadm/cmd/options_manifests.go +++ b/pkg/gardenadm/cmd/options_manifests.go @@ -8,6 +8,8 @@ import ( "fmt" "github.com/spf13/pflag" + + "github.com/gardener/gardener/pkg/gardenadm/botanist" ) // ManifestOptions contains options related to handling the manifest files. @@ -40,4 +42,4 @@ func (o *ManifestOptions) AddFlags(fs *pflag.FlagSet) { } // ConfigDirLocation is the location where `gardenadm init` stores the config directory path. -const ConfigDirLocation = "/etc/gardenadm/config-directory" +const ConfigDirLocation = botanist.GardenadmBaseDir + "/config-directory" diff --git a/test/e2e/gardenadm/unmanagedinfra/gardenadm.go b/test/e2e/gardenadm/unmanagedinfra/gardenadm.go index 5d855321a9d..fb165b7f1ef 100644 --- a/test/e2e/gardenadm/unmanagedinfra/gardenadm.go +++ b/test/e2e/gardenadm/unmanagedinfra/gardenadm.go @@ -200,7 +200,7 @@ var _ = Describe("gardenadm unmanaged infrastructure scenario tests", Label("gar It("should ensure that the config dir location has been stored in the well-known location", func(ctx SpecContext) { Eventually(ctx, func(g Gomega) string { - stdOut, _, err := execute(ctx, 0, "cat", "/etc/gardenadm/config-directory") + stdOut, _, err := execute(ctx, 0, "cat", "/var/lib/gardenadm/config-directory") g.Expect(err).NotTo(HaveOccurred()) return string(stdOut.Contents()) }).Should(Equal(configDirectory)) From f4d12a02a5fe2da94c5a4b1fad1e6ff7dc4f3107 Mon Sep 17 00:00:00 2001 From: Tobias Schlicht <155947440+tobschli@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:39:20 +0100 Subject: [PATCH 043/176] Non-namespaced resource limits (#12916) * Add optional `Count` parameter to resource limit webhook * Adapt components, tests and conversions to take the, now optional `size` limit * Adapt resourcesize handler to also handle `count` limit * Add tests for `count` limit handling * Add validation for `count` field * Add tests for `count` validation * --- SEPARATOR --- * Adapt `admission-controller` `ConfigMap` template to not add keys, when values are empty. Before this, when not specifying e.g. `unrestrictedSubjects`, the key would be added to the YAML, leading to invalid YAML * Adapt documentation * Limit the list to `countLimit + 1` This also included some refactorings, as we did not return an error when the evaluation of the current count of objects fails, to prevent this feature from blocking operations on the objects. * Add logging to hint that limits are only enforced for non-namespaced resources * Fix test by adding rest mapper * Implement PR feedback * Implement PR feedback * Implement PR feedback * Adapt API description of `Count` filed Co-authored-by: Tim Usner * Fix wording Co-authored-by: Rafael Franzke * Inline variables that are only used once Co-authored-by: Rafael Franzke --------- Co-authored-by: Tim Usner Co-authored-by: Rafael Franzke --- .../configmap-componentconfig.yaml | 4 + charts/gardener/controlplane/values.yaml | 4 + .../operator/templates/crd-gardens.yaml | 7 +- docs/api-reference/operator.md | 13 ++ docs/concepts/admission-controller.md | 9 ++ ...tconfig-gardener-admission-controller.yaml | 4 + ...0-crd-operator.gardener.cloud_gardens.yaml | 7 +- example/operator/20-garden.yaml | 4 + .../apis/config/v1alpha1/types.go | 8 +- .../config/v1alpha1/validation/validation.go | 11 +- .../v1alpha1/validation/validation_test.go | 27 ++-- .../config/v1alpha1/zz_generated.deepcopy.go | 11 +- pkg/admissioncontroller/webhook/add.go | 6 +- .../webhook/admission/resourcesize/handler.go | 129 ++++++++++++++++-- .../admission/resourcesize/handler_test.go | 98 +++++++++---- .../v1alpha1/conversion/conversion.go | 1 + .../v1alpha1/conversion/conversion_test.go | 5 +- pkg/apis/operator/v1alpha1/types_garden.go | 6 +- .../v1alpha1/validation/garden_test.go | 2 +- .../v1alpha1/zz_generated.deepcopy.go | 11 +- .../admission_controller_test.go | 4 +- 21 files changed, 307 insertions(+), 64 deletions(-) diff --git a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/configmap-componentconfig.yaml b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/configmap-componentconfig.yaml index 829c93bb420..14015534b56 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/configmap-componentconfig.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/configmap-componentconfig.yaml @@ -52,10 +52,14 @@ data: port: {{ required ".Values.global.admission.config.server.metrics.port is required" .Values.global.admission.config.server.metrics.port }} {{- if .Values.global.admission.config.server.resourceAdmissionConfiguration }} resourceAdmissionConfiguration: + {{- if .Values.global.admission.config.server.resourceAdmissionConfiguration.limits }} limits: {{ toYaml .Values.global.admission.config.server.resourceAdmissionConfiguration.limits | indent 8 }} + {{- end }} + {{- if .Values.global.admission.config.server.resourceAdmissionConfiguration.unrestrictedSubjects }} unrestrictedSubjects: {{ toYaml .Values.global.admission.config.server.resourceAdmissionConfiguration.unrestrictedSubjects | indent 8 }} + {{- end }} operationMode: {{ required ".Values.global.admission.config.server.resourceAdmissionConfiguration.operationMode is required" .Values.global.admission.config.server.resourceAdmissionConfiguration.operationMode }} {{- end }} enableDebugHandlers: {{ .Values.global.admission.config.server.enableDebugHandlers }} diff --git a/charts/gardener/controlplane/values.yaml b/charts/gardener/controlplane/values.yaml index 3801f89fe31..951c857227c 100644 --- a/charts/gardener/controlplane/values.yaml +++ b/charts/gardener/controlplane/values.yaml @@ -313,6 +313,10 @@ global: # apiVersions: ["*"] # resources: ["shoots"] # size: 100Ki + # - apiGroups: ["core.gardener.cloud"] + # apiVersions: ["*"] + # resources: ["projects"] + # count: 200 # unrestrictedSubjects: # - kind: Group # name: gardener.cloud:system:seeds diff --git a/charts/gardener/operator/templates/crd-gardens.yaml b/charts/gardener/operator/templates/crd-gardens.yaml index 1892a6751c3..7d31146cc7c 100644 --- a/charts/gardener/operator/templates/crd-gardens.yaml +++ b/charts/gardener/operator/templates/crd-gardens.yaml @@ -800,6 +800,12 @@ spec: items: type: string type: array + count: + description: Count specifies the maximum number + of resources of the given kind. Only cluster-scoped + resources are considered. + format: int64 + type: integer resources: description: Resources is the name of the resource this rule applies to. WildcardAll represents @@ -816,7 +822,6 @@ spec: x-kubernetes-int-or-string: true required: - resources - - size type: object type: array operationMode: diff --git a/docs/api-reference/operator.md b/docs/api-reference/operator.md index cbbc54ee0cc..27573f77627 100644 --- a/docs/api-reference/operator.md +++ b/docs/api-reference/operator.md @@ -2930,9 +2930,22 @@ k8s.io/apimachinery/pkg/api/resource.Quantity +(Optional)

Size specifies the imposed limit.

+ + +count
+ +int64 + + + +(Optional) +

Count specifies the maximum number of resources of the given kind. Only cluster-scoped resources are considered.

+ +

RuntimeCluster diff --git a/docs/concepts/admission-controller.md b/docs/concepts/admission-controller.md index 07a79df6dbd..456a40b55b1 100644 --- a/docs/concepts/admission-controller.md +++ b/docs/concepts/admission-controller.md @@ -59,6 +59,10 @@ server: apiVersions: ["*"] resources: ["shoots"] size: 100k + - apiGroups: ["core.gardener.cloud"] + apiVersions: ["*"] + resources: ["projects"] + count: 200 - apiGroups: [""] apiVersions: ["v1"] resources: ["secrets"] @@ -89,6 +93,11 @@ Size limitations depend on the individual Gardener setup and choosing the wrong `resourceAdmissionConfiguration.operationMode` allows to control if a violating request is actually denied (default) or only logged. It's recommended to start with `log`, check the logs for exceeding requests, adjust the limits if necessary and finally switch to `block`. +In addition to that, it is also possible to restrict the number of **cluster-scoped** resources, using the `count` field, like in the example above. +This ensures that only a certain number of resources of the given type can exist in the cluster. +Similar to restrictions, subjects configured under `unrestrictedSubjects` are exempt from this restriction. +The count restriction can also be used in combination with size restrictions. + ### SeedRestriction Please refer to [Scoped API Access for Gardenlets](../deployment/gardenlet_api_access.md) for more information. diff --git a/example/20-componentconfig-gardener-admission-controller.yaml b/example/20-componentconfig-gardener-admission-controller.yaml index 174eb804f38..a66be369b6e 100644 --- a/example/20-componentconfig-gardener-admission-controller.yaml +++ b/example/20-componentconfig-gardener-admission-controller.yaml @@ -21,6 +21,10 @@ server: apiVersions: ["*"] resources: ["shoots"] size: 100k + - apiGroups: [ "core.gardener.cloud" ] + apiVersions: [ "*" ] + resources: [ "projects" ] + count: 200 unrestrictedSubjects: - kind: Group name: gardener.cloud:system:seeds diff --git a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml index 1892a6751c3..7d31146cc7c 100644 --- a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml +++ b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml @@ -800,6 +800,12 @@ spec: items: type: string type: array + count: + description: Count specifies the maximum number + of resources of the given kind. Only cluster-scoped + resources are considered. + format: int64 + type: integer resources: description: Resources is the name of the resource this rule applies to. WildcardAll represents @@ -816,7 +822,6 @@ spec: x-kubernetes-int-or-string: true required: - resources - - size type: object type: array operationMode: diff --git a/example/operator/20-garden.yaml b/example/operator/20-garden.yaml index eca42aa9031..829fd259531 100644 --- a/example/operator/20-garden.yaml +++ b/example/operator/20-garden.yaml @@ -250,6 +250,10 @@ spec: # apiVersions: ["*"] # resources: ["shoots"] # size: 100k + # - apiGroups: ["core.gardener.cloud"] + # apiVersions: ["*"] + # resources: ["projects"] + # count: 200 # unrestrictedSubjects: # - kind: Group # name: gardener.cloud:system:seeds diff --git a/pkg/admissioncontroller/apis/config/v1alpha1/types.go b/pkg/admissioncontroller/apis/config/v1alpha1/types.go index 447442079a8..172fdff33fa 100644 --- a/pkg/admissioncontroller/apis/config/v1alpha1/types.go +++ b/pkg/admissioncontroller/apis/config/v1alpha1/types.go @@ -69,7 +69,7 @@ type ResourceAdmissionWebhookMode string // WildcardAll is a character which represents all elements in a set. const WildcardAll = "*" -// ResourceLimit contains settings about a kind and the size each resource should have at most. +// ResourceLimit contains settings about a kind and the size as well as count each resource should have at most. type ResourceLimit struct { // APIGroups is the name of the APIGroup that contains the limited resource. WildcardAll represents all groups. // +optional @@ -80,7 +80,11 @@ type ResourceLimit struct { // Resources is the name of the resource this rule applies to. WildcardAll represents all resources. Resources []string `json:"resources"` // Size specifies the imposed limit. - Size resource.Quantity `json:"size"` + // +optional + Size *resource.Quantity `json:"size,omitempty"` + // Count specifies the maximum number of resources of the given kind. Only cluster-scoped resources are considered. + // +optional + Count *int64 `json:"count,omitempty"` } // Server contains information for HTTP(S) server configuration. diff --git a/pkg/admissioncontroller/apis/config/v1alpha1/validation/validation.go b/pkg/admissioncontroller/apis/config/v1alpha1/validation/validation.go index 64175d6ae32..953ba821421 100644 --- a/pkg/admissioncontroller/apis/config/v1alpha1/validation/validation.go +++ b/pkg/admissioncontroller/apis/config/v1alpha1/validation/validation.go @@ -7,6 +7,7 @@ package validation import ( rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" + apivalidation "k8s.io/apimachinery/pkg/api/validation" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" @@ -112,7 +113,15 @@ func ValidateResourceAdmissionConfiguration(config *admissioncontrollerconfigv1a allErrs = append(allErrs, field.Invalid(fld.Child("versions"), limit.Resources, "must at least have one element")) } - if limit.Size.Cmp(resource.Quantity{}) < 0 { + if limit.Count == nil && limit.Size == nil { + allErrs = append(allErrs, field.Invalid(fld, limit, "at least one of size or count must be specified")) + } + + if limit.Count != nil && apivalidation.ValidateNonnegativeField(*limit.Count, fld.Child("count")).ToAggregate() != nil { + allErrs = append(allErrs, field.Invalid(fld.Child("count"), limit.Count, "value must not be negative")) + } + + if limit.Size != nil && limit.Size.Cmp(resource.Quantity{}) < 0 { allErrs = append(allErrs, field.Invalid(fld.Child("size"), limit.Size.String(), "value must not be negative")) } } diff --git a/pkg/admissioncontroller/apis/config/v1alpha1/validation/validation_test.go b/pkg/admissioncontroller/apis/config/v1alpha1/validation/validation_test.go index 6b927023f04..7a0837909a4 100644 --- a/pkg/admissioncontroller/apis/config/v1alpha1/validation/validation_test.go +++ b/pkg/admissioncontroller/apis/config/v1alpha1/validation/validation_test.go @@ -95,10 +95,11 @@ var _ = Describe("#ValidateAdmissionControllerConfiguration", func() { versions = []string{"v1beta1", "v1alpha1"} resources = []string{"shoot"} size = "1Ki" + count = int64(100) ) DescribeTable("Limits validation", - func(apiGroups []string, versions []string, resources []string, size string, matcher gomegatypes.GomegaMatcher) { + func(apiGroups []string, versions []string, resources []string, size string, count int64, matcher gomegatypes.GomegaMatcher) { s, err := resource.ParseQuantity(size) utilruntime.Must(err) config := &admissioncontrollerconfigv1alpha1.AdmissionControllerConfiguration{ @@ -111,7 +112,8 @@ var _ = Describe("#ValidateAdmissionControllerConfiguration", func() { APIGroups: apiGroups, APIVersions: versions, Resources: resources, - Size: s, + Size: &s, + Count: &count, }, }, }, @@ -123,36 +125,39 @@ var _ = Describe("#ValidateAdmissionControllerConfiguration", func() { Expect(errs).To(matcher) }, - Entry("should allow request", apiGroups, versions, resources, size, + Entry("should allow request", apiGroups, versions, resources, size, count, BeEmpty(), ), - Entry("should deny empty apiGroup", nil, versions, resources, size, + Entry("should deny empty apiGroup", nil, versions, resources, size, count, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{"Field": Equal("server.resourceAdmissionConfiguration.limits[0].apiGroups")}))), ), - Entry("should allow apiGroup w/ zero length", []string{""}, versions, resources, size, + Entry("should allow apiGroup w/ zero length", []string{""}, versions, resources, size, count, BeEmpty(), ), - Entry("should deny empty versions", apiGroups, nil, resources, size, + Entry("should deny empty versions", apiGroups, nil, resources, size, count, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{"Field": Equal("server.resourceAdmissionConfiguration.limits[0].versions")}))), ), - Entry("should deny versions w/ zero length", apiGroups, []string{""}, resources, size, + Entry("should deny versions w/ zero length", apiGroups, []string{""}, resources, size, count, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{"Field": Equal("server.resourceAdmissionConfiguration.limits[0].versions[0]")}))), ), - Entry("should deny empty resources", apiGroups, versions, nil, size, + Entry("should deny empty resources", apiGroups, versions, nil, size, count, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{"Field": Equal("server.resourceAdmissionConfiguration.limits[0].resources")}))), ), - Entry("should deny resources w/ zero length", apiGroups, versions, []string{""}, size, + Entry("should deny resources w/ zero length", apiGroups, versions, []string{""}, size, count, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{"Field": Equal("server.resourceAdmissionConfiguration.limits[0].resources[0]")}))), ), - Entry("should deny invalid size", apiGroups, versions, resources, "-1k", + Entry("should deny invalid size", apiGroups, versions, resources, "-1k", count, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{"Field": Equal("server.resourceAdmissionConfiguration.limits[0].size")}))), ), - Entry("should deny invalid size and resources w/ zero length", apiGroups, versions, []string{resources[0], ""}, "-1k", + Entry("should deny invalid size and resources w/ zero length", apiGroups, versions, []string{resources[0], ""}, "-1k", count, ConsistOf( PointTo(MatchFields(IgnoreExtras, Fields{"Field": Equal("server.resourceAdmissionConfiguration.limits[0].size")})), PointTo(MatchFields(IgnoreExtras, Fields{"Field": Equal("server.resourceAdmissionConfiguration.limits[0].resources[1]")})), ), ), + Entry("should deny invalid count", apiGroups, versions, resources, size, int64(-1), + ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{"Field": Equal("server.resourceAdmissionConfiguration.limits[0].count")}))), + ), ) var ( diff --git a/pkg/admissioncontroller/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/admissioncontroller/apis/config/v1alpha1/zz_generated.deepcopy.go index 9c102649f74..94a461b23f3 100644 --- a/pkg/admissioncontroller/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/admissioncontroller/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -116,7 +116,16 @@ func (in *ResourceLimit) DeepCopyInto(out *ResourceLimit) { *out = make([]string, len(*in)) copy(*out, *in) } - out.Size = in.Size.DeepCopy() + if in.Size != nil { + in, out := &in.Size, &out.Size + x := (*in).DeepCopy() + *out = &x + } + if in.Count != nil { + in, out := &in.Count, &out.Count + *out = new(int64) + **out = **in + } return } diff --git a/pkg/admissioncontroller/webhook/add.go b/pkg/admissioncontroller/webhook/add.go index 941e141e7fa..50b563ec153 100644 --- a/pkg/admissioncontroller/webhook/add.go +++ b/pkg/admissioncontroller/webhook/add.go @@ -82,8 +82,10 @@ func AddToManager( } if err := (&resourcesize.Handler{ - Logger: mgr.GetLogger().WithName("webhook").WithName(resourcesize.HandlerName), - Config: cfg.Server.ResourceAdmissionConfiguration, + Logger: mgr.GetLogger().WithName("webhook").WithName(resourcesize.HandlerName), + Config: cfg.Server.ResourceAdmissionConfiguration, + APIReader: mgr.GetAPIReader(), + RESTMapper: mgr.GetRESTMapper(), }).AddToManager(mgr); err != nil { return fmt.Errorf("failed adding %s webhook handler: %w", resourcesize.HandlerName, err) } diff --git a/pkg/admissioncontroller/webhook/admission/resourcesize/handler.go b/pkg/admissioncontroller/webhook/admission/resourcesize/handler.go index 0f060c85211..b96a0cef908 100644 --- a/pkg/admissioncontroller/webhook/admission/resourcesize/handler.go +++ b/pkg/admissioncontroller/webhook/admission/resourcesize/handler.go @@ -10,17 +10,21 @@ import ( "fmt" "net/http" "strings" + "time" "github.com/go-logr/logr" admissionv1 "k8s.io/api/admission/v1" authenticationv1 "k8s.io/api/authentication/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apiserver/pkg/authentication/serviceaccount" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" admissioncontrollerconfigv1alpha1 "github.com/gardener/gardener/pkg/admissioncontroller/apis/config/v1alpha1" @@ -28,24 +32,31 @@ import ( "github.com/gardener/gardener/pkg/admissioncontroller/metrics" ) -// metricReasonSizeExceeded is a metric reason value for a reason when an object size was exceeded. -const metricReasonSizeExceeded = "Size Exceeded" +const ( + // metricReasonSizeExceeded is a metric reason value for a reason when an object size was exceeded. + metricReasonSizeExceeded = "Size Exceeded" + + // metricReasonCountExceeded is a metric reason value for a reason when resource count was exceeded. + metricReasonCountExceeded = "Count Exceeded" +) // Handler checks the resource sizes. type Handler struct { - Logger logr.Logger - Config *admissioncontrollerconfigv1alpha1.ResourceAdmissionConfiguration + Logger logr.Logger + Config *admissioncontrollerconfigv1alpha1.ResourceAdmissionConfiguration + APIReader client.Reader + RESTMapper meta.RESTMapper } // Handle checks the resource sizes. -func (h *Handler) Handle(_ context.Context, req admission.Request) admission.Response { +func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { var err error switch req.Operation { case admissionv1.Create: - err = h.handle(req) + err = h.handle(ctx, req) case admissionv1.Update: - err = h.handle(req) + err = h.handle(ctx, req) default: return admission.Errored(http.StatusBadRequest, fmt.Errorf("unknown operation request %q", req.Operation)) } @@ -62,7 +73,7 @@ func (h *Handler) Handle(_ context.Context, req admission.Request) admission.Res return admission.Allowed("") } -func (h *Handler) handle(req admission.Request) error { +func (h *Handler) handle(ctx context.Context, req admission.Request) error { log := h.Logger.WithValues("user", req.UserInfo.Username, "resource", req.Resource, "name", req.Name) if req.Namespace != "" { log = log.WithValues("namespace", req.Namespace) @@ -78,11 +89,29 @@ func (h *Handler) handle(req admission.Request) error { requestedResource = req.RequestResource } - limit := findLimitForGVR(h.Config.Limits, requestedResource) - if limit == nil { + limit, count := findRestrictionsForGVR(h.Config.Limits, requestedResource) + if limit == nil && count == nil { return nil } + // Handle resource count limits + if count != nil { + if err := h.handleCountLimit(ctx, req, log, requestedResource, count); err != nil { + return fmt.Errorf("error handling count limit: %w", err) + } + } + + // Handle resource size limits + if limit != nil { + if err := h.handleSizeLimit(req, log, limit); err != nil { + return fmt.Errorf("error handling size limit: %w", err) + } + } + + return nil +} + +func (h *Handler) handleSizeLimit(req admission.Request, log logr.Logger, limit *resource.Quantity) error { objectSize, err := relevantObjectSize(req.Object.Raw) if err != nil { return err @@ -105,6 +134,43 @@ func (h *Handler) handle(req admission.Request) error { return nil } +func (h *Handler) handleCountLimit(ctx context.Context, req admission.Request, log logr.Logger, requestedResource *metav1.GroupVersionResource, count *int64) error { + if req.Namespace != "" { + // We only want to restrict non-namespaced resources + // namespaced resources can be restricted by ResourceQuotas + log.Info("Skipping count check for namespaced resource", "resource", requestedResource.String()) + return nil + } + + // Only count for CREATE operations, not UPDATE + if req.Operation != admissionv1.Create { + return nil + } + + exceedsLimit, err := h.existingResourcesExceedLimit(ctx, req, requestedResource, ptr.Deref(count, 0)) + if err != nil { + return fmt.Errorf("failed to determine if count exceeds limit: %w", err) + } + + if !exceedsLimit { + return nil + } + + if h.Config.OperationMode == nil || *h.Config.OperationMode == admissioncontrollerconfigv1alpha1.AdmissionModeBlock { + log.Info("Maximum resource count exceeded, rejected request", "limit", *count) + metrics.RejectedResources.WithLabelValues( + fmt.Sprint(req.Operation), + req.Kind.Kind, + req.Namespace, + metricReasonCountExceeded, + ).Inc() + return apierrors.NewForbidden(schema.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource}, req.Name, fmt.Errorf("maximum resource count exceeded! max allowed: %d", ptr.Deref(count, 0))) + } + + log.Info("Maximum resource count exceeded, request would be denied in blocking mode", "limit", ptr.Deref(count, 0)) + return nil +} + func relevantObjectSize(rawObject []byte) (int64, error) { var obj map[string]any err := json.Unmarshal(rawObject, &obj) @@ -155,14 +221,49 @@ func isUnrestrictedUser(userInfo authenticationv1.UserInfo, subjects []rbacv1.Su return userMatch(userInfo, subjects) } -func findLimitForGVR(limits []admissioncontrollerconfigv1alpha1.ResourceLimit, gvr *metav1.GroupVersionResource) *resource.Quantity { +func findRestrictionsForGVR(limits []admissioncontrollerconfigv1alpha1.ResourceLimit, gvr *metav1.GroupVersionResource) (*resource.Quantity, *int64) { for _, limit := range limits { - size := limit.Size if admissioncontrollerhelper.APIGroupMatches(limit, gvr.Group) && admissioncontrollerhelper.VersionMatches(limit, gvr.Version) && admissioncontrollerhelper.ResourceMatches(limit, gvr.Resource) { - return &size + return limit.Size, limit.Count } } - return nil + return nil, nil +} + +func (h *Handler) getKindFromGVR(gvr *metav1.GroupVersionResource) (string, error) { + gvk, err := h.RESTMapper.KindFor(schema.GroupVersionResource{ + Group: gvr.Group, + Version: gvr.Version, + Resource: gvr.Resource, + }) + if err != nil { + return "", fmt.Errorf("failed to get Kind for GVR %s: %w", gvr.String(), err) + } + return gvk.Kind, nil +} + +func (h *Handler) existingResourcesExceedLimit(ctx context.Context, _ admission.Request, gvr *metav1.GroupVersionResource, limit int64) (bool, error) { + timeoutContext, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + // Get the proper Kind for this resource + kind, err := h.getKindFromGVR(gvr) + if err != nil { + return false, fmt.Errorf("failed to list resources for calculating object count: %w", err) + } + + list := &metav1.PartialObjectMetadataList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvr.Group, + Version: gvr.Version, + Kind: kind + "List", + }) + // List all resources of this type cluster-wide (since we only count non-namespaced resources) + if err := h.APIReader.List(timeoutContext, list, client.Limit(limit+1)); err != nil { + return false, fmt.Errorf("failed to list resources for counting: %v", err) + } + + return int64(len(list.Items)) >= limit, nil } diff --git a/pkg/admissioncontroller/webhook/admission/resourcesize/handler_test.go b/pkg/admissioncontroller/webhook/admission/resourcesize/handler_test.go index 2e59deee47c..f4ae7ce799b 100644 --- a/pkg/admissioncontroller/webhook/admission/resourcesize/handler_test.go +++ b/pkg/admissioncontroller/webhook/admission/resourcesize/handler_test.go @@ -22,15 +22,18 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apiserver/pkg/authentication/serviceaccount" "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" logzap "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" admissioncontrollerconfigv1alpha1 "github.com/gardener/gardener/pkg/admissioncontroller/apis/config/v1alpha1" . "github.com/gardener/gardener/pkg/admissioncontroller/webhook/admission/resourcesize" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/logger" ) @@ -45,12 +48,13 @@ var _ = Describe("handler", func() { logBuffer *gbytes.Buffer testEncoder runtime.Encoder - projectsSizeLimit, _ = resource.ParseQuantity("0M") - secretSizeLimit, _ = resource.ParseQuantity("1Mi") + seedCountLimit = int64(0) + projectsSizeLimit = resource.MustParse("0M") + secretSizeLimit = resource.MustParse("1Mi") // size of shoot w/ namespace, name, w/o spec - shootsv1beta1SizeLimit, _ = resource.ParseQuantity("342") + shootsv1beta1SizeLimit = resource.MustParse("342") // size of shoot w/ namespace, name, w/o spec -1 byte - shootsv1alpha1SizeLimit, _ = resource.ParseQuantity("342") + shootsv1alpha1SizeLimit = resource.MustParse("342") restrictedUserName = "restrictedUser" unrestrictedUserName = "unrestrictedUser" @@ -82,25 +86,31 @@ var _ = Describe("handler", func() { APIGroups: []string{"*"}, APIVersions: []string{"*"}, Resources: []string{"projects"}, - Size: projectsSizeLimit, + Size: &projectsSizeLimit, }, { APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"secrets"}, - Size: secretSizeLimit, + Size: &secretSizeLimit, }, { APIGroups: []string{"core.gardener.cloud"}, APIVersions: []string{"v1beta1"}, Resources: []string{"shoots"}, - Size: shootsv1beta1SizeLimit, + Size: &shootsv1beta1SizeLimit, }, { APIGroups: []string{"core.gardener.cloud"}, APIVersions: []string{"v1alpha1"}, Resources: []string{"shoots"}, - Size: shootsv1alpha1SizeLimit, + Size: &shootsv1alpha1SizeLimit, + }, + { + APIGroups: []string{"core.gardener.cloud"}, + APIVersions: []string{"v1beta1"}, + Resources: []string{"seeds"}, + Count: &seedCountLimit, }, }, } @@ -153,6 +163,18 @@ var _ = Describe("handler", func() { } } + seed = func() runtime.Object { + return &gardencorev1beta1.Seed{ + TypeMeta: metav1.TypeMeta{ + Kind: "Seed", + APIVersion: gardencorev1beta1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-seed", + }, + } + } + unrestrictedUser = func() authenticationv1.UserInfo { return authenticationv1.UserInfo{ Username: unrestrictedUserName, @@ -193,14 +215,34 @@ var _ = Describe("handler", func() { logBuffer = gbytes.NewBuffer() log = logger.MustNewZapLogger(logger.DebugLevel, logger.FormatJSON, logzap.WriteTo(io.MultiWriter(GinkgoWriter, logBuffer)), logzap.Level(zapcore.Level(0))) - handler = &Handler{Logger: log, Config: config()} + // Create a REST mapper that includes Gardener resources + restMapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{ + corev1.SchemeGroupVersion, + gardencorev1beta1.SchemeGroupVersion, + }) + + // Add Seed resource mappings + restMapper.Add(schema.GroupVersionKind{ + Group: "core.gardener.cloud", + Version: "v1beta1", + Kind: "Seed", + }, meta.RESTScopeRoot) + + fakeClient := fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).WithRESTMapper(restMapper).Build() + handler = &Handler{ + Logger: log, + Config: config(), + APIReader: fakeClient, + RESTMapper: fakeClient.RESTMapper(), + } testEncoder = &json.Serializer{} request = admission.Request{} - request.Operation = admissionv1.Update }) - test := func(objFn func() runtime.Object, userFn func() authenticationv1.UserInfo, expectedAllowed bool) { + test := func(objFn func() runtime.Object, userFn func() authenticationv1.UserInfo, requestOperation admissionv1.Operation, expectedAllowed bool) { + request.Operation = requestOperation + if obj := objFn(); obj != nil { objData, err := runtime.Encode(testEncoder, obj) Expect(err).NotTo(HaveOccurred()) @@ -237,12 +279,12 @@ var _ = Describe("handler", func() { Context("ignored requests", func() { It("empty resource", func() { - test(empty, restrictedUser, true) + test(empty, restrictedUser, admissionv1.Update, true) }) }) It("should pass because size is in range for v1beta1 shoot", func() { - test(shootv1beta1, restrictedUser, true) + test(shootv1beta1, restrictedUser, admissionv1.Update, true) }) It("should pass because size is in range for v1beta1 shoot without considering status", func() { @@ -267,7 +309,7 @@ var _ = Describe("handler", func() { Expect(shootsv1beta1SizeLimit.CmpInt64(int64(len(objData)))).Should(Equal(-1)) return shootWithLargeStatus } - test(largeShoot, restrictedUser, true) + test(largeShoot, restrictedUser, admissionv1.Update, true) }) It("should pass because size is in range for v1beta1 shoot without considering managed fields", func() { @@ -289,42 +331,50 @@ var _ = Describe("handler", func() { return shootWithLargeStatus } - test(largeShoot, restrictedUser, true) + test(largeShoot, restrictedUser, admissionv1.Update, true) }) It("should pass because of unrestricted user", func() { - test(shootv1beta1, unrestrictedUser, true) + test(shootv1beta1, unrestrictedUser, admissionv1.Update, true) }) It("should pass because of unrestricted group", func() { - test(shootv1beta1, unrestrictedGroup, true) + test(shootv1beta1, unrestrictedGroup, admissionv1.Update, true) }) It("should pass because size is in range for secret", func() { - test(secret, restrictedUser, true) + test(secret, restrictedUser, admissionv1.Update, true) }) It("should pass because no limits configured for configMaps", func() { - test(configMap, restrictedUser, true) + test(configMap, restrictedUser, admissionv1.Update, true) }) It("should fail because size is not in range for project", func() { - test(project, restrictedUser, false) + test(project, restrictedUser, admissionv1.Update, false) }) It("should pass because of unrestricted user", func() { - test(project, unrestrictedUser, true) + test(project, unrestrictedUser, admissionv1.Update, true) }) It("should pass because of unrestricted group", func() { - test(project, unrestrictedGroup, true) + test(project, unrestrictedGroup, admissionv1.Update, true) }) It("should pass because of unrestricted service account", func() { - test(project, unrestrictedServiceAccount, true) + test(project, unrestrictedServiceAccount, admissionv1.Update, true) }) It("should fail because of restricted service account", func() { - test(project, restrictedServiceAccount, false) + test(project, restrictedServiceAccount, admissionv1.Update, false) + }) + + It("should fail because of count limit of seeds", func() { + test(seed, restrictedUser, admissionv1.Create, false) + }) + + It("should pass seed creation despite count limit being reached when user has unrestricted privileges", func() { + test(seed, unrestrictedServiceAccount, admissionv1.Create, true) }) }) diff --git a/pkg/apis/operator/v1alpha1/conversion/conversion.go b/pkg/apis/operator/v1alpha1/conversion/conversion.go index 7485e8085c2..9b0fe2f05b3 100644 --- a/pkg/apis/operator/v1alpha1/conversion/conversion.go +++ b/pkg/apis/operator/v1alpha1/conversion/conversion.go @@ -32,6 +32,7 @@ func ConvertToAdmissionControllerResourceAdmissionConfiguration(config *operator APIVersions: limit.APIVersions, Resources: limit.Resources, Size: limit.Size, + Count: limit.Count, }) } diff --git a/pkg/apis/operator/v1alpha1/conversion/conversion_test.go b/pkg/apis/operator/v1alpha1/conversion/conversion_test.go index eacf95c7330..b2af636e255 100644 --- a/pkg/apis/operator/v1alpha1/conversion/conversion_test.go +++ b/pkg/apis/operator/v1alpha1/conversion/conversion_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/gomega/gstruct" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" admissioncontrollerconfigv1alpha1 "github.com/gardener/gardener/pkg/admissioncontroller/apis/config/v1alpha1" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" @@ -33,13 +34,13 @@ var _ = Describe("conversion", func() { APIVersions: []string{"v1beta1"}, APIGroups: []string{"core.gardener.cloud"}, Resources: []string{"shoots"}, - Size: resource.MustParse("1Ki"), + Size: ptr.To(resource.MustParse("1Ki")), }, { APIVersions: []string{"v1"}, APIGroups: []string{""}, Resources: []string{"secrets", "configmaps"}, - Size: resource.MustParse("100Ki"), + Size: ptr.To(resource.MustParse("100Ki")), }, }, UnrestrictedSubjects: []rbacv1.Subject{}, diff --git a/pkg/apis/operator/v1alpha1/types_garden.go b/pkg/apis/operator/v1alpha1/types_garden.go index d9bb48d72bf..bb90a3b81ac 100644 --- a/pkg/apis/operator/v1alpha1/types_garden.go +++ b/pkg/apis/operator/v1alpha1/types_garden.go @@ -568,7 +568,11 @@ type ResourceLimit struct { // Resources is the name of the resource this rule applies to. WildcardAll represents all resources. Resources []string `json:"resources"` // Size specifies the imposed limit. - Size resource.Quantity `json:"size"` + // +optional + Size *resource.Quantity `json:"size"` + // Count specifies the maximum number of resources of the given kind. Only cluster-scoped resources are considered. + // +optional + Count *int64 `json:"count,omitempty"` } // GardenerControllerManagerConfig contains configuration settings for the gardener-controller-manager. diff --git a/pkg/apis/operator/v1alpha1/validation/garden_test.go b/pkg/apis/operator/v1alpha1/validation/garden_test.go index db8961eda02..8ff23745eb2 100644 --- a/pkg/apis/operator/v1alpha1/validation/garden_test.go +++ b/pkg/apis/operator/v1alpha1/validation/garden_test.go @@ -2130,7 +2130,7 @@ var _ = Describe("Validation Tests", func() { APIGroups: apiGroups, APIVersions: versions, Resources: resources, - Size: s, + Size: ptr.To(s), }, }, }, diff --git a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index 0dca64a1877..df655a86277 100644 --- a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -1465,7 +1465,16 @@ func (in *ResourceLimit) DeepCopyInto(out *ResourceLimit) { *out = make([]string, len(*in)) copy(*out, *in) } - out.Size = in.Size.DeepCopy() + if in.Size != nil { + in, out := &in.Size, &out.Size + x := (*in).DeepCopy() + *out = &x + } + if in.Count != nil { + in, out := &in.Count, &out.Count + *out = new(int64) + **out = **in + } return } diff --git a/pkg/component/gardener/admissioncontroller/admission_controller_test.go b/pkg/component/gardener/admissioncontroller/admission_controller_test.go index 6d7fe49904c..c593d8f9392 100644 --- a/pkg/component/gardener/admissioncontroller/admission_controller_test.go +++ b/pkg/component/gardener/admissioncontroller/admission_controller_test.go @@ -103,13 +103,13 @@ var _ = Describe("GardenerAdmissionController", func() { APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"secrets", "configmaps"}, - Size: resource.MustParse("1Mi"), + Size: ptr.To(resource.MustParse("1Mi")), }, { APIGroups: []string{"core.gardener.cloud"}, APIVersions: []string{"v1beta1"}, Resources: []string{"shoots"}, - Size: resource.MustParse("100Ki"), + Size: ptr.To(resource.MustParse("100Ki")), }, }, UnrestrictedSubjects: []rbacv1.Subject{{ From 4ac95434a77ee6798969ebea0cdc970a9ebe3c2e Mon Sep 17 00:00:00 2001 From: Niki Dokovski Date: Wed, 5 Nov 2025 16:39:28 +0100 Subject: [PATCH 044/176] Update logging stack (#13358) * fluent-bit: bump up fluent-bit to 4.1.1 * logging: bump up fluent-bit plugin to v0.68.0 --- imagevector/containers.yaml | 12 ++++++------ .../observability/logging/fluentbit/fluentbit.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index eb2bd901913..4d2cb5a9054 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -577,7 +577,7 @@ images: - name: fluent-bit sourceRepository: github.com/fluent/fluent-operator repository: europe-docker.pkg.dev/gardener-project/releases/3rd/fluent-operator/fluent-bit - tag: "v4.0.9" + tag: "v4.1.1" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -596,7 +596,7 @@ images: name: fluent-bit-to-vali sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/fluent-bit-to-vali - tag: "v0.67.0" + tag: "v0.68.0" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -631,7 +631,7 @@ images: - name: vali-curator sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/vali-curator - tag: "v0.67.0" + tag: "v0.68.0" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -687,7 +687,7 @@ images: name: telegraf-iptables sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/telegraf-iptables - tag: "v0.67.0" + tag: "v0.68.0" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -707,7 +707,7 @@ images: - name: event-logger sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/event-logger - tag: "v0.67.0" + tag: "v0.68.0" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -724,7 +724,7 @@ images: - name: tune2fs sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/tune2fs - tag: "v0.67.0" + tag: "v0.68.0" labels: - name: 'gardener.cloud/cve-categorisation' value: diff --git a/pkg/component/observability/logging/fluentbit/fluentbit.go b/pkg/component/observability/logging/fluentbit/fluentbit.go index c59893a430a..00d86826d76 100644 --- a/pkg/component/observability/logging/fluentbit/fluentbit.go +++ b/pkg/component/observability/logging/fluentbit/fluentbit.go @@ -367,7 +367,7 @@ func (f *fluentBit) getFluentBit() *fluentbitv1alpha2.FluentBit { Command: []string{ "/fluent-bit/bin/fluent-bit-watcher", "-e", - "/fluent-bit/plugins/out_vali.so", + "/fluent-bit/plugins/output_plugin.so", "-c", "/fluent-bit/config/fluent-bit.conf", }, From 2d3c9d9ce5d1fe84737eebc8fb41d04ff8ebdeda Mon Sep 17 00:00:00 2001 From: rhubenov Date: Wed, 5 Nov 2025 20:00:19 +0200 Subject: [PATCH 045/176] Worker pool rollout v2 (#12829) * Add new status field to internal and external Shoot API Co-authored-by: Jeremy Rickards Co-authored-by: Gerrit Schwerthelm * Validate annotation Co-authored-by: Jeremy Rickards Co-authored-by: Gerrit Schwerthel * Add logic for increasing generation & not removing the operation in gardenapi Co-authored-by: Jeremy Rickards Co-authored-by: Gerrit Schwerthelm * Update pending workers in shoot status & attach annotation to machinedeployments Co-authored-by: Gerrit Schwerthelm Co-authored-by: Jeremy Rickards * Make sure annotation is not removed by mcm Co-authored-by: Jeremy Rickards Co-authored-by: Gerrit Schwerthelm * Add e2e test for the annotation Co-authored-by: Gerrit Schwerthelm Co-authored-by: Jeremy Rickards * Add documentation Co-authored-by: Gerrit Schwerthelm Co-authored-by: Jeremy Rickards * Make generate Co-authored-by: Gerrit Schwerthelm Co-authored-by: Jeremy Rickards * Review changes * Forbid annotation for shoots with inplace update enabled * Review changes (docs + formatting) * Include warning for the `InPlace` update strategies in docs * Add tests for forbidding annotation on `Shoot` clusters with `InPlace` strategy * Note highlight & fix note for forbidding worker pools rather than `Shoots` --------- Co-authored-by: Jeremy Rickards Co-authored-by: Gerrit Schwerthelm --- .../operator/templates/crd-gardens.yaml | 16 +- docs/README.md | 1 + docs/api-reference/core.md | 53 +- .../dual-stack-networking-migration.md | 6 + .../worker_pool_manual_rollout.md | 29 + ...0-crd-operator.gardener.cloud_gardens.yaml | 16 +- .../genericactuator/actuator_reconcile.go | 6 + pkg/apis/core/types_shoot.go | 13 +- .../core/v1beta1/constants/types_constants.go | 3 + pkg/apis/core/v1beta1/generated.pb.go | 2237 +++++++++-------- pkg/apis/core/v1beta1/generated.proto | 16 +- pkg/apis/core/v1beta1/helper/shoot.go | 14 + pkg/apis/core/v1beta1/helper/shoot_test.go | 28 + pkg/apis/core/v1beta1/types_shoot.go | 15 +- .../core/v1beta1/zz_generated.conversion.go | 32 + .../core/v1beta1/zz_generated.deepcopy.go | 28 + pkg/apis/core/validation/shoot.go | 52 +- pkg/apis/core/validation/shoot_test.go | 141 ++ pkg/apis/core/zz_generated.deepcopy.go | 28 + pkg/apiserver/openapi/api_violations.report | 1 + pkg/apiserver/openapi/openapi_generated.go | 42 +- pkg/apiserver/registry/core/shoot/strategy.go | 5 +- .../controller/shoot/shoot/reconciler.go | 71 +- .../controller/shoot/shoot/reconciler_test.go | 20 + .../gardener/shoot/create_rotate_delete.go | 78 +- 25 files changed, 1905 insertions(+), 1046 deletions(-) create mode 100644 docs/usage/shoot-operations/worker_pool_manual_rollout.md diff --git a/charts/gardener/operator/templates/crd-gardens.yaml b/charts/gardener/operator/templates/crd-gardens.yaml index 7d31146cc7c..3d85255cfbe 100644 --- a/charts/gardener/operator/templates/crd-gardens.yaml +++ b/charts/gardener/operator/templates/crd-gardens.yaml @@ -2053,13 +2053,13 @@ spec: PendingWorkersRollouts contains the name of a worker pool and the initiation time of their last rollout due to credentials rotation. items: - description: |- - PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout due to - credentials rotation. + description: PendingWorkersRollout contains the name + of a worker pool and the initiation time of their + last rollout. properties: lastInitiationTime: description: LastInitiationTime is the most recent - time when the credential rotation was initiated. + time when the worker rollout was initiated. format: date-time type: string name: @@ -2169,13 +2169,13 @@ spec: PendingWorkersRollouts contains the name of a worker pool and the initiation time of their last rollout due to credentials rotation. items: - description: |- - PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout due to - credentials rotation. + description: PendingWorkersRollout contains the name + of a worker pool and the initiation time of their + last rollout. properties: lastInitiationTime: description: LastInitiationTime is the most recent - time when the credential rotation was initiated. + time when the worker rollout was initiated. format: date-time type: string name: diff --git a/docs/README.md b/docs/README.md index ce088f725dd..cf916537e95 100644 --- a/docs/README.md +++ b/docs/README.md @@ -62,6 +62,7 @@ * [Supported Kubernetes versions](usage/shoot-operations/supported_k8s_versions.md) * [Controlling the Kubernetes versions for specific worker pools](usage/shoot-operations/worker_pool_k8s_versions.md) * [Migration from SecretBinding to CredentialsBinding](usage/shoot-operations/secretbinding-to-credentialsbinding-migration.md) +* [Manual Worker Pool Rollout](usage/shoot-operations/worker_pool_manual_rollout.md) ### High Availability diff --git a/docs/api-reference/core.md b/docs/api-reference/core.md index 370dddc5ad2..9e07bec8a6d 100644 --- a/docs/api-reference/core.md +++ b/docs/api-reference/core.md @@ -8976,6 +8976,39 @@ If not present, the value will be computed based on the “Begin” valu +

ManualWorkerPoolRollout +

+

+(Appears on: +ShootStatus) +

+

+

ManualWorkerPoolRollout contains information about the worker pool rollout progress that has been initiated via the gardener.cloud/operation=rollout-workers annotation.

+

+ + + + + + + + + + + + + +
FieldDescription
+pendingWorkersRollouts
+ + +[]PendingWorkersRollout + + +		 +(Optional) +

PendingWorkersRollouts contains the names of the worker pools that are still pending rollout.

+

MemorySwapConfiguration

@@ -9939,11 +9972,11 @@ string

(Appears on: CARotation, +ManualWorkerPoolRollout, ServiceAccountKeyRotation)

-

PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout due to -credentials rotation.

+

PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout.

@@ -9975,7 +10008,7 @@ Kubernetes meta/v1.Time @@ -13659,6 +13692,20 @@ InPlaceUpdatesStatus

InPlaceUpdates contains information about in-place updates for the Shoot workers.

+ + + +
(Optional) -

LastInitiationTime is the most recent time when the credential rotation was initiated.

+

LastInitiationTime is the most recent time when the worker rollout was initiated.
+manualWorkerPoolRollout
+ + +ManualWorkerPoolRollout + + +		 +(Optional) +

ManualWorkerPoolRollout contains information about the worker pool rollout progress.

+

ShootTemplate diff --git a/docs/usage/networking/dual-stack-networking-migration.md b/docs/usage/networking/dual-stack-networking-migration.md index bbbb6b3423e..e209b509933 100644 --- a/docs/usage/networking/dual-stack-networking-migration.md +++ b/docs/usage/networking/dual-stack-networking-migration.md @@ -48,6 +48,12 @@ spec: The migration process should usually take place during the corresponding shoot maintenance time window. If you wish to run the migration process earlier, then you need to roll the nodes yourself and then trigger a reconcile so that the status of the `DualStackNodesMigrationReady` constraint is set to `true`. Once this is the case a new reconcile needs to be triggered to update the final components as described in step 5. +To help with manual rollout, Gardener provides a way to trigger the rollout per worker pool rollout by annotating the `Shoot` with the `rollout-workers` operation annotation and specifying which pool names you'd like to be rolled out: +``` +kubectl -n annotate shoot gardener.cloud/operation=rollout-workers=[,,...] +``` +For more information, please refer to the [manual worker pool rollout](../shoot-operations/worker_pool_manual_rollout.md) documentation. + ### Step 1: Update Networking Configuration Modify the `spec.networking.ipFamilies` field to include the desired dual-stack configuration. For example, change `[IPv4]` to `[IPv4, IPv6]`. diff --git a/docs/usage/shoot-operations/worker_pool_manual_rollout.md b/docs/usage/shoot-operations/worker_pool_manual_rollout.md new file mode 100644 index 00000000000..826826f9613 --- /dev/null +++ b/docs/usage/shoot-operations/worker_pool_manual_rollout.md @@ -0,0 +1,29 @@ +# Manual Worker Pool Rollout + +> [!NOTE] +> This feature is currently only available for worker pools that are **not** configured with one of the `InPlace` update strategies - either `AutoInPlaceUpdate` or `ManualInPlaceUpdate`. + +There may be cases when an end-user might want to trigger a manual worker pool rollout. +For example, the [dual-stack migration](../networking/dual-stack-networking-migration.md) requires to roll nodes. +This can be accomplished by annotating the `Shoot` with the `rollout-workers` operation annotation and specifying which worker pools you'd like to be rolled out. + +```bash +kubectl -n annotate shoot 'gardener.cloud/operation=rollout-workers=[,,...]' +``` + +Alternatively, you can use `*` to roll out all worker pools: + +```bash +kubectl -n annotate shoot 'gardener.cloud/operation=rollout-workers=*' +``` + +This will cause the status field `manualWorkerPoolRollout` to be set on the `Shoot`. +It will keep track of the worker pools that are currently being rolled out. + +Example status field: +```yaml + manualWorkerPoolRollout: + pendingWorkersRollouts: + - lastInitiationTime: "2025-10-17T14:38:13Z" + name: local +``` diff --git a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml index 7d31146cc7c..3d85255cfbe 100644 --- a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml +++ b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml @@ -2053,13 +2053,13 @@ spec: PendingWorkersRollouts contains the name of a worker pool and the initiation time of their last rollout due to credentials rotation. items: - description: |- - PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout due to - credentials rotation. + description: PendingWorkersRollout contains the name + of a worker pool and the initiation time of their + last rollout. properties: lastInitiationTime: description: LastInitiationTime is the most recent - time when the credential rotation was initiated. + time when the worker rollout was initiated. format: date-time type: string name: @@ -2169,13 +2169,13 @@ spec: PendingWorkersRollouts contains the name of a worker pool and the initiation time of their last rollout due to credentials rotation. items: - description: |- - PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout due to - credentials rotation. + description: PendingWorkersRollout contains the name + of a worker pool and the initiation time of their + last rollout. properties: lastInitiationTime: description: LastInitiationTime is the most recent - time when the credential rotation was initiated. + time when the worker rollout was initiated. format: date-time type: string name: diff --git a/extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go b/extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go index 3a7f9f66f0d..426c4d2f395 100644 --- a/extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go +++ b/extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go @@ -281,6 +281,12 @@ func deployMachineDeployments( }, }, } + if existingMachineDeployment != nil && existingMachineDeployment.Spec.Template.Annotations != nil { + for k, v := range existingMachineDeployment.Spec.Template.Annotations { + metav1.SetMetaDataAnnotation(&machineDeployment.Spec.Template.ObjectMeta, k, v) + } + } + log.Info("Deploying machine deployment", "machineDeploymentName", machineDeployment.Name, "replicas", machineDeployment.Spec.Replicas) return nil }); err != nil { diff --git a/pkg/apis/core/types_shoot.go b/pkg/apis/core/types_shoot.go index 308284982fd..f456c544cf4 100644 --- a/pkg/apis/core/types_shoot.go +++ b/pkg/apis/core/types_shoot.go @@ -169,6 +169,8 @@ type ShootStatus struct { Networking *NetworkingStatus // InPlaceUpdates contains information about in-place updates for the Shoot workers. InPlaceUpdates *InPlaceUpdatesStatus + // ManualWorkerPoolRollout contains information about the worker pool rollout progress. + ManualWorkerPoolRollout *ManualWorkerPoolRollout } // LastMaintenance holds information about a maintenance operation on the Shoot. @@ -251,6 +253,12 @@ type CARotation struct { PendingWorkersRollouts []PendingWorkersRollout } +// ManualWorkerPoolRollout contains information about the worker pool rollout progress that has been initiated via the gardener.cloud/operation=rollout-workers annotation. +type ManualWorkerPoolRollout struct { + // PendingWorkersRollouts contains the names of the worker pools that are still pending rollout. + PendingWorkersRollouts []PendingWorkersRollout +} + // ShootKubeconfigRotation contains information about the kubeconfig credential rotation. type ShootKubeconfigRotation struct { // LastInitiationTime is the most recent time when the kubeconfig credential rotation was initiated. @@ -341,12 +349,11 @@ const ( RotationCompleted CredentialsRotationPhase = "Completed" ) -// PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout due to -// credentials rotation. +// PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout. type PendingWorkersRollout struct { // Name is the name of a worker pool. Name string - // LastInitiationTime is the most recent time when the credential rotation was initiated. + // LastInitiationTime is the most recent time when the worker rollout was initiated. LastInitiationTime *metav1.Time } diff --git a/pkg/apis/core/v1beta1/constants/types_constants.go b/pkg/apis/core/v1beta1/constants/types_constants.go index 4cc93d54be7..3991c6e49e0 100644 --- a/pkg/apis/core/v1beta1/constants/types_constants.go +++ b/pkg/apis/core/v1beta1/constants/types_constants.go @@ -432,6 +432,9 @@ const ( // (comma-separated) when the certificate authorities or service account signing key credentials rotation is in // WaitingForWorkersRollout phase. OperationRotateRolloutWorkers = "rotate-rollout-workers" + // OperationRolloutWorkers is a constant for an annotation triggering the rollout of one or more worker pools + // (comma-separated). + OperationRolloutWorkers = "rollout-workers" // SeedOperationRenewGardenAccessSecrets is a constant for an annotation on a Seed indicating that // all garden access secrets on the seed shall be renewed. SeedOperationRenewGardenAccessSecrets = "renew-garden-access-secrets" // #nosec G101 -- No credential. diff --git a/pkg/apis/core/v1beta1/generated.pb.go b/pkg/apis/core/v1beta1/generated.pb.go index 605b666ba3b..849860f37b9 100644 --- a/pkg/apis/core/v1beta1/generated.pb.go +++ b/pkg/apis/core/v1beta1/generated.pb.go @@ -3179,10 +3179,38 @@ func (m *MaintenanceTimeWindow) XXX_DiscardUnknown() { var xxx_messageInfo_MaintenanceTimeWindow proto.InternalMessageInfo +func (m *ManualWorkerPoolRollout) Reset() { *m = ManualWorkerPoolRollout{} } +func (*ManualWorkerPoolRollout) ProtoMessage() {} +func (*ManualWorkerPoolRollout) Descriptor() ([]byte, []int) { + return fileDescriptor_ca37af0df9a5bbd2, []int{112} +} +func (m *ManualWorkerPoolRollout) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ManualWorkerPoolRollout) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *ManualWorkerPoolRollout) XXX_Merge(src proto.Message) { + xxx_messageInfo_ManualWorkerPoolRollout.Merge(m, src) +} +func (m *ManualWorkerPoolRollout) XXX_Size() int { + return m.Size() +} +func (m *ManualWorkerPoolRollout) XXX_DiscardUnknown() { + xxx_messageInfo_ManualWorkerPoolRollout.DiscardUnknown(m) +} + +var xxx_messageInfo_ManualWorkerPoolRollout proto.InternalMessageInfo + func (m *MemorySwapConfiguration) Reset() { *m = MemorySwapConfiguration{} } func (*MemorySwapConfiguration) ProtoMessage() {} func (*MemorySwapConfiguration) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{112} + return fileDescriptor_ca37af0df9a5bbd2, []int{113} } func (m *MemorySwapConfiguration) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3210,7 +3238,7 @@ var xxx_messageInfo_MemorySwapConfiguration proto.InternalMessageInfo func (m *Monitoring) Reset() { *m = Monitoring{} } func (*Monitoring) ProtoMessage() {} func (*Monitoring) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{113} + return fileDescriptor_ca37af0df9a5bbd2, []int{114} } func (m *Monitoring) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3238,7 +3266,7 @@ var xxx_messageInfo_Monitoring proto.InternalMessageInfo func (m *NamedResourceReference) Reset() { *m = NamedResourceReference{} } func (*NamedResourceReference) ProtoMessage() {} func (*NamedResourceReference) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{114} + return fileDescriptor_ca37af0df9a5bbd2, []int{115} } func (m *NamedResourceReference) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3266,7 +3294,7 @@ var xxx_messageInfo_NamedResourceReference proto.InternalMessageInfo func (m *NamespacedCloudProfile) Reset() { *m = NamespacedCloudProfile{} } func (*NamespacedCloudProfile) ProtoMessage() {} func (*NamespacedCloudProfile) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{115} + return fileDescriptor_ca37af0df9a5bbd2, []int{116} } func (m *NamespacedCloudProfile) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3294,7 +3322,7 @@ var xxx_messageInfo_NamespacedCloudProfile proto.InternalMessageInfo func (m *NamespacedCloudProfileList) Reset() { *m = NamespacedCloudProfileList{} } func (*NamespacedCloudProfileList) ProtoMessage() {} func (*NamespacedCloudProfileList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{116} + return fileDescriptor_ca37af0df9a5bbd2, []int{117} } func (m *NamespacedCloudProfileList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3322,7 +3350,7 @@ var xxx_messageInfo_NamespacedCloudProfileList proto.InternalMessageInfo func (m *NamespacedCloudProfileSpec) Reset() { *m = NamespacedCloudProfileSpec{} } func (*NamespacedCloudProfileSpec) ProtoMessage() {} func (*NamespacedCloudProfileSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{117} + return fileDescriptor_ca37af0df9a5bbd2, []int{118} } func (m *NamespacedCloudProfileSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3350,7 +3378,7 @@ var xxx_messageInfo_NamespacedCloudProfileSpec proto.InternalMessageInfo func (m *NamespacedCloudProfileStatus) Reset() { *m = NamespacedCloudProfileStatus{} } func (*NamespacedCloudProfileStatus) ProtoMessage() {} func (*NamespacedCloudProfileStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{118} + return fileDescriptor_ca37af0df9a5bbd2, []int{119} } func (m *NamespacedCloudProfileStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3378,7 +3406,7 @@ var xxx_messageInfo_NamespacedCloudProfileStatus proto.InternalMessageInfo func (m *Networking) Reset() { *m = Networking{} } func (*Networking) ProtoMessage() {} func (*Networking) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{119} + return fileDescriptor_ca37af0df9a5bbd2, []int{120} } func (m *Networking) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3406,7 +3434,7 @@ var xxx_messageInfo_Networking proto.InternalMessageInfo func (m *NetworkingStatus) Reset() { *m = NetworkingStatus{} } func (*NetworkingStatus) ProtoMessage() {} func (*NetworkingStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{120} + return fileDescriptor_ca37af0df9a5bbd2, []int{121} } func (m *NetworkingStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3434,7 +3462,7 @@ var xxx_messageInfo_NetworkingStatus proto.InternalMessageInfo func (m *NginxIngress) Reset() { *m = NginxIngress{} } func (*NginxIngress) ProtoMessage() {} func (*NginxIngress) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{121} + return fileDescriptor_ca37af0df9a5bbd2, []int{122} } func (m *NginxIngress) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3462,7 +3490,7 @@ var xxx_messageInfo_NginxIngress proto.InternalMessageInfo func (m *NodeLocalDNS) Reset() { *m = NodeLocalDNS{} } func (*NodeLocalDNS) ProtoMessage() {} func (*NodeLocalDNS) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{122} + return fileDescriptor_ca37af0df9a5bbd2, []int{123} } func (m *NodeLocalDNS) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3490,7 +3518,7 @@ var xxx_messageInfo_NodeLocalDNS proto.InternalMessageInfo func (m *OCIRepository) Reset() { *m = OCIRepository{} } func (*OCIRepository) ProtoMessage() {} func (*OCIRepository) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{123} + return fileDescriptor_ca37af0df9a5bbd2, []int{124} } func (m *OCIRepository) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3518,7 +3546,7 @@ var xxx_messageInfo_OCIRepository proto.InternalMessageInfo func (m *OIDCConfig) Reset() { *m = OIDCConfig{} } func (*OIDCConfig) ProtoMessage() {} func (*OIDCConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{124} + return fileDescriptor_ca37af0df9a5bbd2, []int{125} } func (m *OIDCConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3546,7 +3574,7 @@ var xxx_messageInfo_OIDCConfig proto.InternalMessageInfo func (m *ObservabilityRotation) Reset() { *m = ObservabilityRotation{} } func (*ObservabilityRotation) ProtoMessage() {} func (*ObservabilityRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{125} + return fileDescriptor_ca37af0df9a5bbd2, []int{126} } func (m *ObservabilityRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3574,7 +3602,7 @@ var xxx_messageInfo_ObservabilityRotation proto.InternalMessageInfo func (m *OpenIDConnectClientAuthentication) Reset() { *m = OpenIDConnectClientAuthentication{} } func (*OpenIDConnectClientAuthentication) ProtoMessage() {} func (*OpenIDConnectClientAuthentication) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{126} + return fileDescriptor_ca37af0df9a5bbd2, []int{127} } func (m *OpenIDConnectClientAuthentication) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3602,7 +3630,7 @@ var xxx_messageInfo_OpenIDConnectClientAuthentication proto.InternalMessageInfo func (m *PendingWorkerUpdates) Reset() { *m = PendingWorkerUpdates{} } func (*PendingWorkerUpdates) ProtoMessage() {} func (*PendingWorkerUpdates) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{127} + return fileDescriptor_ca37af0df9a5bbd2, []int{128} } func (m *PendingWorkerUpdates) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3630,7 +3658,7 @@ var xxx_messageInfo_PendingWorkerUpdates proto.InternalMessageInfo func (m *PendingWorkersRollout) Reset() { *m = PendingWorkersRollout{} } func (*PendingWorkersRollout) ProtoMessage() {} func (*PendingWorkersRollout) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{128} + return fileDescriptor_ca37af0df9a5bbd2, []int{129} } func (m *PendingWorkersRollout) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3658,7 +3686,7 @@ var xxx_messageInfo_PendingWorkersRollout proto.InternalMessageInfo func (m *Project) Reset() { *m = Project{} } func (*Project) ProtoMessage() {} func (*Project) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{129} + return fileDescriptor_ca37af0df9a5bbd2, []int{130} } func (m *Project) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3686,7 +3714,7 @@ var xxx_messageInfo_Project proto.InternalMessageInfo func (m *ProjectList) Reset() { *m = ProjectList{} } func (*ProjectList) ProtoMessage() {} func (*ProjectList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{130} + return fileDescriptor_ca37af0df9a5bbd2, []int{131} } func (m *ProjectList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3714,7 +3742,7 @@ var xxx_messageInfo_ProjectList proto.InternalMessageInfo func (m *ProjectMember) Reset() { *m = ProjectMember{} } func (*ProjectMember) ProtoMessage() {} func (*ProjectMember) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{131} + return fileDescriptor_ca37af0df9a5bbd2, []int{132} } func (m *ProjectMember) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3742,7 +3770,7 @@ var xxx_messageInfo_ProjectMember proto.InternalMessageInfo func (m *ProjectSpec) Reset() { *m = ProjectSpec{} } func (*ProjectSpec) ProtoMessage() {} func (*ProjectSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{132} + return fileDescriptor_ca37af0df9a5bbd2, []int{133} } func (m *ProjectSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3770,7 +3798,7 @@ var xxx_messageInfo_ProjectSpec proto.InternalMessageInfo func (m *ProjectStatus) Reset() { *m = ProjectStatus{} } func (*ProjectStatus) ProtoMessage() {} func (*ProjectStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{133} + return fileDescriptor_ca37af0df9a5bbd2, []int{134} } func (m *ProjectStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3798,7 +3826,7 @@ var xxx_messageInfo_ProjectStatus proto.InternalMessageInfo func (m *ProjectTolerations) Reset() { *m = ProjectTolerations{} } func (*ProjectTolerations) ProtoMessage() {} func (*ProjectTolerations) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{134} + return fileDescriptor_ca37af0df9a5bbd2, []int{135} } func (m *ProjectTolerations) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3826,7 +3854,7 @@ var xxx_messageInfo_ProjectTolerations proto.InternalMessageInfo func (m *Provider) Reset() { *m = Provider{} } func (*Provider) ProtoMessage() {} func (*Provider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{135} + return fileDescriptor_ca37af0df9a5bbd2, []int{136} } func (m *Provider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3854,7 +3882,7 @@ var xxx_messageInfo_Provider proto.InternalMessageInfo func (m *Quota) Reset() { *m = Quota{} } func (*Quota) ProtoMessage() {} func (*Quota) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{136} + return fileDescriptor_ca37af0df9a5bbd2, []int{137} } func (m *Quota) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3882,7 +3910,7 @@ var xxx_messageInfo_Quota proto.InternalMessageInfo func (m *QuotaList) Reset() { *m = QuotaList{} } func (*QuotaList) ProtoMessage() {} func (*QuotaList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{137} + return fileDescriptor_ca37af0df9a5bbd2, []int{138} } func (m *QuotaList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3910,7 +3938,7 @@ var xxx_messageInfo_QuotaList proto.InternalMessageInfo func (m *QuotaSpec) Reset() { *m = QuotaSpec{} } func (*QuotaSpec) ProtoMessage() {} func (*QuotaSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{138} + return fileDescriptor_ca37af0df9a5bbd2, []int{139} } func (m *QuotaSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3938,7 +3966,7 @@ var xxx_messageInfo_QuotaSpec proto.InternalMessageInfo func (m *Region) Reset() { *m = Region{} } func (*Region) ProtoMessage() {} func (*Region) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{139} + return fileDescriptor_ca37af0df9a5bbd2, []int{140} } func (m *Region) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3966,7 +3994,7 @@ var xxx_messageInfo_Region proto.InternalMessageInfo func (m *ResourceData) Reset() { *m = ResourceData{} } func (*ResourceData) ProtoMessage() {} func (*ResourceData) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{140} + return fileDescriptor_ca37af0df9a5bbd2, []int{141} } func (m *ResourceData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3994,7 +4022,7 @@ var xxx_messageInfo_ResourceData proto.InternalMessageInfo func (m *ResourceWatchCacheSize) Reset() { *m = ResourceWatchCacheSize{} } func (*ResourceWatchCacheSize) ProtoMessage() {} func (*ResourceWatchCacheSize) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{141} + return fileDescriptor_ca37af0df9a5bbd2, []int{142} } func (m *ResourceWatchCacheSize) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4022,7 +4050,7 @@ var xxx_messageInfo_ResourceWatchCacheSize proto.InternalMessageInfo func (m *SSHAccess) Reset() { *m = SSHAccess{} } func (*SSHAccess) ProtoMessage() {} func (*SSHAccess) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{142} + return fileDescriptor_ca37af0df9a5bbd2, []int{143} } func (m *SSHAccess) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4050,7 +4078,7 @@ var xxx_messageInfo_SSHAccess proto.InternalMessageInfo func (m *SecretBinding) Reset() { *m = SecretBinding{} } func (*SecretBinding) ProtoMessage() {} func (*SecretBinding) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{143} + return fileDescriptor_ca37af0df9a5bbd2, []int{144} } func (m *SecretBinding) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4078,7 +4106,7 @@ var xxx_messageInfo_SecretBinding proto.InternalMessageInfo func (m *SecretBindingList) Reset() { *m = SecretBindingList{} } func (*SecretBindingList) ProtoMessage() {} func (*SecretBindingList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{144} + return fileDescriptor_ca37af0df9a5bbd2, []int{145} } func (m *SecretBindingList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4106,7 +4134,7 @@ var xxx_messageInfo_SecretBindingList proto.InternalMessageInfo func (m *SecretBindingProvider) Reset() { *m = SecretBindingProvider{} } func (*SecretBindingProvider) ProtoMessage() {} func (*SecretBindingProvider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{145} + return fileDescriptor_ca37af0df9a5bbd2, []int{146} } func (m *SecretBindingProvider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4134,7 +4162,7 @@ var xxx_messageInfo_SecretBindingProvider proto.InternalMessageInfo func (m *Seed) Reset() { *m = Seed{} } func (*Seed) ProtoMessage() {} func (*Seed) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{146} + return fileDescriptor_ca37af0df9a5bbd2, []int{147} } func (m *Seed) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4162,7 +4190,7 @@ var xxx_messageInfo_Seed proto.InternalMessageInfo func (m *SeedDNS) Reset() { *m = SeedDNS{} } func (*SeedDNS) ProtoMessage() {} func (*SeedDNS) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{147} + return fileDescriptor_ca37af0df9a5bbd2, []int{148} } func (m *SeedDNS) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4190,7 +4218,7 @@ var xxx_messageInfo_SeedDNS proto.InternalMessageInfo func (m *SeedDNSProvider) Reset() { *m = SeedDNSProvider{} } func (*SeedDNSProvider) ProtoMessage() {} func (*SeedDNSProvider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{148} + return fileDescriptor_ca37af0df9a5bbd2, []int{149} } func (m *SeedDNSProvider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4218,7 +4246,7 @@ var xxx_messageInfo_SeedDNSProvider proto.InternalMessageInfo func (m *SeedDNSProviderConfig) Reset() { *m = SeedDNSProviderConfig{} } func (*SeedDNSProviderConfig) ProtoMessage() {} func (*SeedDNSProviderConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{149} + return fileDescriptor_ca37af0df9a5bbd2, []int{150} } func (m *SeedDNSProviderConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4246,7 +4274,7 @@ var xxx_messageInfo_SeedDNSProviderConfig proto.InternalMessageInfo func (m *SeedList) Reset() { *m = SeedList{} } func (*SeedList) ProtoMessage() {} func (*SeedList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{150} + return fileDescriptor_ca37af0df9a5bbd2, []int{151} } func (m *SeedList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4274,7 +4302,7 @@ var xxx_messageInfo_SeedList proto.InternalMessageInfo func (m *SeedNetworks) Reset() { *m = SeedNetworks{} } func (*SeedNetworks) ProtoMessage() {} func (*SeedNetworks) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{151} + return fileDescriptor_ca37af0df9a5bbd2, []int{152} } func (m *SeedNetworks) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4302,7 +4330,7 @@ var xxx_messageInfo_SeedNetworks proto.InternalMessageInfo func (m *SeedProvider) Reset() { *m = SeedProvider{} } func (*SeedProvider) ProtoMessage() {} func (*SeedProvider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{152} + return fileDescriptor_ca37af0df9a5bbd2, []int{153} } func (m *SeedProvider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4330,7 +4358,7 @@ var xxx_messageInfo_SeedProvider proto.InternalMessageInfo func (m *SeedSelector) Reset() { *m = SeedSelector{} } func (*SeedSelector) ProtoMessage() {} func (*SeedSelector) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{153} + return fileDescriptor_ca37af0df9a5bbd2, []int{154} } func (m *SeedSelector) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4358,7 +4386,7 @@ var xxx_messageInfo_SeedSelector proto.InternalMessageInfo func (m *SeedSettingDependencyWatchdog) Reset() { *m = SeedSettingDependencyWatchdog{} } func (*SeedSettingDependencyWatchdog) ProtoMessage() {} func (*SeedSettingDependencyWatchdog) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{154} + return fileDescriptor_ca37af0df9a5bbd2, []int{155} } func (m *SeedSettingDependencyWatchdog) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4386,7 +4414,7 @@ var xxx_messageInfo_SeedSettingDependencyWatchdog proto.InternalMessageInfo func (m *SeedSettingDependencyWatchdogProber) Reset() { *m = SeedSettingDependencyWatchdogProber{} } func (*SeedSettingDependencyWatchdogProber) ProtoMessage() {} func (*SeedSettingDependencyWatchdogProber) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{155} + return fileDescriptor_ca37af0df9a5bbd2, []int{156} } func (m *SeedSettingDependencyWatchdogProber) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4414,7 +4442,7 @@ var xxx_messageInfo_SeedSettingDependencyWatchdogProber proto.InternalMessageInf func (m *SeedSettingDependencyWatchdogWeeder) Reset() { *m = SeedSettingDependencyWatchdogWeeder{} } func (*SeedSettingDependencyWatchdogWeeder) ProtoMessage() {} func (*SeedSettingDependencyWatchdogWeeder) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{156} + return fileDescriptor_ca37af0df9a5bbd2, []int{157} } func (m *SeedSettingDependencyWatchdogWeeder) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4442,7 +4470,7 @@ var xxx_messageInfo_SeedSettingDependencyWatchdogWeeder proto.InternalMessageInf func (m *SeedSettingExcessCapacityReservation) Reset() { *m = SeedSettingExcessCapacityReservation{} } func (*SeedSettingExcessCapacityReservation) ProtoMessage() {} func (*SeedSettingExcessCapacityReservation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{157} + return fileDescriptor_ca37af0df9a5bbd2, []int{158} } func (m *SeedSettingExcessCapacityReservation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4472,7 +4500,7 @@ func (m *SeedSettingExcessCapacityReservationConfig) Reset() { } func (*SeedSettingExcessCapacityReservationConfig) ProtoMessage() {} func (*SeedSettingExcessCapacityReservationConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{158} + return fileDescriptor_ca37af0df9a5bbd2, []int{159} } func (m *SeedSettingExcessCapacityReservationConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4500,7 +4528,7 @@ var xxx_messageInfo_SeedSettingExcessCapacityReservationConfig proto.InternalMes func (m *SeedSettingLoadBalancerServices) Reset() { *m = SeedSettingLoadBalancerServices{} } func (*SeedSettingLoadBalancerServices) ProtoMessage() {} func (*SeedSettingLoadBalancerServices) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{159} + return fileDescriptor_ca37af0df9a5bbd2, []int{160} } func (m *SeedSettingLoadBalancerServices) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4530,7 +4558,7 @@ func (m *SeedSettingLoadBalancerServicesZonalIngress) Reset() { } func (*SeedSettingLoadBalancerServicesZonalIngress) ProtoMessage() {} func (*SeedSettingLoadBalancerServicesZonalIngress) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{160} + return fileDescriptor_ca37af0df9a5bbd2, []int{161} } func (m *SeedSettingLoadBalancerServicesZonalIngress) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4558,7 +4586,7 @@ var xxx_messageInfo_SeedSettingLoadBalancerServicesZonalIngress proto.InternalMe func (m *SeedSettingLoadBalancerServicesZones) Reset() { *m = SeedSettingLoadBalancerServicesZones{} } func (*SeedSettingLoadBalancerServicesZones) ProtoMessage() {} func (*SeedSettingLoadBalancerServicesZones) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{161} + return fileDescriptor_ca37af0df9a5bbd2, []int{162} } func (m *SeedSettingLoadBalancerServicesZones) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4586,7 +4614,7 @@ var xxx_messageInfo_SeedSettingLoadBalancerServicesZones proto.InternalMessageIn func (m *SeedSettingScheduling) Reset() { *m = SeedSettingScheduling{} } func (*SeedSettingScheduling) ProtoMessage() {} func (*SeedSettingScheduling) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{162} + return fileDescriptor_ca37af0df9a5bbd2, []int{163} } func (m *SeedSettingScheduling) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4614,7 +4642,7 @@ var xxx_messageInfo_SeedSettingScheduling proto.InternalMessageInfo func (m *SeedSettingTopologyAwareRouting) Reset() { *m = SeedSettingTopologyAwareRouting{} } func (*SeedSettingTopologyAwareRouting) ProtoMessage() {} func (*SeedSettingTopologyAwareRouting) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{163} + return fileDescriptor_ca37af0df9a5bbd2, []int{164} } func (m *SeedSettingTopologyAwareRouting) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4642,7 +4670,7 @@ var xxx_messageInfo_SeedSettingTopologyAwareRouting proto.InternalMessageInfo func (m *SeedSettingVerticalPodAutoscaler) Reset() { *m = SeedSettingVerticalPodAutoscaler{} } func (*SeedSettingVerticalPodAutoscaler) ProtoMessage() {} func (*SeedSettingVerticalPodAutoscaler) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{164} + return fileDescriptor_ca37af0df9a5bbd2, []int{165} } func (m *SeedSettingVerticalPodAutoscaler) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4670,7 +4698,7 @@ var xxx_messageInfo_SeedSettingVerticalPodAutoscaler proto.InternalMessageInfo func (m *SeedSettings) Reset() { *m = SeedSettings{} } func (*SeedSettings) ProtoMessage() {} func (*SeedSettings) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{165} + return fileDescriptor_ca37af0df9a5bbd2, []int{166} } func (m *SeedSettings) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4698,7 +4726,7 @@ var xxx_messageInfo_SeedSettings proto.InternalMessageInfo func (m *SeedSpec) Reset() { *m = SeedSpec{} } func (*SeedSpec) ProtoMessage() {} func (*SeedSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{166} + return fileDescriptor_ca37af0df9a5bbd2, []int{167} } func (m *SeedSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4726,7 +4754,7 @@ var xxx_messageInfo_SeedSpec proto.InternalMessageInfo func (m *SeedStatus) Reset() { *m = SeedStatus{} } func (*SeedStatus) ProtoMessage() {} func (*SeedStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{167} + return fileDescriptor_ca37af0df9a5bbd2, []int{168} } func (m *SeedStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4754,7 +4782,7 @@ var xxx_messageInfo_SeedStatus proto.InternalMessageInfo func (m *SeedTaint) Reset() { *m = SeedTaint{} } func (*SeedTaint) ProtoMessage() {} func (*SeedTaint) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{168} + return fileDescriptor_ca37af0df9a5bbd2, []int{169} } func (m *SeedTaint) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4782,7 +4810,7 @@ var xxx_messageInfo_SeedTaint proto.InternalMessageInfo func (m *SeedTemplate) Reset() { *m = SeedTemplate{} } func (*SeedTemplate) ProtoMessage() {} func (*SeedTemplate) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{169} + return fileDescriptor_ca37af0df9a5bbd2, []int{170} } func (m *SeedTemplate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4810,7 +4838,7 @@ var xxx_messageInfo_SeedTemplate proto.InternalMessageInfo func (m *SeedVolume) Reset() { *m = SeedVolume{} } func (*SeedVolume) ProtoMessage() {} func (*SeedVolume) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{170} + return fileDescriptor_ca37af0df9a5bbd2, []int{171} } func (m *SeedVolume) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4838,7 +4866,7 @@ var xxx_messageInfo_SeedVolume proto.InternalMessageInfo func (m *SeedVolumeProvider) Reset() { *m = SeedVolumeProvider{} } func (*SeedVolumeProvider) ProtoMessage() {} func (*SeedVolumeProvider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{171} + return fileDescriptor_ca37af0df9a5bbd2, []int{172} } func (m *SeedVolumeProvider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4866,7 +4894,7 @@ var xxx_messageInfo_SeedVolumeProvider proto.InternalMessageInfo func (m *ServiceAccountConfig) Reset() { *m = ServiceAccountConfig{} } func (*ServiceAccountConfig) ProtoMessage() {} func (*ServiceAccountConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{172} + return fileDescriptor_ca37af0df9a5bbd2, []int{173} } func (m *ServiceAccountConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4894,7 +4922,7 @@ var xxx_messageInfo_ServiceAccountConfig proto.InternalMessageInfo func (m *ServiceAccountKeyRotation) Reset() { *m = ServiceAccountKeyRotation{} } func (*ServiceAccountKeyRotation) ProtoMessage() {} func (*ServiceAccountKeyRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{173} + return fileDescriptor_ca37af0df9a5bbd2, []int{174} } func (m *ServiceAccountKeyRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4922,7 +4950,7 @@ var xxx_messageInfo_ServiceAccountKeyRotation proto.InternalMessageInfo func (m *Shoot) Reset() { *m = Shoot{} } func (*Shoot) ProtoMessage() {} func (*Shoot) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{174} + return fileDescriptor_ca37af0df9a5bbd2, []int{175} } func (m *Shoot) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4950,7 +4978,7 @@ var xxx_messageInfo_Shoot proto.InternalMessageInfo func (m *ShootAdvertisedAddress) Reset() { *m = ShootAdvertisedAddress{} } func (*ShootAdvertisedAddress) ProtoMessage() {} func (*ShootAdvertisedAddress) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{175} + return fileDescriptor_ca37af0df9a5bbd2, []int{176} } func (m *ShootAdvertisedAddress) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4978,7 +5006,7 @@ var xxx_messageInfo_ShootAdvertisedAddress proto.InternalMessageInfo func (m *ShootCredentials) Reset() { *m = ShootCredentials{} } func (*ShootCredentials) ProtoMessage() {} func (*ShootCredentials) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{176} + return fileDescriptor_ca37af0df9a5bbd2, []int{177} } func (m *ShootCredentials) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5006,7 +5034,7 @@ var xxx_messageInfo_ShootCredentials proto.InternalMessageInfo func (m *ShootCredentialsRotation) Reset() { *m = ShootCredentialsRotation{} } func (*ShootCredentialsRotation) ProtoMessage() {} func (*ShootCredentialsRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{177} + return fileDescriptor_ca37af0df9a5bbd2, []int{178} } func (m *ShootCredentialsRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5034,7 +5062,7 @@ var xxx_messageInfo_ShootCredentialsRotation proto.InternalMessageInfo func (m *ShootKubeconfigRotation) Reset() { *m = ShootKubeconfigRotation{} } func (*ShootKubeconfigRotation) ProtoMessage() {} func (*ShootKubeconfigRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{178} + return fileDescriptor_ca37af0df9a5bbd2, []int{179} } func (m *ShootKubeconfigRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5062,7 +5090,7 @@ var xxx_messageInfo_ShootKubeconfigRotation proto.InternalMessageInfo func (m *ShootList) Reset() { *m = ShootList{} } func (*ShootList) ProtoMessage() {} func (*ShootList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{179} + return fileDescriptor_ca37af0df9a5bbd2, []int{180} } func (m *ShootList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5090,7 +5118,7 @@ var xxx_messageInfo_ShootList proto.InternalMessageInfo func (m *ShootMachineImage) Reset() { *m = ShootMachineImage{} } func (*ShootMachineImage) ProtoMessage() {} func (*ShootMachineImage) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{180} + return fileDescriptor_ca37af0df9a5bbd2, []int{181} } func (m *ShootMachineImage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5118,7 +5146,7 @@ var xxx_messageInfo_ShootMachineImage proto.InternalMessageInfo func (m *ShootNetworks) Reset() { *m = ShootNetworks{} } func (*ShootNetworks) ProtoMessage() {} func (*ShootNetworks) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{181} + return fileDescriptor_ca37af0df9a5bbd2, []int{182} } func (m *ShootNetworks) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5146,7 +5174,7 @@ var xxx_messageInfo_ShootNetworks proto.InternalMessageInfo func (m *ShootSSHKeypairRotation) Reset() { *m = ShootSSHKeypairRotation{} } func (*ShootSSHKeypairRotation) ProtoMessage() {} func (*ShootSSHKeypairRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{182} + return fileDescriptor_ca37af0df9a5bbd2, []int{183} } func (m *ShootSSHKeypairRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5174,7 +5202,7 @@ var xxx_messageInfo_ShootSSHKeypairRotation proto.InternalMessageInfo func (m *ShootSpec) Reset() { *m = ShootSpec{} } func (*ShootSpec) ProtoMessage() {} func (*ShootSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{183} + return fileDescriptor_ca37af0df9a5bbd2, []int{184} } func (m *ShootSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5202,7 +5230,7 @@ var xxx_messageInfo_ShootSpec proto.InternalMessageInfo func (m *ShootState) Reset() { *m = ShootState{} } func (*ShootState) ProtoMessage() {} func (*ShootState) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{184} + return fileDescriptor_ca37af0df9a5bbd2, []int{185} } func (m *ShootState) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5230,7 +5258,7 @@ var xxx_messageInfo_ShootState proto.InternalMessageInfo func (m *ShootStateList) Reset() { *m = ShootStateList{} } func (*ShootStateList) ProtoMessage() {} func (*ShootStateList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{185} + return fileDescriptor_ca37af0df9a5bbd2, []int{186} } func (m *ShootStateList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5258,7 +5286,7 @@ var xxx_messageInfo_ShootStateList proto.InternalMessageInfo func (m *ShootStateSpec) Reset() { *m = ShootStateSpec{} } func (*ShootStateSpec) ProtoMessage() {} func (*ShootStateSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{186} + return fileDescriptor_ca37af0df9a5bbd2, []int{187} } func (m *ShootStateSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5286,7 +5314,7 @@ var xxx_messageInfo_ShootStateSpec proto.InternalMessageInfo func (m *ShootStatus) Reset() { *m = ShootStatus{} } func (*ShootStatus) ProtoMessage() {} func (*ShootStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{187} + return fileDescriptor_ca37af0df9a5bbd2, []int{188} } func (m *ShootStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5314,7 +5342,7 @@ var xxx_messageInfo_ShootStatus proto.InternalMessageInfo func (m *ShootTemplate) Reset() { *m = ShootTemplate{} } func (*ShootTemplate) ProtoMessage() {} func (*ShootTemplate) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{188} + return fileDescriptor_ca37af0df9a5bbd2, []int{189} } func (m *ShootTemplate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5342,7 +5370,7 @@ var xxx_messageInfo_ShootTemplate proto.InternalMessageInfo func (m *StructuredAuthentication) Reset() { *m = StructuredAuthentication{} } func (*StructuredAuthentication) ProtoMessage() {} func (*StructuredAuthentication) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{189} + return fileDescriptor_ca37af0df9a5bbd2, []int{190} } func (m *StructuredAuthentication) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5370,7 +5398,7 @@ var xxx_messageInfo_StructuredAuthentication proto.InternalMessageInfo func (m *StructuredAuthorization) Reset() { *m = StructuredAuthorization{} } func (*StructuredAuthorization) ProtoMessage() {} func (*StructuredAuthorization) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{190} + return fileDescriptor_ca37af0df9a5bbd2, []int{191} } func (m *StructuredAuthorization) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5398,7 +5426,7 @@ var xxx_messageInfo_StructuredAuthorization proto.InternalMessageInfo func (m *SystemComponents) Reset() { *m = SystemComponents{} } func (*SystemComponents) ProtoMessage() {} func (*SystemComponents) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{191} + return fileDescriptor_ca37af0df9a5bbd2, []int{192} } func (m *SystemComponents) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5426,7 +5454,7 @@ var xxx_messageInfo_SystemComponents proto.InternalMessageInfo func (m *Toleration) Reset() { *m = Toleration{} } func (*Toleration) ProtoMessage() {} func (*Toleration) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{192} + return fileDescriptor_ca37af0df9a5bbd2, []int{193} } func (m *Toleration) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5454,7 +5482,7 @@ var xxx_messageInfo_Toleration proto.InternalMessageInfo func (m *VerticalPodAutoscaler) Reset() { *m = VerticalPodAutoscaler{} } func (*VerticalPodAutoscaler) ProtoMessage() {} func (*VerticalPodAutoscaler) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{193} + return fileDescriptor_ca37af0df9a5bbd2, []int{194} } func (m *VerticalPodAutoscaler) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5482,7 +5510,7 @@ var xxx_messageInfo_VerticalPodAutoscaler proto.InternalMessageInfo func (m *Volume) Reset() { *m = Volume{} } func (*Volume) ProtoMessage() {} func (*Volume) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{194} + return fileDescriptor_ca37af0df9a5bbd2, []int{195} } func (m *Volume) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5510,7 +5538,7 @@ var xxx_messageInfo_Volume proto.InternalMessageInfo func (m *VolumeType) Reset() { *m = VolumeType{} } func (*VolumeType) ProtoMessage() {} func (*VolumeType) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{195} + return fileDescriptor_ca37af0df9a5bbd2, []int{196} } func (m *VolumeType) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5538,7 +5566,7 @@ var xxx_messageInfo_VolumeType proto.InternalMessageInfo func (m *WatchCacheSizes) Reset() { *m = WatchCacheSizes{} } func (*WatchCacheSizes) ProtoMessage() {} func (*WatchCacheSizes) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{196} + return fileDescriptor_ca37af0df9a5bbd2, []int{197} } func (m *WatchCacheSizes) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5566,7 +5594,7 @@ var xxx_messageInfo_WatchCacheSizes proto.InternalMessageInfo func (m *Worker) Reset() { *m = Worker{} } func (*Worker) ProtoMessage() {} func (*Worker) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{197} + return fileDescriptor_ca37af0df9a5bbd2, []int{198} } func (m *Worker) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5594,7 +5622,7 @@ var xxx_messageInfo_Worker proto.InternalMessageInfo func (m *WorkerControlPlane) Reset() { *m = WorkerControlPlane{} } func (*WorkerControlPlane) ProtoMessage() {} func (*WorkerControlPlane) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{198} + return fileDescriptor_ca37af0df9a5bbd2, []int{199} } func (m *WorkerControlPlane) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5622,7 +5650,7 @@ var xxx_messageInfo_WorkerControlPlane proto.InternalMessageInfo func (m *WorkerKubernetes) Reset() { *m = WorkerKubernetes{} } func (*WorkerKubernetes) ProtoMessage() {} func (*WorkerKubernetes) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{199} + return fileDescriptor_ca37af0df9a5bbd2, []int{200} } func (m *WorkerKubernetes) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5650,7 +5678,7 @@ var xxx_messageInfo_WorkerKubernetes proto.InternalMessageInfo func (m *WorkerSystemComponents) Reset() { *m = WorkerSystemComponents{} } func (*WorkerSystemComponents) ProtoMessage() {} func (*WorkerSystemComponents) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{200} + return fileDescriptor_ca37af0df9a5bbd2, []int{201} } func (m *WorkerSystemComponents) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5678,7 +5706,7 @@ var xxx_messageInfo_WorkerSystemComponents proto.InternalMessageInfo func (m *WorkersSettings) Reset() { *m = WorkersSettings{} } func (*WorkersSettings) ProtoMessage() {} func (*WorkersSettings) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{201} + return fileDescriptor_ca37af0df9a5bbd2, []int{202} } func (m *WorkersSettings) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5825,6 +5853,7 @@ func init() { proto.RegisterType((*Maintenance)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.Maintenance") proto.RegisterType((*MaintenanceAutoUpdate)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.MaintenanceAutoUpdate") proto.RegisterType((*MaintenanceTimeWindow)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.MaintenanceTimeWindow") + proto.RegisterType((*ManualWorkerPoolRollout)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.ManualWorkerPoolRollout") proto.RegisterType((*MemorySwapConfiguration)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.MemorySwapConfiguration") proto.RegisterType((*Monitoring)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.Monitoring") proto.RegisterType((*NamedResourceReference)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.NamedResourceReference") @@ -5940,924 +5969,926 @@ func init() { } var fileDescriptor_ca37af0df9a5bbd2 = []byte{ - // 14661 bytes of a gzipped FileDescriptorProto + // 14698 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x6b, 0x70, 0x24, 0xd9, 0x55, 0x20, 0xec, 0x2c, 0xbd, 0x8f, 0x1e, 0x2d, 0xdd, 0x56, 0x77, 0xab, 0xd5, 0x8f, 0x6a, 0xe7, 0x8c, 0x4d, 0x0f, 0x63, 0xab, 0x99, 0xf1, 0xd8, 0xe3, 0x19, 0x33, 0x0f, 0xa9, 0x4a, 0xea, 0x2e, - 0xb7, 0xa4, 0xd6, 0xdc, 0x92, 0x66, 0xc6, 0x33, 0x30, 0x26, 0x95, 0x75, 0x55, 0xca, 0xe9, 0xac, - 0xcc, 0x9a, 0xcc, 0x2c, 0xb5, 0xd4, 0x63, 0xe3, 0x07, 0x98, 0xcf, 0x36, 0xc6, 0x1f, 0xf0, 0x01, - 0xfe, 0x6c, 0xb3, 0xc1, 0x18, 0x02, 0xb3, 0xbb, 0xb0, 0x66, 0x83, 0x0d, 0x36, 0x02, 0x88, 0x65, - 0x17, 0x22, 0x58, 0x0c, 0x01, 0x04, 0x81, 0x61, 0xd7, 0x2c, 0x50, 0x60, 0xad, 0x17, 0x36, 0x62, - 0x9f, 0xb1, 0x44, 0x2c, 0x11, 0xbd, 0x04, 0xbb, 0x71, 0x1f, 0x99, 0x79, 0xf3, 0x55, 0x2a, 0x65, - 0x49, 0xf2, 0xcc, 0xe2, 0x5f, 0x52, 0xdd, 0x73, 0xef, 0x39, 0xf7, 0x95, 0xe7, 0x9e, 0x73, 0xee, - 0xb9, 0xe7, 0xc0, 0x42, 0xdd, 0xf0, 0xb6, 0x5b, 0x9b, 0x73, 0xba, 0xdd, 0xb8, 0x56, 0xd7, 0x9c, - 0x1a, 0xb1, 0x88, 0x13, 0xfe, 0xd3, 0xbc, 0x5d, 0xbf, 0xa6, 0x35, 0x0d, 0xf7, 0x9a, 0x6e, 0x3b, - 0xe4, 0xda, 0xce, 0x43, 0x9b, 0xc4, 0xd3, 0x1e, 0xba, 0x56, 0xa7, 0x30, 0xcd, 0x23, 0xb5, 0xb9, - 0xa6, 0x63, 0x7b, 0x36, 0x7a, 0x38, 0xc4, 0x31, 0xe7, 0x37, 0x0d, 0xff, 0x69, 0xde, 0xae, 0xcf, - 0x51, 0x1c, 0x73, 0x14, 0xc7, 0x9c, 0xc0, 0x31, 0xfb, 0x76, 0x99, 0xae, 0x5d, 0xb7, 0xaf, 0x31, - 0x54, 0x9b, 0xad, 0x2d, 0xf6, 0x8b, 0xfd, 0x60, 0xff, 0x71, 0x12, 0xb3, 0x0f, 0xdc, 0x7e, 0xb7, - 0x3b, 0x67, 0xd8, 0xb4, 0x33, 0xd7, 0xb4, 0x96, 0x67, 0xbb, 0xba, 0x66, 0x1a, 0x56, 0xfd, 0xda, - 0x4e, 0xa2, 0x37, 0xb3, 0xaa, 0x54, 0x55, 0x74, 0xbb, 0x63, 0x1d, 0x67, 0x53, 0xd3, 0xd3, 0xea, - 0xdc, 0x08, 0xeb, 0x90, 0x5d, 0x8f, 0x58, 0xae, 0x61, 0x5b, 0xee, 0xdb, 0xe9, 0x48, 0x88, 0xb3, - 0x23, 0xcf, 0x4d, 0xa4, 0x42, 0x1a, 0xa6, 0x47, 0x42, 0x4c, 0x0d, 0x4d, 0xdf, 0x36, 0x2c, 0xe2, - 0xec, 0xf9, 0xcd, 0xaf, 0x39, 0xc4, 0xb5, 0x5b, 0x8e, 0x4e, 0x0e, 0xd5, 0xca, 0xbd, 0xd6, 0x20, - 0x9e, 0x96, 0x46, 0xeb, 0x5a, 0x56, 0x2b, 0xa7, 0x65, 0x79, 0x46, 0x23, 0x49, 0xe6, 0x5d, 0x07, - 0x35, 0x70, 0xf5, 0x6d, 0xd2, 0xd0, 0x12, 0xed, 0xde, 0x91, 0xd5, 0xae, 0xe5, 0x19, 0xe6, 0x35, - 0xc3, 0xf2, 0x5c, 0xcf, 0x89, 0x37, 0x52, 0x3f, 0xa9, 0xc0, 0xe4, 0xfc, 0x5a, 0xa5, 0xca, 0x66, - 0x70, 0xd9, 0xae, 0xd7, 0x0d, 0xab, 0x8e, 0x1e, 0x84, 0x91, 0x1d, 0xe2, 0x6c, 0xda, 0xae, 0xe1, - 0xed, 0xcd, 0x28, 0x57, 0x94, 0xab, 0x03, 0x0b, 0xe3, 0xfb, 0xed, 0xe2, 0xc8, 0xb3, 0x7e, 0x21, - 0x0e, 0xe1, 0xa8, 0x02, 0xa7, 0xb7, 0x3d, 0xaf, 0x39, 0xaf, 0xeb, 0xc4, 0x75, 0x83, 0x1a, 0x33, - 0x05, 0xd6, 0xec, 0xdc, 0x7e, 0xbb, 0x78, 0xfa, 0xc6, 0xfa, 0xfa, 0x5a, 0x0c, 0x8c, 0xd3, 0xda, - 0xa8, 0xbf, 0xa0, 0xc0, 0x54, 0xd0, 0x19, 0x4c, 0x5e, 0x69, 0x11, 0xd7, 0x73, 0x11, 0x86, 0xb3, - 0x0d, 0x6d, 0x77, 0xd5, 0xb6, 0x56, 0x5a, 0x9e, 0xe6, 0x19, 0x56, 0xbd, 0x62, 0x6d, 0x99, 0x46, - 0x7d, 0xdb, 0x13, 0x5d, 0x9b, 0xdd, 0x6f, 0x17, 0xcf, 0xae, 0xa4, 0xd6, 0xc0, 0x19, 0x2d, 0x69, - 0xa7, 0x1b, 0xda, 0x6e, 0x02, 0xa1, 0xd4, 0xe9, 0x95, 0x24, 0x18, 0xa7, 0xb5, 0x51, 0xdf, 0x09, - 0x53, 0x7c, 0x1c, 0x98, 0xb8, 0x9e, 0x63, 0xe8, 0x9e, 0x61, 0x5b, 0xe8, 0x0a, 0xf4, 0x5b, 0x5a, - 0x83, 0xb0, 0x1e, 0x8e, 0x2c, 0x8c, 0x7d, 0xb9, 0x5d, 0x7c, 0xd3, 0x7e, 0xbb, 0xd8, 0xbf, 0xaa, - 0x35, 0x08, 0x66, 0x10, 0xf5, 0x7f, 0x16, 0xe0, 0x62, 0xa2, 0xdd, 0x73, 0x86, 0xb7, 0x7d, 0xab, - 0x49, 0xff, 0x73, 0xd1, 0xa7, 0x15, 0x98, 0xd2, 0xe2, 0x15, 0x18, 0xc2, 0xd1, 0x87, 0x17, 0xe7, - 0x0e, 0xff, 0x81, 0xcf, 0x25, 0xa8, 0x2d, 0x9c, 0x17, 0xfd, 0x4a, 0x0e, 0x00, 0x27, 0x49, 0xa3, - 0x8f, 0x2b, 0x30, 0x64, 0xf3, 0xce, 0xcd, 0x14, 0xae, 0xf4, 0x5d, 0x1d, 0x7d, 0xf8, 0x3b, 0x8f, - 0xa4, 0x1b, 0xd2, 0xa0, 0xe7, 0xc4, 0xdf, 0x45, 0xcb, 0x73, 0xf6, 0x16, 0x4e, 0x89, 0xee, 0x0d, - 0x89, 0x52, 0xec, 0x93, 0x9f, 0x7d, 0x1c, 0xc6, 0xe4, 0x9a, 0x68, 0x12, 0xfa, 0x6e, 0x13, 0xbe, - 0x55, 0x47, 0x30, 0xfd, 0x17, 0x4d, 0xc3, 0xc0, 0x8e, 0x66, 0xb6, 0x08, 0x5b, 0xd2, 0x11, 0xcc, - 0x7f, 0x3c, 0x5e, 0x78, 0xb7, 0xa2, 0x3e, 0x0c, 0x03, 0xf3, 0xb5, 0x9a, 0x6d, 0xa1, 0x07, 0x60, - 0x88, 0x58, 0xda, 0xa6, 0x49, 0x6a, 0xac, 0xe1, 0x70, 0x48, 0x6f, 0x91, 0x17, 0x63, 0x1f, 0xae, - 0xfe, 0x68, 0x01, 0x06, 0x59, 0x23, 0x17, 0xfd, 0xb0, 0x02, 0xa7, 0x6f, 0xb7, 0x36, 0x89, 0x63, - 0x11, 0x8f, 0xb8, 0x65, 0xcd, 0xdd, 0xde, 0xb4, 0x35, 0xa7, 0x26, 0x16, 0xe6, 0x7a, 0x9e, 0x19, - 0xb9, 0x99, 0x44, 0xc7, 0xf7, 0x60, 0x0a, 0x00, 0xa7, 0x11, 0x47, 0x3b, 0x30, 0x66, 0xd5, 0x0d, - 0x6b, 0xb7, 0x62, 0xd5, 0x1d, 0xe2, 0xba, 0x6c, 0xd0, 0xa3, 0x0f, 0x3f, 0x9d, 0xa7, 0x33, 0xab, - 0x12, 0x9e, 0x85, 0xc9, 0xfd, 0x76, 0x71, 0x4c, 0x2e, 0xc1, 0x11, 0x3a, 0xea, 0xdf, 0x29, 0x70, - 0x6a, 0xbe, 0xd6, 0x30, 0x5c, 0xca, 0x69, 0xd7, 0xcc, 0x56, 0xdd, 0xe8, 0x62, 0xeb, 0xa3, 0x67, - 0x60, 0x50, 0xb7, 0xad, 0x2d, 0xa3, 0x2e, 0xfa, 0xf9, 0xf6, 0x39, 0xce, 0xb9, 0xe6, 0x64, 0xce, - 0xc5, 0xba, 0x27, 0x38, 0xde, 0x1c, 0xd6, 0xee, 0x2c, 0xfa, 0x0c, 0x7d, 0x01, 0xf6, 0xdb, 0xc5, - 0xc1, 0x12, 0x43, 0x80, 0x05, 0x22, 0x74, 0x15, 0x86, 0x6b, 0x86, 0xcb, 0x17, 0xb3, 0x8f, 0x2d, - 0xe6, 0xd8, 0x7e, 0xbb, 0x38, 0x5c, 0x16, 0x65, 0x38, 0x80, 0xa2, 0x65, 0x98, 0xa6, 0x33, 0xc8, - 0xdb, 0x55, 0x89, 0xee, 0x10, 0x8f, 0x76, 0x6d, 0xa6, 0x9f, 0x75, 0x77, 0x66, 0xbf, 0x5d, 0x9c, - 0xbe, 0x99, 0x02, 0xc7, 0xa9, 0xad, 0xd4, 0x25, 0x18, 0x9e, 0x37, 0x89, 0x43, 0x19, 0x02, 0x7a, - 0x1c, 0x26, 0x48, 0x43, 0x33, 0x4c, 0x4c, 0x74, 0x62, 0xec, 0x10, 0xc7, 0x9d, 0x51, 0xae, 0xf4, - 0x5d, 0x1d, 0x59, 0x40, 0xfb, 0xed, 0xe2, 0xc4, 0x62, 0x04, 0x82, 0x63, 0x35, 0xd5, 0x8f, 0x28, - 0x30, 0x3a, 0xdf, 0xaa, 0x19, 0x1e, 0x1f, 0x17, 0x72, 0x60, 0x54, 0xa3, 0x3f, 0xd7, 0x6c, 0xd3, - 0xd0, 0xf7, 0xc4, 0xe6, 0x7a, 0x2a, 0xd7, 0xe7, 0x16, 0xa2, 0x59, 0x38, 0xb5, 0xdf, 0x2e, 0x8e, - 0x4a, 0x05, 0x58, 0x26, 0xa2, 0x6e, 0x83, 0x0c, 0x43, 0xef, 0x83, 0x31, 0x3e, 0xdc, 0x15, 0xad, - 0x89, 0xc9, 0x96, 0xe8, 0xc3, 0x7d, 0xd2, 0x5a, 0xf9, 0x84, 0xe6, 0x6e, 0x6d, 0xbe, 0x4c, 0x74, - 0x0f, 0x93, 0x2d, 0xe2, 0x10, 0x4b, 0x27, 0x7c, 0xdb, 0x94, 0xa4, 0xc6, 0x38, 0x82, 0x4a, 0xfd, - 0xff, 0x14, 0xb8, 0x34, 0xdf, 0xf2, 0xb6, 0x6d, 0xc7, 0xb8, 0x4b, 0x9c, 0x70, 0xba, 0x03, 0x0c, - 0xe8, 0x49, 0x98, 0xd0, 0x82, 0x0a, 0xab, 0xe1, 0x76, 0x3a, 0x2b, 0xb6, 0xd3, 0xc4, 0x7c, 0x04, - 0x8a, 0x63, 0xb5, 0xd1, 0xc3, 0x00, 0x6e, 0xb8, 0xb6, 0x8c, 0x07, 0x2c, 0x20, 0xd1, 0x16, 0xa4, - 0x55, 0x95, 0x6a, 0xa9, 0x7f, 0x4e, 0x8f, 0xc2, 0x1d, 0xcd, 0x30, 0xb5, 0x4d, 0xc3, 0x34, 0xbc, - 0xbd, 0x17, 0x6c, 0x8b, 0x74, 0xb1, 0x9b, 0x37, 0xe0, 0x5c, 0xcb, 0xd2, 0x78, 0x3b, 0x93, 0xac, - 0xf0, 0xfd, 0xbb, 0xbe, 0xd7, 0x24, 0x9c, 0x4b, 0x8e, 0x2c, 0x5c, 0xd8, 0x6f, 0x17, 0xcf, 0x6d, - 0xa4, 0x57, 0xc1, 0x59, 0x6d, 0xe9, 0xa9, 0x27, 0x81, 0x9e, 0xb5, 0xcd, 0x56, 0x43, 0x60, 0xed, - 0x63, 0x58, 0xd9, 0xa9, 0xb7, 0x91, 0x5a, 0x03, 0x67, 0xb4, 0x54, 0xbf, 0x58, 0x80, 0xc1, 0x05, - 0x4d, 0xbf, 0xdd, 0x6a, 0xa2, 0xb7, 0xc1, 0x70, 0xd3, 0xb1, 0x77, 0x8c, 0x1a, 0x71, 0xc4, 0xd8, - 0x26, 0xc5, 0xd8, 0x86, 0xd7, 0x44, 0x39, 0x0e, 0x6a, 0x20, 0x03, 0x26, 0xfc, 0xff, 0x4b, 0x3d, - 0x7c, 0xb9, 0xec, 0x4b, 0x58, 0x8b, 0x20, 0xc2, 0x31, 0xc4, 0x48, 0x85, 0x41, 0x87, 0xd4, 0xe9, - 0x51, 0xd7, 0xc7, 0xba, 0xc5, 0xbe, 0x76, 0xcc, 0x4a, 0xb0, 0x80, 0xa0, 0xf7, 0xc3, 0x84, 0xee, - 0x90, 0x1a, 0xb1, 0x3c, 0x43, 0x33, 0x5d, 0xba, 0x39, 0x07, 0xba, 0xdf, 0x9c, 0xac, 0x13, 0xa5, - 0x48, 0x73, 0x1c, 0x43, 0xa7, 0x7e, 0xb9, 0x00, 0x63, 0x7c, 0xa2, 0x16, 0x5a, 0xfa, 0x6d, 0xe2, - 0xa1, 0xef, 0x82, 0x61, 0x2a, 0xdf, 0xd5, 0x34, 0x4f, 0x13, 0x1f, 0xc2, 0xb7, 0x65, 0x0e, 0x9d, - 0x7d, 0x83, 0xb4, 0x76, 0x48, 0x7d, 0x85, 0x78, 0x5a, 0xb8, 0xff, 0xc2, 0x32, 0x1c, 0x60, 0x45, - 0x5b, 0xd0, 0xef, 0x36, 0x89, 0x2e, 0x26, 0xb6, 0x9c, 0xe7, 0x53, 0x97, 0x7b, 0x5c, 0x6d, 0x12, - 0x3d, 0xdc, 0xae, 0xf4, 0x17, 0x66, 0xf8, 0x91, 0x05, 0x83, 0xae, 0xa7, 0x79, 0x2d, 0x97, 0xcd, - 0xef, 0xe8, 0xc3, 0x4b, 0x3d, 0x53, 0x62, 0xd8, 0x16, 0x26, 0x04, 0xad, 0x41, 0xfe, 0x1b, 0x0b, - 0x2a, 0xea, 0xbf, 0x55, 0x60, 0x52, 0xae, 0xbe, 0x6c, 0xb8, 0x1e, 0xfa, 0x8e, 0xc4, 0x74, 0xce, - 0x75, 0x37, 0x9d, 0xb4, 0x35, 0x9b, 0xcc, 0x60, 0xb7, 0xfa, 0x25, 0xd2, 0x54, 0x12, 0x18, 0x30, - 0x3c, 0xd2, 0xf0, 0xa5, 0x94, 0xa7, 0x7b, 0x1d, 0xe1, 0xc2, 0xb8, 0x20, 0x36, 0x50, 0xa1, 0x68, - 0x31, 0xc7, 0xae, 0x7e, 0x17, 0x4c, 0xcb, 0xb5, 0xfc, 0x7d, 0x4d, 0x59, 0x86, 0xb7, 0xd7, 0x4c, - 0xb0, 0x0c, 0xfa, 0x09, 0x62, 0x06, 0x41, 0x6f, 0x0d, 0xf6, 0x38, 0xe7, 0x4c, 0xc1, 0xdc, 0x45, - 0xf7, 0xb9, 0xfa, 0xe1, 0xbe, 0xe8, 0xdc, 0xd1, 0x65, 0x44, 0x3b, 0xb1, 0x2f, 0x77, 0xf4, 0xe1, - 0x1b, 0xbd, 0x0e, 0xd0, 0xef, 0xfa, 0xeb, 0x85, 0x07, 0x5c, 0x85, 0x61, 0x97, 0x90, 0x9a, 0x74, - 0x2e, 0xb3, 0xd3, 0xbc, 0x2a, 0xca, 0x70, 0x00, 0x3d, 0x7e, 0x4e, 0xf0, 0x5a, 0x3f, 0xa0, 0xe4, - 0x6e, 0x97, 0x27, 0x83, 0x97, 0x88, 0xa5, 0xe8, 0x65, 0x32, 0xc4, 0x87, 0x13, 0x43, 0x8c, 0xee, - 0xc2, 0xb8, 0xa9, 0xb9, 0xde, 0xad, 0x26, 0xd5, 0xdb, 0xfc, 0x3d, 0x33, 0xfa, 0xf0, 0x7c, 0x9e, - 0x45, 0x5f, 0x96, 0x11, 0x2d, 0x4c, 0xed, 0xb7, 0x8b, 0xe3, 0x91, 0x22, 0x1c, 0x25, 0x85, 0x5e, - 0x86, 0x11, 0x5a, 0xb0, 0xe8, 0x38, 0xb6, 0x23, 0xf8, 0xc5, 0x13, 0x79, 0xe9, 0x32, 0x24, 0x5c, - 0x8f, 0x0c, 0x7e, 0xe2, 0x10, 0x3d, 0x7a, 0x2f, 0x20, 0x7b, 0x93, 0x69, 0xf2, 0xb5, 0xeb, 0x5c, - 0x49, 0xa5, 0x83, 0xa5, 0xcb, 0xdf, 0xb7, 0x30, 0x2b, 0xf6, 0x25, 0xba, 0x95, 0xa8, 0x81, 0x53, - 0x5a, 0xa1, 0xdb, 0x80, 0x02, 0x45, 0x97, 0x9f, 0xf6, 0x07, 0x6c, 0x8d, 0xa0, 0x92, 0xd8, 0x1a, - 0x67, 0x29, 0xb1, 0xeb, 0x09, 0x14, 0x38, 0x05, 0xad, 0xfa, 0x1b, 0x05, 0x18, 0xe5, 0x5b, 0x84, - 0x2b, 0x23, 0xc7, 0x7f, 0x56, 0x90, 0xc8, 0x59, 0x51, 0xca, 0xff, 0xf9, 0xb3, 0x0e, 0x67, 0x1e, - 0x15, 0x8d, 0xd8, 0x51, 0xb1, 0xd8, 0x2b, 0xa1, 0xce, 0x27, 0xc5, 0x1f, 0x29, 0x70, 0x4a, 0xaa, - 0x7d, 0x02, 0x07, 0x45, 0x2d, 0x7a, 0x50, 0x3c, 0xd5, 0xe3, 0xf8, 0x32, 0xce, 0x09, 0x3b, 0x32, - 0x2c, 0xc6, 0xc3, 0x1f, 0x06, 0xd8, 0x64, 0xec, 0x44, 0x12, 0x6d, 0x83, 0x25, 0x5f, 0x08, 0x20, - 0x58, 0xaa, 0x15, 0x61, 0x8a, 0x85, 0x4e, 0x4c, 0x51, 0xfd, 0x0f, 0x7d, 0x30, 0x95, 0x98, 0xf6, - 0x24, 0x1f, 0x51, 0xbe, 0x41, 0x7c, 0xa4, 0xf0, 0x8d, 0xe0, 0x23, 0x7d, 0xb9, 0xf8, 0x48, 0xf7, - 0x07, 0x91, 0x03, 0xa8, 0x61, 0xd4, 0x79, 0xb3, 0xaa, 0xa7, 0x39, 0xde, 0xba, 0xd1, 0x20, 0x82, - 0xe3, 0x7c, 0x6b, 0x77, 0x5b, 0x96, 0xb6, 0xe0, 0x8c, 0x67, 0x25, 0x81, 0x09, 0xa7, 0x60, 0x57, - 0xbf, 0xa7, 0x00, 0x43, 0x0b, 0x9a, 0xcb, 0x7a, 0xfa, 0x41, 0x18, 0x13, 0xa8, 0x2b, 0x0d, 0xad, - 0x4e, 0x7a, 0x31, 0x47, 0x08, 0x94, 0x2b, 0x12, 0x3a, 0xae, 0xd1, 0xc9, 0x25, 0x38, 0x42, 0x0e, - 0xed, 0xc1, 0x68, 0x23, 0xd4, 0x5e, 0xc4, 0x12, 0x2f, 0xf5, 0x4e, 0x9d, 0x62, 0xe3, 0x6a, 0xab, - 0x54, 0x80, 0x65, 0x5a, 0xea, 0x4b, 0x70, 0x3a, 0xa5, 0xc7, 0x5d, 0x28, 0x6e, 0x6f, 0x81, 0x21, - 0xaa, 0x7b, 0x87, 0x62, 0xd8, 0xe8, 0x7e, 0xbb, 0x38, 0xf4, 0x2c, 0x2f, 0xc2, 0x3e, 0x4c, 0x7d, - 0x17, 0x15, 0x00, 0xe2, 0x7d, 0xea, 0xc6, 0xc0, 0x37, 0x00, 0x50, 0x9a, 0xc7, 0xb6, 0xc7, 0xb7, - 0xd2, 0x53, 0x30, 0xd0, 0xdc, 0xd6, 0x5c, 0xbf, 0xc5, 0x03, 0x3e, 0xab, 0x58, 0xa3, 0x85, 0xf7, - 0xda, 0xc5, 0x19, 0x59, 0x10, 0x11, 0x8d, 0x18, 0x0c, 0xf3, 0x76, 0x74, 0x87, 0xd1, 0x4d, 0x5e, - 0xb2, 0x1b, 0x4d, 0x93, 0x50, 0x28, 0xdb, 0x61, 0x85, 0x7c, 0x3b, 0x6c, 0x39, 0x81, 0x09, 0xa7, - 0x60, 0xf7, 0x69, 0x56, 0x2c, 0xc3, 0x33, 0xb4, 0x80, 0x66, 0x5f, 0x7e, 0x9a, 0x51, 0x4c, 0x38, - 0x05, 0x3b, 0xfa, 0xa4, 0x02, 0xb3, 0xd1, 0xe2, 0x25, 0xc3, 0x32, 0xdc, 0x6d, 0x52, 0x63, 0xc4, - 0xfb, 0x0f, 0x4d, 0xfc, 0xf2, 0x7e, 0xbb, 0x38, 0xbb, 0x9c, 0x89, 0x11, 0x77, 0xa0, 0x86, 0x7e, - 0x40, 0x81, 0x0b, 0xb1, 0x79, 0x71, 0x8c, 0x7a, 0x9d, 0x38, 0xa2, 0x37, 0x87, 0xff, 0xc0, 0x8b, - 0xfb, 0xed, 0xe2, 0x85, 0xe5, 0x6c, 0x94, 0xb8, 0x13, 0x3d, 0xf4, 0x05, 0x05, 0xce, 0x36, 0x89, - 0x55, 0x33, 0xac, 0xfa, 0x73, 0xb6, 0x73, 0x9b, 0x38, 0x2e, 0xb6, 0x4d, 0xd3, 0x6e, 0x79, 0xee, - 0xcc, 0x20, 0x3b, 0xc3, 0x2a, 0x79, 0xbe, 0xb9, 0xb5, 0x34, 0x8c, 0x0b, 0x97, 0xc5, 0x16, 0x3d, - 0x9b, 0x0a, 0x76, 0x71, 0x46, 0x47, 0xd4, 0x5f, 0x57, 0xa0, 0xaf, 0x84, 0x2b, 0xe8, 0xc1, 0xc8, - 0x27, 0x72, 0x4e, 0xfe, 0x44, 0xee, 0xb5, 0x8b, 0x43, 0x25, 0x5c, 0x91, 0x3e, 0xc6, 0x1f, 0x50, - 0x60, 0x4a, 0xb7, 0x2d, 0x4f, 0xa3, 0x73, 0x87, 0xb9, 0xac, 0xec, 0x9f, 0xcb, 0xb9, 0x94, 0xe1, - 0x52, 0x0c, 0x59, 0x68, 0xec, 0x8e, 0x43, 0x5c, 0x9c, 0xa4, 0xac, 0xfe, 0xa4, 0x02, 0xd3, 0x25, - 0xad, 0x29, 0x4c, 0x41, 0x65, 0xb2, 0x65, 0xd0, 0x0d, 0xd2, 0x8d, 0x65, 0x1f, 0x6d, 0xc3, 0x20, - 0xb3, 0x36, 0xbb, 0xbd, 0xe8, 0xf2, 0x21, 0xed, 0x67, 0x19, 0x2e, 0x6e, 0x07, 0xe1, 0xff, 0x63, - 0x81, 0x5f, 0x7d, 0x02, 0x26, 0xe3, 0xf5, 0x50, 0xd1, 0x97, 0x69, 0xb8, 0xf1, 0x71, 0x24, 0x2e, - 0x8e, 0x3c, 0x3e, 0xfc, 0xff, 0xbf, 0x56, 0x7c, 0xd3, 0x87, 0xff, 0xf4, 0xca, 0x9b, 0xd4, 0xaf, - 0x2a, 0x30, 0x56, 0x32, 0xed, 0x56, 0x6d, 0xcd, 0xb1, 0xb7, 0x0c, 0x93, 0xbc, 0x31, 0xac, 0x1c, - 0x72, 0x8f, 0xb3, 0x44, 0x57, 0x66, 0x75, 0x90, 0x2b, 0xbe, 0x41, 0xac, 0x0e, 0x72, 0x97, 0x33, - 0xa4, 0xc9, 0x17, 0xe1, 0x8c, 0x5c, 0x2b, 0x34, 0x99, 0x5e, 0x81, 0xfe, 0xdb, 0x86, 0x55, 0x8b, - 0x6f, 0xcc, 0x9b, 0x86, 0x55, 0xc3, 0x0c, 0x12, 0x6c, 0xdd, 0x42, 0xe6, 0x99, 0xf5, 0xd7, 0x23, - 0xd1, 0x69, 0x63, 0xc2, 0xea, 0x55, 0x18, 0xd6, 0xb5, 0x85, 0x96, 0x55, 0x33, 0x83, 0x5d, 0x4f, - 0xa7, 0xa0, 0x34, 0xcf, 0xcb, 0x70, 0x00, 0x45, 0x77, 0x01, 0xc2, 0xdb, 0x89, 0x5e, 0x84, 0x80, - 0xf0, 0xe2, 0xa3, 0x4a, 0x3c, 0xcf, 0xb0, 0xea, 0x6e, 0xb8, 0xaf, 0x42, 0x18, 0x96, 0xa8, 0xa1, - 0x0f, 0xc2, 0xb8, 0x2c, 0x91, 0x70, 0x33, 0x69, 0xce, 0x65, 0x88, 0x88, 0x3e, 0x67, 0x04, 0xe1, - 0x71, 0xb9, 0xd4, 0xc5, 0x51, 0x6a, 0x68, 0x2f, 0x90, 0xbf, 0xb8, 0x91, 0xb6, 0x3f, 0xbf, 0x46, - 0x21, 0x8b, 0x3e, 0xd3, 0x82, 0xf8, 0x58, 0xc4, 0x68, 0x1c, 0x21, 0x95, 0x62, 0x98, 0x19, 0x38, - 0x2e, 0xc3, 0x0c, 0x81, 0x21, 0x6e, 0x9a, 0xf2, 0x8f, 0x9b, 0xc7, 0xf3, 0x0c, 0x90, 0x5b, 0xb9, - 0xc2, 0xeb, 0x36, 0xfe, 0xdb, 0xc5, 0x3e, 0x6e, 0xb4, 0x03, 0x63, 0x54, 0xb0, 0xae, 0x12, 0x93, - 0xe8, 0x9e, 0xed, 0xcc, 0x0c, 0xe5, 0xbf, 0xce, 0xaa, 0x4a, 0x78, 0xb8, 0x14, 0x2b, 0x97, 0xe0, - 0x08, 0x9d, 0xc0, 0x72, 0x37, 0x9c, 0x69, 0xb9, 0x6b, 0xc1, 0xe8, 0x8e, 0x64, 0x8a, 0x1f, 0x61, - 0x93, 0xf0, 0x64, 0x9e, 0x8e, 0x85, 0x76, 0xf9, 0x85, 0xd3, 0x82, 0xd0, 0xa8, 0x6c, 0xc3, 0x97, - 0xe9, 0xa0, 0x4d, 0x18, 0xda, 0xe4, 0x32, 0xe8, 0x0c, 0xb0, 0xb9, 0x78, 0x4f, 0x0f, 0xa2, 0x35, - 0x97, 0x73, 0xc5, 0x0f, 0xec, 0x23, 0x46, 0x2f, 0xc1, 0xa0, 0x69, 0x34, 0x0c, 0xcf, 0x9d, 0x19, - 0x65, 0x24, 0x72, 0x2d, 0xed, 0x32, 0xc3, 0xc0, 0x0f, 0x2b, 0xfe, 0x3f, 0x16, 0x58, 0xd1, 0x67, - 0x14, 0x38, 0x2d, 0xb6, 0x61, 0x70, 0x68, 0x19, 0xc4, 0x9d, 0x19, 0x63, 0x73, 0x78, 0xa3, 0xb7, - 0x43, 0x32, 0x3c, 0xa0, 0x17, 0x2e, 0x88, 0xd9, 0x3c, 0xbd, 0x92, 0x24, 0x86, 0xd3, 0x7a, 0xa0, - 0xfe, 0xcb, 0x71, 0x98, 0x2a, 0x99, 0x2d, 0xd7, 0x23, 0xce, 0xbc, 0xf0, 0x64, 0x21, 0x0e, 0xfa, - 0xa8, 0x02, 0x67, 0xd9, 0xbf, 0x65, 0xfb, 0x8e, 0x55, 0x26, 0xa6, 0xb6, 0x37, 0xbf, 0x45, 0x6b, - 0xd4, 0x6a, 0x87, 0x3b, 0x3c, 0xca, 0x2d, 0xa1, 0x26, 0xb3, 0x1b, 0x9b, 0x6a, 0x2a, 0x46, 0x9c, - 0x41, 0x09, 0x7d, 0xbf, 0x02, 0xe7, 0x53, 0x40, 0x65, 0x62, 0x12, 0xcf, 0x17, 0xfe, 0x0f, 0xdb, - 0x8f, 0x4b, 0xfb, 0xed, 0xe2, 0xf9, 0x6a, 0x16, 0x52, 0x9c, 0x4d, 0x0f, 0x7d, 0x5a, 0x81, 0xd9, - 0x14, 0xe8, 0x92, 0x66, 0x98, 0x2d, 0xc7, 0xd7, 0x0b, 0x0e, 0xdb, 0x1d, 0x26, 0x9e, 0x57, 0x33, - 0xb1, 0xe2, 0x0e, 0x14, 0xd1, 0x87, 0xe0, 0x4c, 0x00, 0xdd, 0xb0, 0x2c, 0x42, 0x6a, 0x11, 0x2d, - 0xe1, 0xb0, 0x5d, 0x39, 0xbf, 0xdf, 0x2e, 0x9e, 0xa9, 0xa6, 0x21, 0xc4, 0xe9, 0x74, 0x50, 0x1d, - 0x2e, 0x85, 0x00, 0xcf, 0x30, 0x8d, 0xbb, 0x5c, 0x91, 0xd9, 0x76, 0x88, 0xbb, 0x6d, 0x9b, 0x35, - 0xc6, 0x8a, 0x95, 0x85, 0x37, 0xef, 0xb7, 0x8b, 0x97, 0xaa, 0x9d, 0x2a, 0xe2, 0xce, 0x78, 0x50, - 0x0d, 0xc6, 0x5c, 0x5d, 0xb3, 0x2a, 0x96, 0x47, 0x9c, 0x1d, 0xcd, 0x9c, 0x19, 0xcc, 0x35, 0x40, - 0xce, 0x00, 0x25, 0x3c, 0x38, 0x82, 0x15, 0xbd, 0x1b, 0x86, 0xc9, 0x6e, 0x53, 0xb3, 0x6a, 0x84, - 0x33, 0xdd, 0x91, 0x85, 0x8b, 0xf4, 0xa8, 0x5f, 0x14, 0x65, 0xf7, 0xda, 0xc5, 0x31, 0xff, 0xff, - 0x15, 0xbb, 0x46, 0x70, 0x50, 0x1b, 0x7d, 0x00, 0xa6, 0x99, 0xab, 0x4d, 0x8d, 0xb0, 0x23, 0xc4, - 0xf5, 0x75, 0xc5, 0xe1, 0x5c, 0xfd, 0x64, 0xd7, 0xf0, 0x2b, 0x29, 0xf8, 0x70, 0x2a, 0x15, 0xba, - 0x0c, 0x0d, 0x6d, 0xf7, 0xba, 0xa3, 0xe9, 0x64, 0xab, 0x65, 0xae, 0x13, 0xa7, 0x61, 0x58, 0xdc, - 0x58, 0x42, 0x74, 0xdb, 0xaa, 0x51, 0x46, 0xad, 0x5c, 0x1d, 0xe0, 0xcb, 0xb0, 0xd2, 0xa9, 0x22, - 0xee, 0x8c, 0x07, 0x3d, 0x02, 0x63, 0x46, 0xdd, 0xb2, 0x1d, 0xb2, 0xae, 0x19, 0x96, 0xe7, 0xce, - 0x00, 0x13, 0xb2, 0xd9, 0xb4, 0x56, 0xa4, 0x72, 0x1c, 0xa9, 0x85, 0x76, 0x00, 0x59, 0xe4, 0xce, - 0x9a, 0x5d, 0x63, 0x5b, 0x60, 0xa3, 0xc9, 0x36, 0xb2, 0x60, 0xb3, 0x87, 0x9d, 0x1a, 0xa6, 0x4a, - 0xaf, 0x26, 0xb0, 0xe1, 0x14, 0x0a, 0x68, 0x09, 0x50, 0x43, 0xdb, 0x5d, 0x6c, 0x34, 0xbd, 0xbd, - 0x85, 0x96, 0x79, 0x5b, 0x70, 0x8d, 0x31, 0x36, 0x17, 0xdc, 0xd0, 0x94, 0x80, 0xe2, 0x94, 0x16, - 0x48, 0x83, 0x0b, 0x7c, 0x3c, 0x65, 0x8d, 0x34, 0x6c, 0xcb, 0x25, 0x9e, 0x2b, 0x6d, 0xd2, 0x99, - 0x71, 0xe6, 0x70, 0xc1, 0x14, 0xdb, 0x4a, 0x76, 0x35, 0xdc, 0x09, 0x47, 0xd4, 0xe5, 0x6c, 0xe2, - 0x00, 0x97, 0xb3, 0x47, 0x61, 0xdc, 0xf5, 0x34, 0xc7, 0x6b, 0x35, 0xc5, 0x32, 0x9c, 0x62, 0xcb, - 0xc0, 0xec, 0x90, 0x55, 0x19, 0x80, 0xa3, 0xf5, 0xe8, 0xf2, 0x71, 0x63, 0xb3, 0x68, 0x37, 0x19, - 0x2e, 0x5f, 0x55, 0x2a, 0xc7, 0x91, 0x5a, 0x68, 0x03, 0xce, 0x35, 0xb4, 0xdd, 0xe0, 0xf3, 0x5d, - 0xd3, 0x1c, 0xcd, 0x34, 0x89, 0x69, 0xb8, 0x8d, 0x99, 0x29, 0xd6, 0x53, 0x76, 0xc3, 0xbf, 0x92, - 0x5e, 0x05, 0x67, 0xb5, 0x15, 0x3e, 0x68, 0x65, 0x47, 0x33, 0x22, 0x28, 0x51, 0xc4, 0x07, 0x2d, - 0x0e, 0xc6, 0x69, 0x6d, 0xd4, 0xff, 0xd1, 0x0f, 0x33, 0x89, 0x13, 0xcc, 0x77, 0x24, 0x3b, 0x90, - 0x47, 0x29, 0x47, 0xc4, 0xa3, 0x9a, 0x70, 0x25, 0xa8, 0x70, 0xbd, 0xd9, 0x4a, 0xa5, 0x55, 0x60, - 0xb4, 0xee, 0xdf, 0x6f, 0x17, 0xaf, 0x54, 0x0f, 0xa8, 0x8b, 0x0f, 0xc4, 0x96, 0xcd, 0xff, 0xfb, - 0x4e, 0x88, 0xff, 0x7f, 0x00, 0xa6, 0x25, 0x80, 0x43, 0xb4, 0xda, 0x5e, 0x0f, 0xe7, 0x0f, 0x63, - 0x7b, 0xd5, 0x14, 0x7c, 0x38, 0x95, 0x4a, 0x26, 0xd3, 0x1d, 0x38, 0x09, 0xa6, 0xab, 0xb6, 0xfb, - 0x60, 0xa4, 0x64, 0x5b, 0x35, 0x6e, 0x17, 0x79, 0x28, 0x72, 0xeb, 0x7d, 0x49, 0x96, 0x9d, 0xef, - 0xb5, 0x8b, 0xe3, 0x41, 0x45, 0x49, 0x98, 0x7e, 0x2c, 0xb8, 0x5f, 0xe2, 0x1a, 0xe9, 0x9b, 0xa3, - 0x17, 0x43, 0xf7, 0xda, 0xc5, 0x53, 0x41, 0xb3, 0xe8, 0x5d, 0x11, 0xe5, 0xa8, 0xa6, 0xe6, 0x7a, - 0xeb, 0x8e, 0x66, 0xb9, 0x46, 0x0f, 0x86, 0xc9, 0xe0, 0x42, 0x60, 0x39, 0x81, 0x0d, 0xa7, 0x50, - 0x40, 0x2f, 0xc3, 0x04, 0x2d, 0xdd, 0x68, 0xd6, 0x34, 0x8f, 0xe4, 0xb4, 0x47, 0x06, 0x3e, 0x4c, - 0xcb, 0x11, 0x4c, 0x38, 0x86, 0x99, 0x7b, 0x09, 0x68, 0xae, 0x6d, 0xb1, 0xf5, 0x8c, 0x78, 0x09, - 0xd0, 0x52, 0x2c, 0xa0, 0xe8, 0x01, 0x18, 0x6a, 0x10, 0xd7, 0xd5, 0xea, 0x84, 0x49, 0x05, 0x23, - 0xa1, 0x62, 0xb5, 0xc2, 0x8b, 0xb1, 0x0f, 0x47, 0x6f, 0x83, 0x01, 0xdd, 0xae, 0x11, 0x77, 0x66, - 0x88, 0x31, 0x3e, 0x7a, 0x06, 0x0c, 0x94, 0x68, 0xc1, 0xbd, 0x76, 0x71, 0x84, 0x5d, 0x9f, 0xd0, - 0x5f, 0x98, 0x57, 0x52, 0x7f, 0x42, 0x81, 0xc9, 0xb8, 0xb1, 0xac, 0x0b, 0xef, 0x86, 0x93, 0x73, - 0x14, 0x50, 0x3f, 0xa3, 0xc0, 0x18, 0xed, 0xa1, 0x63, 0x9b, 0x6b, 0xa6, 0x66, 0x11, 0xf4, 0x7d, - 0x0a, 0x4c, 0x6e, 0x1b, 0xf5, 0x6d, 0xd9, 0x8f, 0x4b, 0x88, 0xeb, 0xb9, 0x8c, 0x4d, 0x37, 0x62, - 0xb8, 0x16, 0xa6, 0xf7, 0xdb, 0xc5, 0xc9, 0x78, 0x29, 0x4e, 0xd0, 0x54, 0xff, 0xa2, 0x00, 0xe7, - 0xe4, 0x9e, 0xcd, 0x87, 0x2e, 0xf2, 0xe8, 0x8f, 0x14, 0x80, 0x86, 0x61, 0xcd, 0x9b, 0xa6, 0x7d, - 0x87, 0x39, 0x9f, 0x52, 0x05, 0xe8, 0xc5, 0xbc, 0x46, 0xce, 0x14, 0x0a, 0x73, 0x2b, 0x01, 0x76, - 0x7e, 0x31, 0xf9, 0xbc, 0x6f, 0x3c, 0x09, 0x01, 0xf7, 0xda, 0xc5, 0x62, 0xd2, 0x2f, 0x7f, 0x0e, - 0x0b, 0xe7, 0xf7, 0x65, 0xc3, 0xf5, 0x3e, 0xfa, 0xe7, 0x1d, 0xab, 0xf0, 0x1b, 0xca, 0x70, 0x20, - 0xb3, 0x0d, 0x38, 0x15, 0x23, 0x9c, 0xe2, 0x98, 0x5b, 0x96, 0x1d, 0x73, 0x0f, 0x60, 0x52, 0x73, - 0xbe, 0x2b, 0xfe, 0xdc, 0x33, 0x2d, 0xcd, 0xf2, 0xe8, 0x4c, 0x4b, 0x8e, 0xbc, 0x7f, 0x56, 0x80, - 0x69, 0x31, 0x01, 0x26, 0x55, 0x51, 0x9a, 0xa6, 0xbd, 0xd7, 0x20, 0xd6, 0x49, 0x38, 0x6b, 0xf9, - 0x1f, 0x41, 0x21, 0xf3, 0x23, 0x68, 0x24, 0x3e, 0x82, 0xbe, 0x3c, 0x1f, 0x41, 0xc0, 0x2b, 0x0e, - 0x30, 0xcc, 0x60, 0x38, 0x6b, 0x58, 0xb4, 0xa3, 0xd7, 0xd9, 0x86, 0x09, 0x5d, 0x2a, 0x19, 0x7f, - 0x1a, 0xe6, 0xba, 0x67, 0x25, 0xb5, 0x06, 0xce, 0x68, 0xa9, 0xfe, 0x95, 0x02, 0x33, 0x69, 0xf3, - 0x7b, 0x02, 0xb6, 0xd4, 0x46, 0xd4, 0x96, 0x7a, 0xa3, 0x87, 0x6f, 0x23, 0xd2, 0xf5, 0x0c, 0x9b, - 0xea, 0x5f, 0x16, 0xe0, 0x6c, 0x58, 0xbd, 0x62, 0xb9, 0x9e, 0x66, 0x9a, 0x5c, 0x2e, 0x3d, 0xfe, - 0xbd, 0xd4, 0x8c, 0x98, 0xc4, 0x57, 0x7b, 0x1b, 0xaa, 0xdc, 0xf7, 0x4c, 0xbf, 0x8e, 0xdd, 0x98, - 0x5f, 0xc7, 0xda, 0x11, 0xd2, 0xec, 0xec, 0xe2, 0xf1, 0x9f, 0x14, 0x98, 0x4d, 0x6f, 0x78, 0x02, - 0x9b, 0xca, 0x8e, 0x6e, 0xaa, 0xf7, 0x1e, 0xdd, 0xa8, 0x33, 0xb6, 0xd5, 0x2f, 0x14, 0xb2, 0x46, - 0xcb, 0xec, 0xea, 0x5b, 0x70, 0xca, 0x21, 0x75, 0xc3, 0xf5, 0x84, 0x03, 0xc2, 0xe1, 0x7c, 0xac, - 0xfd, 0xfb, 0xb4, 0x53, 0x38, 0x8a, 0x03, 0xc7, 0x91, 0xa2, 0x55, 0x18, 0x72, 0x09, 0xa9, 0x51, - 0xfc, 0x85, 0xee, 0xf1, 0x07, 0x42, 0x44, 0x95, 0xb7, 0xc5, 0x3e, 0x12, 0xf4, 0x1d, 0x30, 0x5e, - 0x0b, 0xbe, 0x28, 0x8a, 0xb5, 0xaf, 0x7b, 0xac, 0x4c, 0x45, 0x2b, 0xcb, 0xad, 0x71, 0x14, 0x99, - 0xfa, 0xb7, 0x0a, 0x5c, 0xec, 0xb4, 0xb7, 0xd0, 0x2b, 0x00, 0xba, 0x2f, 0x15, 0xba, 0xe2, 0xf0, - 0x7c, 0x22, 0xe7, 0x5a, 0x72, 0x2c, 0xe1, 0x07, 0x1a, 0x14, 0xb9, 0x58, 0x22, 0x92, 0xe2, 0xed, - 0x57, 0x38, 0x26, 0x6f, 0x3f, 0xf5, 0x3f, 0x2b, 0x32, 0x2b, 0x92, 0xd7, 0xf6, 0x8d, 0xc6, 0x8a, - 0xe4, 0xbe, 0x67, 0xde, 0xd3, 0x7d, 0xa5, 0x00, 0x57, 0xd2, 0x9b, 0x48, 0xe7, 0xf9, 0xd3, 0x30, - 0xd8, 0xe4, 0xef, 0x20, 0xb8, 0x4b, 0xf8, 0x55, 0xca, 0x59, 0xf8, 0x2b, 0x85, 0x7b, 0xed, 0xe2, - 0x6c, 0x1a, 0xa3, 0x17, 0xef, 0x1b, 0x44, 0x3b, 0x64, 0xc4, 0x2e, 0x14, 0xb8, 0xd0, 0xfe, 0x8e, - 0x2e, 0x99, 0x8b, 0xb6, 0x49, 0xcc, 0xae, 0xef, 0x10, 0x3e, 0xa2, 0xc0, 0x44, 0x64, 0x47, 0xbb, - 0x33, 0x03, 0x6c, 0x8f, 0xe6, 0x72, 0xb4, 0x8a, 0x7c, 0x2a, 0xa1, 0x34, 0x10, 0x29, 0x76, 0x71, - 0x8c, 0x60, 0x8c, 0xcd, 0xca, 0xb3, 0xfa, 0x86, 0x63, 0xb3, 0x72, 0xe7, 0x33, 0xd8, 0xec, 0x8f, - 0x17, 0xb2, 0x46, 0xcb, 0xd8, 0xec, 0x1d, 0x18, 0xf1, 0xc5, 0x48, 0x9f, 0x5d, 0x2c, 0xf5, 0xda, - 0x27, 0x8e, 0x6e, 0x61, 0x4a, 0xf4, 0x67, 0xc4, 0x2f, 0x71, 0x71, 0x48, 0x0b, 0x7d, 0xaf, 0x02, - 0x10, 0x2e, 0x8c, 0xf8, 0xa8, 0xd6, 0x8f, 0x6e, 0x3a, 0x24, 0xb1, 0x66, 0x82, 0x7e, 0xd2, 0xd2, - 0xa6, 0x90, 0xe8, 0xaa, 0xff, 0xa6, 0x1f, 0x50, 0xb2, 0xef, 0xdd, 0x5d, 0x17, 0x1f, 0x20, 0xe4, - 0x36, 0x61, 0xd2, 0xa1, 0xd2, 0xa2, 0x6e, 0x98, 0x4c, 0x65, 0xb5, 0x5b, 0x5e, 0x4e, 0xcb, 0x07, - 0x53, 0xab, 0x70, 0x0c, 0x17, 0x4e, 0x60, 0x47, 0x6f, 0x81, 0xa1, 0xa6, 0x63, 0x34, 0x34, 0x67, - 0x8f, 0x29, 0xc5, 0xc3, 0xfc, 0x2e, 0x6b, 0x8d, 0x17, 0x61, 0x1f, 0x86, 0x3e, 0x00, 0x23, 0xa6, - 0xb1, 0x45, 0xf4, 0x3d, 0xdd, 0x24, 0xc2, 0x54, 0x7e, 0xeb, 0x68, 0xd6, 0x7c, 0xd9, 0x47, 0x2b, - 0x3c, 0x10, 0xfd, 0x9f, 0x38, 0x24, 0x88, 0x2a, 0x70, 0xfa, 0x0e, 0xf3, 0x89, 0x31, 0x89, 0xeb, - 0x56, 0x5b, 0xcd, 0xa6, 0xed, 0x78, 0xa4, 0xc6, 0x0c, 0xea, 0xc3, 0xdc, 0xb0, 0xf7, 0x5c, 0x12, - 0x8c, 0xd3, 0xda, 0xa0, 0x27, 0x00, 0xb4, 0x96, 0x67, 0xf3, 0x07, 0x89, 0x33, 0xc3, 0x4c, 0x6b, - 0xbf, 0x44, 0x17, 0x7b, 0x3e, 0x28, 0xbd, 0xd7, 0x2e, 0x8e, 0x0a, 0xdb, 0x1f, 0x5b, 0x1a, 0xa9, - 0x01, 0x7a, 0x11, 0xa6, 0x75, 0x0e, 0x2a, 0xd9, 0x8d, 0xa6, 0xe6, 0x19, 0x42, 0x23, 0x1e, 0x61, - 0x88, 0xbe, 0x65, 0xbf, 0x5d, 0x9c, 0x2e, 0xa5, 0xc0, 0xe3, 0x28, 0x53, 0x91, 0xa8, 0x9f, 0x2c, - 0xc0, 0x85, 0x0e, 0x13, 0x84, 0x30, 0xfd, 0xf0, 0xc4, 0xfa, 0x89, 0x6d, 0xf6, 0x08, 0xff, 0x58, - 0x44, 0xe1, 0xbd, 0x76, 0xf1, 0xbe, 0x0e, 0x08, 0xaa, 0x74, 0x9f, 0x93, 0xfa, 0x1e, 0x0e, 0xd1, - 0xa0, 0x0a, 0x0c, 0xd6, 0xc2, 0xbb, 0xaf, 0x91, 0x85, 0x87, 0xe8, 0x51, 0xc0, 0xad, 0xd4, 0xdd, - 0x62, 0x13, 0x08, 0xd0, 0x32, 0x0c, 0x71, 0x9f, 0x4a, 0x22, 0x8e, 0x95, 0x87, 0x99, 0xc9, 0x84, - 0x17, 0x75, 0x8b, 0xcc, 0x47, 0xa1, 0xfe, 0x8d, 0x02, 0x43, 0x25, 0xdb, 0x21, 0xe5, 0xd5, 0x2a, - 0xda, 0x83, 0x51, 0xe9, 0x45, 0xbc, 0x60, 0xb1, 0x39, 0x79, 0x0e, 0xc3, 0x28, 0xa9, 0xf6, 0xfe, - 0x1b, 0xbe, 0xa0, 0x00, 0xcb, 0xb4, 0xd0, 0x2b, 0x74, 0xce, 0xef, 0x38, 0x86, 0x47, 0x09, 0xf7, - 0xe2, 0x64, 0xc3, 0x09, 0x63, 0x1f, 0x17, 0xdf, 0xed, 0xc1, 0x4f, 0x1c, 0x52, 0x51, 0xd7, 0x28, - 0x7b, 0x89, 0x77, 0x13, 0x3d, 0x0e, 0xfd, 0x0d, 0xbb, 0xe6, 0xaf, 0xfb, 0x5b, 0x7d, 0xe6, 0xb1, - 0x62, 0xd7, 0xe8, 0xdc, 0x9e, 0x4d, 0xb6, 0x60, 0xf7, 0x49, 0xac, 0x8d, 0xba, 0x0a, 0x93, 0x71, - 0xfa, 0xe8, 0x71, 0x98, 0xd0, 0xed, 0x46, 0xc3, 0xb6, 0xaa, 0xad, 0xad, 0x2d, 0x63, 0x97, 0x44, - 0x1e, 0x57, 0x96, 0x22, 0x10, 0x1c, 0xab, 0xa9, 0x7e, 0x5e, 0x81, 0x3e, 0xba, 0x2e, 0x2a, 0x0c, - 0xd6, 0xec, 0x86, 0x66, 0x58, 0xa2, 0x57, 0xec, 0x96, 0xba, 0xcc, 0x4a, 0xb0, 0x80, 0xa0, 0x26, - 0x8c, 0xf8, 0x12, 0x59, 0x4f, 0x6e, 0xe1, 0xe5, 0xd5, 0x6a, 0xf0, 0xaa, 0x26, 0x38, 0x26, 0xfc, - 0x12, 0x17, 0x87, 0x44, 0x54, 0x0d, 0xa6, 0xca, 0xab, 0xd5, 0x8a, 0xa5, 0x9b, 0xad, 0x1a, 0x59, - 0xdc, 0x65, 0x7f, 0x28, 0x9f, 0x33, 0x78, 0x89, 0x18, 0x27, 0xe3, 0x73, 0xa2, 0x12, 0xf6, 0x61, - 0xb4, 0x1a, 0xe1, 0x2d, 0xc4, 0x5b, 0x43, 0x56, 0x4d, 0x20, 0xc1, 0x3e, 0x4c, 0xfd, 0x6a, 0x01, - 0x46, 0xa5, 0x0e, 0x21, 0x13, 0x86, 0xf8, 0x70, 0xdd, 0x5e, 0xde, 0x93, 0x27, 0x7a, 0xcd, 0xa9, - 0xf3, 0x09, 0x75, 0xb1, 0x4f, 0x42, 0xe6, 0xd9, 0x85, 0x0e, 0x3c, 0x7b, 0x2e, 0xf2, 0x64, 0x93, - 0x7f, 0x92, 0x13, 0xd9, 0xcf, 0x35, 0xd1, 0x45, 0x71, 0x3c, 0x71, 0xbf, 0xec, 0xe1, 0xd8, 0xd1, - 0xb4, 0x05, 0x03, 0x77, 0x6d, 0x8b, 0xb8, 0xc2, 0x16, 0x7e, 0x44, 0x03, 0x64, 0xde, 0x74, 0x2f, - 0x50, 0xbc, 0x98, 0xa3, 0x57, 0xbf, 0xa0, 0x00, 0x94, 0x35, 0x4f, 0xe3, 0xae, 0x1b, 0x5d, 0x78, - 0x07, 0x5e, 0x8c, 0x9c, 0xaa, 0xc3, 0x89, 0x97, 0x61, 0xfd, 0xae, 0x71, 0xd7, 0x1f, 0x7e, 0x20, - 0xad, 0x73, 0xec, 0x55, 0xe3, 0x2e, 0xc1, 0x0c, 0x8e, 0x1e, 0x84, 0x11, 0x62, 0xe9, 0xce, 0x5e, - 0x93, 0x1e, 0x2c, 0xdc, 0xc4, 0xc3, 0xbe, 0xd0, 0x45, 0xbf, 0x10, 0x87, 0x70, 0xf5, 0x21, 0x88, - 0xaa, 0x5c, 0x5d, 0x38, 0x2f, 0xff, 0x9d, 0x02, 0xe7, 0xca, 0x2d, 0xcd, 0x9c, 0x6f, 0xd2, 0x8d, - 0xaa, 0x99, 0x4b, 0x36, 0xf7, 0x01, 0xa0, 0x7a, 0xc8, 0xdb, 0x60, 0xd8, 0x17, 0x72, 0xe2, 0x4f, - 0x47, 0x7d, 0x46, 0x89, 0x83, 0x1a, 0x48, 0x83, 0x61, 0xd7, 0x17, 0xbb, 0x0b, 0x3d, 0x88, 0xdd, - 0x3e, 0x89, 0x40, 0xec, 0x0e, 0xd0, 0x72, 0xe3, 0x17, 0x5b, 0xa0, 0x2a, 0x71, 0x76, 0x0c, 0x9d, - 0xcc, 0xeb, 0xba, 0xdd, 0xb2, 0x3c, 0x57, 0x3c, 0x05, 0x17, 0xc6, 0xaf, 0xb4, 0x1a, 0x38, 0xa3, - 0xa5, 0xfa, 0x65, 0x05, 0xfa, 0x17, 0xd7, 0x4b, 0x65, 0xf4, 0x1d, 0xd0, 0x1f, 0xb0, 0x8c, 0x9c, - 0xae, 0x3e, 0x14, 0x0f, 0xb7, 0xd3, 0xf1, 0xf5, 0x5e, 0xa1, 0x0c, 0x87, 0x61, 0x45, 0x9b, 0x30, - 0x48, 0x76, 0x08, 0xed, 0x6a, 0xe1, 0x48, 0xf0, 0x33, 0x96, 0xb6, 0xc8, 0x30, 0x62, 0x81, 0x59, - 0xfd, 0x94, 0x02, 0x10, 0x56, 0x41, 0xdf, 0x9d, 0x76, 0x3a, 0xdd, 0x3c, 0x42, 0xeb, 0x73, 0xe7, - 0x23, 0x4a, 0xfd, 0xd5, 0x01, 0x38, 0x4f, 0xbb, 0x23, 0xb6, 0xaa, 0x61, 0x5b, 0x37, 0xc9, 0xde, - 0x37, 0xdd, 0xe4, 0xbf, 0xe9, 0x26, 0x7f, 0x84, 0x6e, 0xf2, 0x2f, 0xc2, 0x79, 0xba, 0xe5, 0x04, - 0x98, 0x30, 0x9f, 0xa1, 0x35, 0x87, 0x34, 0x35, 0x87, 0xd4, 0x98, 0x3e, 0x30, 0xcc, 0xbd, 0xa0, - 0xe6, 0xb3, 0x2a, 0xe1, 0xec, 0xf6, 0xea, 0x53, 0x30, 0x19, 0xee, 0x5d, 0xf1, 0x51, 0x3d, 0x18, - 0x57, 0x32, 0x47, 0x7c, 0x89, 0x29, 0xa9, 0x18, 0xaa, 0xf7, 0x14, 0x98, 0x5c, 0xdc, 0x6d, 0x1a, - 0x0e, 0x7b, 0x9f, 0xcf, 0x9f, 0x99, 0xa0, 0x07, 0xc2, 0xd7, 0x28, 0x4a, 0xf4, 0x16, 0x2f, 0xfe, - 0x22, 0x05, 0x6d, 0xc1, 0x04, 0x61, 0xcd, 0x99, 0x16, 0xa8, 0x79, 0x79, 0xb6, 0x37, 0x0f, 0x4a, - 0x11, 0xc1, 0x82, 0x63, 0x58, 0x51, 0x15, 0x26, 0x74, 0x53, 0x73, 0x5d, 0x63, 0xcb, 0xd0, 0xc3, - 0x57, 0x54, 0x23, 0x0b, 0x0f, 0x32, 0x99, 0x2b, 0x02, 0xb9, 0xd7, 0x2e, 0x9e, 0x11, 0xfd, 0x8c, - 0x02, 0x70, 0x0c, 0x85, 0xfa, 0xd9, 0x02, 0x8c, 0x2f, 0xee, 0x36, 0x6d, 0xb7, 0xe5, 0x10, 0x56, - 0xf5, 0x04, 0xec, 0x5a, 0x0f, 0xc0, 0xd0, 0xb6, 0x66, 0xd5, 0x4c, 0xe2, 0x88, 0x63, 0x37, 0x98, - 0xdb, 0x1b, 0xbc, 0x18, 0xfb, 0x70, 0xf4, 0x2a, 0x80, 0xab, 0x6f, 0x93, 0x5a, 0x8b, 0x31, 0xc7, - 0xbe, 0xfc, 0xcc, 0x31, 0x32, 0xc6, 0x6a, 0x80, 0x52, 0x88, 0x34, 0xc1, 0x6f, 0x2c, 0x91, 0x53, - 0xff, 0x58, 0x81, 0xa9, 0x48, 0xbb, 0x13, 0x30, 0xd7, 0x6c, 0x45, 0xcd, 0x35, 0xf3, 0x3d, 0x8f, - 0x35, 0xc3, 0x4a, 0xf3, 0xf1, 0x02, 0x9c, 0xcb, 0x98, 0x93, 0x84, 0xbf, 0xaf, 0x72, 0x42, 0xfe, - 0xbe, 0x2d, 0x18, 0xf5, 0x6c, 0x53, 0x3c, 0xf6, 0xf3, 0x67, 0x20, 0xd7, 0x11, 0xbc, 0x1e, 0xa0, - 0x09, 0xbd, 0x79, 0xc3, 0x32, 0x17, 0xcb, 0x74, 0xd4, 0x5f, 0x57, 0x60, 0x24, 0xb0, 0x0a, 0xbf, - 0xae, 0x2e, 0xd4, 0xbb, 0x8f, 0xa3, 0xa3, 0xfe, 0x4e, 0x01, 0xce, 0x06, 0xb8, 0x7d, 0x36, 0x57, - 0xf5, 0x28, 0xdf, 0x38, 0xd8, 0xb4, 0x74, 0x31, 0xf2, 0x12, 0x61, 0x38, 0xf9, 0x30, 0xaf, 0xd9, - 0x72, 0x9a, 0xb6, 0xeb, 0xcb, 0xc1, 0x5c, 0x61, 0xe0, 0x45, 0xd8, 0x87, 0xa1, 0x55, 0x18, 0x70, - 0x29, 0x3d, 0x71, 0xd6, 0x1d, 0x72, 0x36, 0x98, 0x28, 0xcf, 0xfa, 0x8b, 0x39, 0x1a, 0xf4, 0xaa, - 0xcc, 0xc3, 0x07, 0xf2, 0x1b, 0x2f, 0xe9, 0x48, 0x6a, 0x81, 0x24, 0x1c, 0x5e, 0x7d, 0x74, 0x32, - 0x16, 0xaa, 0xcb, 0x30, 0x29, 0x9c, 0x5a, 0xf9, 0xb6, 0xb1, 0x74, 0x82, 0xde, 0x1d, 0xd9, 0x19, - 0xf7, 0xc7, 0x5c, 0x6a, 0xa6, 0xe3, 0xf5, 0xc3, 0x1d, 0xa3, 0xba, 0x30, 0x7c, 0x5d, 0x74, 0x12, - 0xcd, 0x42, 0xc1, 0xf0, 0xd7, 0x02, 0x04, 0x8e, 0x42, 0xa5, 0x8c, 0x0b, 0x46, 0x17, 0x2f, 0x42, - 0xe4, 0x63, 0xa9, 0xaf, 0xf3, 0xb1, 0xa4, 0x7e, 0xbd, 0x00, 0xd3, 0x3e, 0x55, 0x7f, 0x8c, 0x65, - 0x71, 0x5b, 0x7e, 0x80, 0x52, 0x74, 0xb0, 0xa9, 0xf1, 0x16, 0xf4, 0x33, 0x06, 0x98, 0xeb, 0x16, - 0x3d, 0x40, 0x48, 0xbb, 0x83, 0x19, 0x22, 0xf4, 0x01, 0x18, 0x34, 0xa9, 0x86, 0xe1, 0x3f, 0xd5, - 0xc8, 0x65, 0x98, 0x4d, 0x1b, 0x2e, 0x57, 0x5c, 0x44, 0x08, 0xb3, 0xe0, 0x22, 0x94, 0x17, 0x62, - 0x41, 0x73, 0xf6, 0x31, 0x18, 0x95, 0xaa, 0x1d, 0x2a, 0x7e, 0xd9, 0xe7, 0x0b, 0x30, 0x73, 0x83, - 0x98, 0x8d, 0x54, 0xd7, 0x87, 0x22, 0x0c, 0xe8, 0xdb, 0x9a, 0xc3, 0x43, 0xe3, 0x8d, 0xf1, 0x4d, - 0x5e, 0xa2, 0x05, 0x98, 0x97, 0x53, 0x85, 0x23, 0xf2, 0x38, 0xed, 0x49, 0x69, 0x26, 0xc3, 0x98, - 0x89, 0xef, 0x0f, 0x82, 0x2a, 0x86, 0x03, 0x8f, 0x54, 0xa0, 0xc7, 0xcb, 0x7b, 0xab, 0xb7, 0x56, - 0xd3, 0x9e, 0xa5, 0xa1, 0xbb, 0x30, 0x6e, 0xeb, 0x06, 0x26, 0x4d, 0xdb, 0x35, 0x3c, 0xdb, 0xd9, - 0x13, 0x8b, 0x96, 0xeb, 0x68, 0xb9, 0x55, 0xaa, 0x84, 0x88, 0xf8, 0xf5, 0x61, 0xa4, 0x08, 0x47, - 0x49, 0xa9, 0x5f, 0x52, 0x60, 0xf4, 0x86, 0xb1, 0x49, 0x1c, 0xee, 0xb7, 0xcb, 0x2c, 0x24, 0x91, - 0x20, 0x6f, 0xa3, 0x69, 0x01, 0xde, 0xd0, 0x2e, 0x8c, 0x88, 0x73, 0x38, 0x78, 0x75, 0x78, 0x3d, - 0x9f, 0xbf, 0x50, 0x40, 0x5a, 0x9c, 0x6f, 0xd2, 0x87, 0xef, 0x97, 0xb8, 0x38, 0x24, 0xa6, 0xbe, - 0x0a, 0xa7, 0x53, 0x1a, 0xd1, 0x85, 0x64, 0xae, 0xab, 0xe2, 0xa3, 0xf1, 0xb9, 0x15, 0x5d, 0x48, - 0x56, 0x8e, 0xce, 0x43, 0x1f, 0xb1, 0x6a, 0xe2, 0x8b, 0x19, 0xda, 0x6f, 0x17, 0xfb, 0x16, 0xad, - 0x1a, 0xa6, 0x65, 0x94, 0x89, 0x9b, 0x76, 0x44, 0x62, 0x63, 0x4c, 0x7c, 0x59, 0x94, 0xe1, 0x00, - 0xca, 0x3c, 0xbc, 0xe2, 0xce, 0x4c, 0x54, 0xb3, 0x98, 0xdc, 0x8a, 0xf1, 0x96, 0x5e, 0x7c, 0xa8, - 0xe2, 0x7c, 0x6a, 0x61, 0x46, 0x4c, 0x48, 0x82, 0xe3, 0xe1, 0x04, 0x5d, 0xf5, 0x97, 0xfb, 0xe1, - 0xd2, 0x0d, 0xdb, 0x31, 0xee, 0xda, 0x96, 0xa7, 0x99, 0x6b, 0x76, 0x2d, 0xf4, 0x6f, 0x15, 0x47, - 0xd6, 0xc7, 0x14, 0x38, 0xa7, 0x37, 0x5b, 0x5c, 0x33, 0xf1, 0x5d, 0x44, 0xd7, 0x88, 0x63, 0xd8, - 0x79, 0x1f, 0x6a, 0x30, 0x77, 0xde, 0xd2, 0xda, 0x46, 0x1a, 0x4a, 0x9c, 0x45, 0x8b, 0xbd, 0x17, - 0xa9, 0xd9, 0x77, 0x2c, 0xd6, 0xb9, 0xaa, 0xc7, 0x66, 0xf3, 0x6e, 0xb8, 0x08, 0x39, 0xdf, 0x8b, - 0x94, 0x53, 0x31, 0xe2, 0x0c, 0x4a, 0xe8, 0x43, 0x70, 0xc6, 0xe0, 0x9d, 0xc3, 0x44, 0xab, 0x19, - 0x16, 0x71, 0x5d, 0xee, 0x6c, 0xde, 0xc3, 0x83, 0x88, 0x4a, 0x1a, 0x42, 0x9c, 0x4e, 0x07, 0xbd, - 0x04, 0xe0, 0xee, 0x59, 0xba, 0x98, 0xff, 0x7c, 0x8e, 0xa8, 0x5c, 0x44, 0x0e, 0xb0, 0x60, 0x09, - 0x23, 0x55, 0xb4, 0xbc, 0x60, 0x53, 0x0e, 0x32, 0x67, 0x62, 0xa6, 0x68, 0x85, 0x7b, 0x28, 0x84, - 0xab, 0x9f, 0x52, 0x60, 0xa2, 0x62, 0xad, 0x99, 0x9a, 0x4e, 0xb8, 0x5f, 0xa5, 0x8b, 0xae, 0xc1, - 0x88, 0x1b, 0xdc, 0xc8, 0x70, 0x8e, 0x10, 0x7e, 0x9f, 0xc1, 0x5d, 0x4c, 0x58, 0x87, 0x79, 0x69, - 0x1b, 0x96, 0x38, 0xec, 0x96, 0x6c, 0x87, 0x23, 0x12, 0xdf, 0x1d, 0xf7, 0xd2, 0x4e, 0x82, 0x71, - 0x5a, 0x1b, 0xf5, 0xe7, 0x15, 0x98, 0x8e, 0x76, 0x47, 0xb8, 0x34, 0xfc, 0x98, 0x02, 0xd3, 0x91, - 0xc7, 0xd4, 0x02, 0xdc, 0x4b, 0x7c, 0xa7, 0xb5, 0x14, 0x7c, 0xdc, 0xc5, 0x37, 0x0d, 0x82, 0x53, - 0xe9, 0xab, 0x3f, 0xa7, 0xc0, 0x90, 0x88, 0xf5, 0x88, 0xde, 0x1a, 0xb3, 0x9e, 0x07, 0x47, 0x5b, - 0xcc, 0x82, 0xbe, 0xc7, 0xfc, 0x33, 0xc4, 0xd1, 0x24, 0x4e, 0x99, 0x5c, 0xe6, 0x57, 0x41, 0x38, - 0x3c, 0xe7, 0x22, 0x7e, 0x1a, 0xfe, 0xd5, 0x8c, 0x44, 0x4c, 0x7d, 0x4d, 0x81, 0xa9, 0x44, 0xab, - 0x2e, 0xc4, 0xd1, 0x13, 0xf4, 0x58, 0xfd, 0x4a, 0x3f, 0xdd, 0x92, 0x1e, 0xe5, 0xf7, 0x26, 0x37, - 0x6c, 0x9f, 0x80, 0xfe, 0xfb, 0x20, 0x8c, 0x18, 0x8d, 0x46, 0xcb, 0x63, 0x97, 0x88, 0x03, 0xa1, - 0xb5, 0xb8, 0xe2, 0x17, 0xe2, 0x10, 0x8e, 0x2c, 0x21, 0x69, 0xf1, 0x53, 0x70, 0x39, 0xdf, 0xca, - 0xc9, 0x03, 0x9c, 0xa3, 0x52, 0x11, 0x17, 0x87, 0xd2, 0x04, 0xb1, 0xef, 0x53, 0x00, 0x5c, 0xcf, - 0x31, 0xac, 0x3a, 0x2d, 0x14, 0xd2, 0x18, 0x3e, 0x02, 0xb2, 0xd5, 0x00, 0x29, 0x27, 0x1e, 0xc6, - 0x7f, 0x0c, 0x00, 0x58, 0xa2, 0x8c, 0xe6, 0x85, 0x10, 0xca, 0x8f, 0xcc, 0xb7, 0xc7, 0xc4, 0xed, - 0x4b, 0x29, 0x9e, 0xb0, 0x9c, 0x50, 0x28, 0xa5, 0xce, 0x3e, 0x0a, 0x23, 0x01, 0xbd, 0x83, 0x84, - 0xba, 0x31, 0x49, 0xa8, 0x9b, 0x7d, 0x02, 0x4e, 0xc5, 0xba, 0x7b, 0x28, 0x99, 0xf0, 0x4f, 0x14, - 0x40, 0xd1, 0xd1, 0x9f, 0x80, 0xe5, 0xa0, 0x1e, 0xb5, 0x1c, 0x2c, 0xf4, 0xbe, 0x64, 0x19, 0xa6, - 0x83, 0x9f, 0x41, 0xc0, 0x42, 0xe1, 0x06, 0xa1, 0xa1, 0xc5, 0xc9, 0x4f, 0x05, 0x95, 0xf0, 0x05, - 0xb8, 0xf8, 0x72, 0x7b, 0x10, 0x54, 0x6e, 0xc6, 0x70, 0x85, 0x82, 0x4a, 0x1c, 0x82, 0x13, 0x74, - 0xd1, 0x27, 0x14, 0x98, 0xd4, 0xa2, 0xa1, 0x70, 0xfd, 0x99, 0xc9, 0x15, 0xa0, 0x2b, 0x16, 0x56, - 0x37, 0xec, 0x4b, 0x0c, 0xe0, 0xe2, 0x04, 0x59, 0xf4, 0x08, 0x8c, 0x69, 0x4d, 0x63, 0xbe, 0x55, - 0x33, 0xa8, 0xe6, 0xe9, 0x47, 0x0c, 0x65, 0xd6, 0x90, 0xf9, 0xb5, 0x4a, 0x50, 0x8e, 0x23, 0xb5, - 0x82, 0x98, 0xb3, 0xa5, 0xd0, 0x71, 0xb8, 0x97, 0x98, 0xb3, 0x62, 0x0e, 0xc3, 0x98, 0xb3, 0x62, - 0xea, 0x64, 0x22, 0xc8, 0x02, 0xb0, 0x8d, 0x9a, 0x2e, 0x48, 0x0e, 0xe6, 0xbf, 0x03, 0xb9, 0x55, - 0x29, 0x97, 0x04, 0x45, 0x26, 0x3e, 0x84, 0xbf, 0xb1, 0x44, 0x01, 0x7d, 0x46, 0x81, 0x71, 0xc1, - 0xbb, 0x05, 0xcd, 0x21, 0xb6, 0x44, 0x2f, 0xe4, 0xdd, 0x2f, 0xb1, 0x3d, 0x39, 0x87, 0x65, 0xe4, - 0x9c, 0xef, 0x04, 0x01, 0x04, 0x22, 0x30, 0x1c, 0xed, 0x07, 0x93, 0x01, 0xdc, 0xc8, 0x25, 0x94, - 0xe8, 0xe0, 0x70, 0x7e, 0x19, 0xa0, 0x9a, 0x82, 0x4f, 0x3c, 0x32, 0x4a, 0x81, 0xe0, 0x54, 0xfa, - 0x54, 0xae, 0x3d, 0x75, 0x47, 0xf3, 0xf4, 0xed, 0x92, 0xa6, 0x6f, 0xb3, 0x3b, 0x48, 0xfe, 0x9c, - 0x32, 0xe7, 0xbe, 0x7e, 0x2e, 0x8a, 0x6a, 0xe1, 0xf4, 0x7e, 0xbb, 0x78, 0x2a, 0x56, 0x88, 0xe3, - 0x04, 0x91, 0x0d, 0xc3, 0x8e, 0x88, 0x07, 0x2f, 0x5e, 0xc0, 0xe7, 0x0b, 0x81, 0x1e, 0x0f, 0x2e, - 0xcf, 0x35, 0x23, 0xff, 0x17, 0x0e, 0x88, 0xa0, 0x3a, 0x5c, 0xe2, 0xba, 0xe1, 0xbc, 0x65, 0x5b, - 0x7b, 0x0d, 0xbb, 0xe5, 0xce, 0xb7, 0xbc, 0x6d, 0x62, 0x79, 0xbe, 0x29, 0x7c, 0x94, 0x1d, 0xa3, - 0xec, 0xd1, 0xdc, 0x62, 0xa7, 0x8a, 0xb8, 0x33, 0x1e, 0xf4, 0x3c, 0x0c, 0xb3, 0x7b, 0xba, 0xf5, - 0xf5, 0x65, 0xf6, 0x32, 0xf3, 0xf0, 0xe2, 0x32, 0x1b, 0xc2, 0xa2, 0xc0, 0x81, 0x03, 0x6c, 0xe8, - 0x36, 0x0c, 0x99, 0x3c, 0xa0, 0x3f, 0x7b, 0xa1, 0x99, 0x93, 0x29, 0xc6, 0x93, 0x03, 0x70, 0x05, - 0x5a, 0xfc, 0xc0, 0x3e, 0x05, 0xd4, 0x84, 0x2b, 0x35, 0xb2, 0xa5, 0xb5, 0x4c, 0x6f, 0xd5, 0xf6, - 0x30, 0x7b, 0xa1, 0x16, 0x58, 0x3c, 0xfd, 0x47, 0xb8, 0x13, 0x2c, 0x06, 0x1b, 0x7b, 0xfb, 0x57, - 0x3e, 0xa0, 0x2e, 0x3e, 0x10, 0x1b, 0xda, 0x83, 0xfb, 0x44, 0x1d, 0xf6, 0x24, 0x4e, 0xdf, 0xa6, - 0xb3, 0x9c, 0x24, 0x7a, 0x8a, 0x11, 0xfd, 0x96, 0xfd, 0x76, 0xf1, 0xbe, 0xf2, 0xc1, 0xd5, 0x71, - 0x37, 0x38, 0xd9, 0x2b, 0x23, 0x12, 0xbb, 0x02, 0x9a, 0x99, 0xcc, 0x3f, 0xc7, 0xf1, 0xeb, 0x24, - 0xee, 0x0e, 0x17, 0x2f, 0xc5, 0x09, 0x9a, 0xe8, 0x8b, 0x0a, 0xcc, 0xb8, 0x9e, 0xd3, 0xd2, 0xbd, - 0x96, 0x43, 0x6a, 0xb1, 0x1d, 0x3a, 0xc5, 0x3a, 0x94, 0x4b, 0x80, 0xab, 0x66, 0xe0, 0x64, 0xcf, - 0xc1, 0x67, 0xb2, 0xa0, 0x38, 0xb3, 0x2f, 0xe8, 0x27, 0x15, 0x38, 0x17, 0x05, 0x52, 0x9d, 0x9e, - 0xf7, 0x13, 0xe5, 0xbf, 0x64, 0xa9, 0xa6, 0xa3, 0xe4, 0x1a, 0x7c, 0x06, 0x10, 0x67, 0x75, 0x24, - 0x7e, 0x33, 0x7e, 0xfa, 0x84, 0x6f, 0xc6, 0x67, 0x9f, 0x06, 0x94, 0x3c, 0x3e, 0x0e, 0x92, 0x03, - 0x87, 0x65, 0x39, 0xf0, 0x73, 0x03, 0x70, 0x81, 0x9e, 0x4a, 0xa1, 0xf6, 0xb3, 0xa2, 0x59, 0x5a, - 0xfd, 0xf5, 0x29, 0x31, 0x7d, 0x49, 0x81, 0x73, 0xdb, 0xe9, 0xa6, 0x1d, 0xa1, 0x7f, 0x3d, 0x93, - 0xcb, 0x04, 0xd7, 0xc9, 0x5a, 0xc4, 0x19, 0x76, 0xc7, 0x2a, 0x38, 0xab, 0x53, 0xe8, 0x69, 0x98, - 0xb4, 0xec, 0x1a, 0x29, 0x55, 0xca, 0x78, 0x45, 0x73, 0x6f, 0x57, 0x7d, 0x47, 0x9d, 0x01, 0xfe, - 0xbd, 0xae, 0xc6, 0x60, 0x38, 0x51, 0x1b, 0xed, 0x00, 0x6a, 0xda, 0xb5, 0xc5, 0x1d, 0x9e, 0xf8, - 0xa2, 0x37, 0x97, 0x59, 0xe6, 0x2c, 0xb0, 0x96, 0xc0, 0x86, 0x53, 0x28, 0x30, 0xdb, 0x14, 0xed, - 0xcc, 0x8a, 0x6d, 0x19, 0x9e, 0xed, 0xb0, 0x00, 0x07, 0x3d, 0x99, 0x68, 0x98, 0x6d, 0x6a, 0x35, - 0x15, 0x23, 0xce, 0xa0, 0xa4, 0xfe, 0x77, 0x05, 0x4e, 0xd1, 0x6d, 0xb1, 0xe6, 0xd8, 0xbb, 0x7b, - 0xaf, 0xc7, 0x0d, 0xf9, 0x80, 0xf0, 0x59, 0xe4, 0xb6, 0x9d, 0x33, 0x92, 0xbf, 0xe2, 0x08, 0xeb, - 0x73, 0xe8, 0xa2, 0x28, 0x9b, 0x95, 0xfb, 0xb2, 0xcd, 0xca, 0xea, 0x67, 0x0a, 0x5c, 0x73, 0xf1, - 0xcd, 0xba, 0xaf, 0xcb, 0xef, 0xf0, 0x51, 0x18, 0xa7, 0x65, 0x2b, 0xda, 0xee, 0x5a, 0xf9, 0x59, - 0xdb, 0xf4, 0x5f, 0x63, 0x33, 0x5b, 0xfb, 0x4d, 0x19, 0x80, 0xa3, 0xf5, 0xd0, 0xe3, 0x30, 0xd4, - 0xe4, 0x71, 0xc2, 0x84, 0xce, 0x7c, 0x85, 0x3b, 0xf6, 0xb1, 0xa2, 0x7b, 0xed, 0xe2, 0x54, 0x78, - 0xc5, 0xeb, 0x47, 0x2b, 0xf3, 0x1b, 0xa8, 0xbf, 0x7c, 0x16, 0x18, 0x72, 0x93, 0x78, 0xaf, 0xc7, - 0x39, 0x79, 0x08, 0x46, 0xf5, 0x66, 0xab, 0xb4, 0x54, 0x7d, 0xa6, 0x65, 0x33, 0x5b, 0x08, 0xcb, - 0x0f, 0x43, 0xb9, 0x77, 0x69, 0x6d, 0xc3, 0x2f, 0xc6, 0x72, 0x1d, 0xca, 0x1d, 0xf4, 0x66, 0x4b, - 0xf0, 0xdb, 0x35, 0xf9, 0xbd, 0x0a, 0xe3, 0x0e, 0xa5, 0xb5, 0x8d, 0x08, 0x0c, 0x27, 0x6a, 0xa3, - 0x0f, 0xc1, 0x18, 0x11, 0x1f, 0xee, 0x0d, 0xcd, 0xa9, 0x09, 0xbe, 0x50, 0xc9, 0x3b, 0xf8, 0x60, - 0x6a, 0x7d, 0x6e, 0xc0, 0x35, 0xc0, 0x45, 0x89, 0x04, 0x8e, 0x10, 0x44, 0x2f, 0xc2, 0x79, 0xff, - 0x37, 0x5d, 0x65, 0xbb, 0x16, 0x67, 0x14, 0x03, 0xdc, 0x6d, 0x66, 0x31, 0xab, 0x12, 0xce, 0x6e, - 0x8f, 0x7e, 0x56, 0x81, 0xb3, 0x01, 0xd4, 0xb0, 0x8c, 0x46, 0xab, 0x81, 0x89, 0x6e, 0x6a, 0x46, - 0x43, 0xe8, 0x7d, 0xcf, 0x1d, 0xd9, 0x40, 0xa3, 0xe8, 0x39, 0xb3, 0x4a, 0x87, 0xe1, 0x8c, 0x2e, - 0xa1, 0xd7, 0x14, 0xb8, 0xe2, 0x83, 0xd6, 0x1c, 0xe2, 0xba, 0x2d, 0x87, 0x84, 0xb1, 0x00, 0xc4, - 0x94, 0x0c, 0xe5, 0xe2, 0x9d, 0x4c, 0x00, 0x5e, 0x3c, 0x00, 0x37, 0x3e, 0x90, 0xba, 0xbc, 0x5d, - 0xaa, 0xf6, 0x96, 0x27, 0x14, 0xc5, 0xe3, 0xda, 0x2e, 0x94, 0x04, 0x8e, 0x10, 0x44, 0x3f, 0xaf, - 0xc0, 0x39, 0xb9, 0x40, 0xde, 0x2d, 0x5c, 0x43, 0x7c, 0xfe, 0xc8, 0x3a, 0x13, 0xc3, 0xcf, 0x25, - 0xbc, 0x0c, 0x20, 0xce, 0xea, 0x15, 0x65, 0xdb, 0x0d, 0xb6, 0x31, 0xb9, 0x16, 0x39, 0xc0, 0xd9, - 0x36, 0xdf, 0xab, 0x2e, 0xf6, 0x61, 0xe8, 0x11, 0x18, 0x6b, 0xda, 0xb5, 0x35, 0xa3, 0xe6, 0xb2, - 0x18, 0x66, 0x4c, 0xd7, 0xeb, 0xe3, 0xd3, 0xb1, 0x66, 0xd7, 0xd6, 0x2a, 0x65, 0x5e, 0x8e, 0x23, - 0xb5, 0xd0, 0x1c, 0xc0, 0x96, 0x66, 0x98, 0xd5, 0x3b, 0x5a, 0xf3, 0x96, 0x1f, 0x14, 0x87, 0xd9, - 0x22, 0x96, 0x82, 0x52, 0x2c, 0xd5, 0xa0, 0xeb, 0x47, 0xf9, 0x0e, 0x26, 0x3c, 0xec, 0x34, 0x53, - 0x8f, 0x8e, 0x62, 0xfd, 0x7c, 0x84, 0xbc, 0xc3, 0x37, 0x25, 0x12, 0x38, 0x42, 0x10, 0x7d, 0x4c, - 0x81, 0x09, 0x77, 0xcf, 0xf5, 0x48, 0x23, 0xe8, 0xc3, 0xa9, 0xa3, 0xee, 0x03, 0xb3, 0x89, 0x57, - 0x23, 0x44, 0x70, 0x8c, 0x28, 0x0b, 0x2f, 0xd4, 0xd0, 0xea, 0xe4, 0x7a, 0xe9, 0x86, 0x51, 0xdf, - 0x0e, 0xa2, 0xbb, 0xac, 0x11, 0x47, 0x27, 0x96, 0xc7, 0x14, 0xab, 0x01, 0x11, 0x5e, 0x28, 0xbb, - 0x1a, 0xee, 0x84, 0x03, 0xbd, 0x04, 0xb3, 0x02, 0xbc, 0x6c, 0xdf, 0x49, 0x50, 0xe0, 0x51, 0x7c, - 0x98, 0x03, 0x64, 0x25, 0xb3, 0x16, 0xee, 0x80, 0x01, 0x55, 0xe0, 0xb4, 0x4b, 0x1c, 0x76, 0x27, - 0xc8, 0x23, 0x42, 0xae, 0xb5, 0x4c, 0xd3, 0x65, 0xaa, 0x8d, 0x78, 0xf2, 0x53, 0x4d, 0x82, 0x71, - 0x5a, 0x1b, 0xf4, 0x44, 0xf0, 0x2c, 0x78, 0x8f, 0x16, 0x3c, 0xb3, 0x56, 0x65, 0x9a, 0xc8, 0x00, - 0xb7, 0x94, 0xe0, 0x28, 0x08, 0xc7, 0xeb, 0xd2, 0xd3, 0xdc, 0x2f, 0x5a, 0x68, 0x39, 0xae, 0x37, + 0xb7, 0xa4, 0xae, 0xb9, 0x25, 0xcd, 0x8c, 0x67, 0x60, 0x4c, 0x2a, 0xeb, 0xaa, 0x94, 0xd3, 0x59, + 0x99, 0x35, 0x99, 0x59, 0x6a, 0xa9, 0xc7, 0xc6, 0x0f, 0x30, 0x9f, 0x6d, 0x0c, 0x1f, 0xf0, 0x01, + 0xfe, 0x6c, 0xb3, 0xc1, 0x60, 0x02, 0xb3, 0xbb, 0xb0, 0x66, 0x03, 0x82, 0x8d, 0x00, 0x02, 0x76, + 0x21, 0x82, 0xc5, 0x10, 0x40, 0x10, 0x18, 0x76, 0xcd, 0x02, 0x02, 0x6b, 0xbd, 0xb0, 0x11, 0xfb, + 0x8c, 0x25, 0x62, 0x89, 0xe8, 0x25, 0xd8, 0x8d, 0xfb, 0xc8, 0xcc, 0x9b, 0xaf, 0x52, 0x29, 0x4b, + 0x92, 0x67, 0x16, 0xff, 0x92, 0xea, 0x9e, 0x7b, 0xcf, 0xb9, 0xaf, 0x3c, 0xf7, 0x9c, 0x73, 0xcf, + 0x3d, 0x07, 0x16, 0x1a, 0x86, 0xb7, 0xd5, 0xde, 0x98, 0xd3, 0xed, 0xe6, 0xb5, 0x86, 0xe6, 0xd4, + 0x89, 0x45, 0x9c, 0xf0, 0x9f, 0xd6, 0xed, 0xc6, 0x35, 0xad, 0x65, 0xb8, 0xd7, 0x74, 0xdb, 0x21, + 0xd7, 0xb6, 0x1f, 0xda, 0x20, 0x9e, 0xf6, 0xd0, 0xb5, 0x06, 0x85, 0x69, 0x1e, 0xa9, 0xcf, 0xb5, + 0x1c, 0xdb, 0xb3, 0xd1, 0xc3, 0x21, 0x8e, 0x39, 0xbf, 0x69, 0xf8, 0x4f, 0xeb, 0x76, 0x63, 0x8e, + 0xe2, 0x98, 0xa3, 0x38, 0xe6, 0x04, 0x8e, 0xd9, 0xb7, 0xcb, 0x74, 0xed, 0x86, 0x7d, 0x8d, 0xa1, + 0xda, 0x68, 0x6f, 0xb2, 0x5f, 0xec, 0x07, 0xfb, 0x8f, 0x93, 0x98, 0x7d, 0xe0, 0xf6, 0xbb, 0xdd, + 0x39, 0xc3, 0xa6, 0x9d, 0xb9, 0xa6, 0xb5, 0x3d, 0xdb, 0xd5, 0x35, 0xd3, 0xb0, 0x1a, 0xd7, 0xb6, + 0x13, 0xbd, 0x99, 0x55, 0xa5, 0xaa, 0xa2, 0xdb, 0x1d, 0xeb, 0x38, 0x1b, 0x9a, 0x9e, 0x56, 0xe7, + 0x46, 0x58, 0x87, 0xec, 0x78, 0xc4, 0x72, 0x0d, 0xdb, 0x72, 0xdf, 0x4e, 0x47, 0x42, 0x9c, 0x6d, + 0x79, 0x6e, 0x22, 0x15, 0xd2, 0x30, 0x3d, 0x12, 0x62, 0x6a, 0x6a, 0xfa, 0x96, 0x61, 0x11, 0x67, + 0xd7, 0x6f, 0x7e, 0xcd, 0x21, 0xae, 0xdd, 0x76, 0x74, 0x72, 0xa8, 0x56, 0xee, 0xb5, 0x26, 0xf1, + 0xb4, 0x34, 0x5a, 0xd7, 0xb2, 0x5a, 0x39, 0x6d, 0xcb, 0x33, 0x9a, 0x49, 0x32, 0xef, 0x3a, 0xa8, + 0x81, 0xab, 0x6f, 0x91, 0xa6, 0x96, 0x68, 0xf7, 0x8e, 0xac, 0x76, 0x6d, 0xcf, 0x30, 0xaf, 0x19, + 0x96, 0xe7, 0x7a, 0x4e, 0xbc, 0x91, 0xfa, 0x49, 0x05, 0x26, 0xe7, 0xab, 0x95, 0x1a, 0x9b, 0xc1, + 0x65, 0xbb, 0xd1, 0x30, 0xac, 0x06, 0x7a, 0x10, 0x46, 0xb6, 0x89, 0xb3, 0x61, 0xbb, 0x86, 0xb7, + 0x3b, 0xa3, 0x5c, 0x51, 0xae, 0x0e, 0x2c, 0x8c, 0xef, 0xef, 0x15, 0x47, 0x9e, 0xf5, 0x0b, 0x71, + 0x08, 0x47, 0x15, 0x38, 0xbd, 0xe5, 0x79, 0xad, 0x79, 0x5d, 0x27, 0xae, 0x1b, 0xd4, 0x98, 0x29, + 0xb0, 0x66, 0xe7, 0xf6, 0xf7, 0x8a, 0xa7, 0x6f, 0xac, 0xad, 0x55, 0x63, 0x60, 0x9c, 0xd6, 0x46, + 0xfd, 0x79, 0x05, 0xa6, 0x82, 0xce, 0x60, 0xf2, 0x4a, 0x9b, 0xb8, 0x9e, 0x8b, 0x30, 0x9c, 0x6d, + 0x6a, 0x3b, 0xab, 0xb6, 0xb5, 0xd2, 0xf6, 0x34, 0xcf, 0xb0, 0x1a, 0x15, 0x6b, 0xd3, 0x34, 0x1a, + 0x5b, 0x9e, 0xe8, 0xda, 0xec, 0xfe, 0x5e, 0xf1, 0xec, 0x4a, 0x6a, 0x0d, 0x9c, 0xd1, 0x92, 0x76, + 0xba, 0xa9, 0xed, 0x24, 0x10, 0x4a, 0x9d, 0x5e, 0x49, 0x82, 0x71, 0x5a, 0x1b, 0xf5, 0x9d, 0x30, + 0xc5, 0xc7, 0x81, 0x89, 0xeb, 0x39, 0x86, 0xee, 0x19, 0xb6, 0x85, 0xae, 0x40, 0xbf, 0xa5, 0x35, + 0x09, 0xeb, 0xe1, 0xc8, 0xc2, 0xd8, 0x97, 0xf6, 0x8a, 0x6f, 0xda, 0xdf, 0x2b, 0xf6, 0xaf, 0x6a, + 0x4d, 0x82, 0x19, 0x44, 0xfd, 0x9f, 0x05, 0xb8, 0x98, 0x68, 0xf7, 0x9c, 0xe1, 0x6d, 0xdd, 0x6a, + 0xd1, 0xff, 0x5c, 0xf4, 0xfd, 0x0a, 0x4c, 0x69, 0xf1, 0x0a, 0x0c, 0xe1, 0xe8, 0xc3, 0x8b, 0x73, + 0x87, 0xff, 0xc0, 0xe7, 0x12, 0xd4, 0x16, 0xce, 0x8b, 0x7e, 0x25, 0x07, 0x80, 0x93, 0xa4, 0xd1, + 0xc7, 0x15, 0x18, 0xb2, 0x79, 0xe7, 0x66, 0x0a, 0x57, 0xfa, 0xae, 0x8e, 0x3e, 0xfc, 0xed, 0x47, + 0xd2, 0x0d, 0x69, 0xd0, 0x73, 0xe2, 0xef, 0xa2, 0xe5, 0x39, 0xbb, 0x0b, 0xa7, 0x44, 0xf7, 0x86, + 0x44, 0x29, 0xf6, 0xc9, 0xcf, 0x3e, 0x0e, 0x63, 0x72, 0x4d, 0x34, 0x09, 0x7d, 0xb7, 0x09, 0xdf, + 0xaa, 0x23, 0x98, 0xfe, 0x8b, 0xa6, 0x61, 0x60, 0x5b, 0x33, 0xdb, 0x84, 0x2d, 0xe9, 0x08, 0xe6, + 0x3f, 0x1e, 0x2f, 0xbc, 0x5b, 0x51, 0x1f, 0x86, 0x81, 0xf9, 0x7a, 0xdd, 0xb6, 0xd0, 0x03, 0x30, + 0x44, 0x2c, 0x6d, 0xc3, 0x24, 0x75, 0xd6, 0x70, 0x38, 0xa4, 0xb7, 0xc8, 0x8b, 0xb1, 0x0f, 0x57, + 0x7f, 0xa4, 0x00, 0x83, 0xac, 0x91, 0x8b, 0x7e, 0x48, 0x81, 0xd3, 0xb7, 0xdb, 0x1b, 0xc4, 0xb1, + 0x88, 0x47, 0xdc, 0xb2, 0xe6, 0x6e, 0x6d, 0xd8, 0x9a, 0x53, 0x17, 0x0b, 0x73, 0x3d, 0xcf, 0x8c, + 0xdc, 0x4c, 0xa2, 0xe3, 0x7b, 0x30, 0x05, 0x80, 0xd3, 0x88, 0xa3, 0x6d, 0x18, 0xb3, 0x1a, 0x86, + 0xb5, 0x53, 0xb1, 0x1a, 0x0e, 0x71, 0x5d, 0x36, 0xe8, 0xd1, 0x87, 0x9f, 0xce, 0xd3, 0x99, 0x55, + 0x09, 0xcf, 0xc2, 0xe4, 0xfe, 0x5e, 0x71, 0x4c, 0x2e, 0xc1, 0x11, 0x3a, 0xea, 0xdf, 0x2b, 0x70, + 0x6a, 0xbe, 0xde, 0x34, 0x5c, 0xca, 0x69, 0xab, 0x66, 0xbb, 0x61, 0x74, 0xb1, 0xf5, 0xd1, 0x33, + 0x30, 0xa8, 0xdb, 0xd6, 0xa6, 0xd1, 0x10, 0xfd, 0x7c, 0xfb, 0x1c, 0xe7, 0x5c, 0x73, 0x32, 0xe7, + 0x62, 0xdd, 0x13, 0x1c, 0x6f, 0x0e, 0x6b, 0x77, 0x16, 0x7d, 0x86, 0xbe, 0x00, 0xfb, 0x7b, 0xc5, + 0xc1, 0x12, 0x43, 0x80, 0x05, 0x22, 0x74, 0x15, 0x86, 0xeb, 0x86, 0xcb, 0x17, 0xb3, 0x8f, 0x2d, + 0xe6, 0xd8, 0xfe, 0x5e, 0x71, 0xb8, 0x2c, 0xca, 0x70, 0x00, 0x45, 0xcb, 0x30, 0x4d, 0x67, 0x90, + 0xb7, 0xab, 0x11, 0xdd, 0x21, 0x1e, 0xed, 0xda, 0x4c, 0x3f, 0xeb, 0xee, 0xcc, 0xfe, 0x5e, 0x71, + 0xfa, 0x66, 0x0a, 0x1c, 0xa7, 0xb6, 0x52, 0x97, 0x60, 0x78, 0xde, 0x24, 0x0e, 0x65, 0x08, 0xe8, + 0x71, 0x98, 0x20, 0x4d, 0xcd, 0x30, 0x31, 0xd1, 0x89, 0xb1, 0x4d, 0x1c, 0x77, 0x46, 0xb9, 0xd2, + 0x77, 0x75, 0x64, 0x01, 0xed, 0xef, 0x15, 0x27, 0x16, 0x23, 0x10, 0x1c, 0xab, 0xa9, 0x7e, 0x44, + 0x81, 0xd1, 0xf9, 0x76, 0xdd, 0xf0, 0xf8, 0xb8, 0x90, 0x03, 0xa3, 0x1a, 0xfd, 0x59, 0xb5, 0x4d, + 0x43, 0xdf, 0x15, 0x9b, 0xeb, 0xa9, 0x5c, 0x9f, 0x5b, 0x88, 0x66, 0xe1, 0xd4, 0xfe, 0x5e, 0x71, + 0x54, 0x2a, 0xc0, 0x32, 0x11, 0x75, 0x0b, 0x64, 0x18, 0x7a, 0x1f, 0x8c, 0xf1, 0xe1, 0xae, 0x68, + 0x2d, 0x4c, 0x36, 0x45, 0x1f, 0xee, 0x93, 0xd6, 0xca, 0x27, 0x34, 0x77, 0x6b, 0xe3, 0x65, 0xa2, + 0x7b, 0x98, 0x6c, 0x12, 0x87, 0x58, 0x3a, 0xe1, 0xdb, 0xa6, 0x24, 0x35, 0xc6, 0x11, 0x54, 0xea, + 0xff, 0xa7, 0xc0, 0xa5, 0xf9, 0xb6, 0xb7, 0x65, 0x3b, 0xc6, 0x5d, 0xe2, 0x84, 0xd3, 0x1d, 0x60, + 0x40, 0x4f, 0xc2, 0x84, 0x16, 0x54, 0x58, 0x0d, 0xb7, 0xd3, 0x59, 0xb1, 0x9d, 0x26, 0xe6, 0x23, + 0x50, 0x1c, 0xab, 0x8d, 0x1e, 0x06, 0x70, 0xc3, 0xb5, 0x65, 0x3c, 0x60, 0x01, 0x89, 0xb6, 0x20, + 0xad, 0xaa, 0x54, 0x4b, 0xfd, 0x0b, 0x7a, 0x14, 0x6e, 0x6b, 0x86, 0xa9, 0x6d, 0x18, 0xa6, 0xe1, + 0xed, 0xbe, 0x60, 0x5b, 0xa4, 0x8b, 0xdd, 0xbc, 0x0e, 0xe7, 0xda, 0x96, 0xc6, 0xdb, 0x99, 0x64, + 0x85, 0xef, 0xdf, 0xb5, 0xdd, 0x16, 0xe1, 0x5c, 0x72, 0x64, 0xe1, 0xc2, 0xfe, 0x5e, 0xf1, 0xdc, + 0x7a, 0x7a, 0x15, 0x9c, 0xd5, 0x96, 0x9e, 0x7a, 0x12, 0xe8, 0x59, 0xdb, 0x6c, 0x37, 0x05, 0xd6, + 0x3e, 0x86, 0x95, 0x9d, 0x7a, 0xeb, 0xa9, 0x35, 0x70, 0x46, 0x4b, 0xf5, 0x0b, 0x05, 0x18, 0x5c, + 0xd0, 0xf4, 0xdb, 0xed, 0x16, 0x7a, 0x1b, 0x0c, 0xb7, 0x1c, 0x7b, 0xdb, 0xa8, 0x13, 0x47, 0x8c, + 0x6d, 0x52, 0x8c, 0x6d, 0xb8, 0x2a, 0xca, 0x71, 0x50, 0x03, 0x19, 0x30, 0xe1, 0xff, 0x5f, 0xea, + 0xe1, 0xcb, 0x65, 0x5f, 0x42, 0x35, 0x82, 0x08, 0xc7, 0x10, 0x23, 0x15, 0x06, 0x1d, 0xd2, 0xa0, + 0x47, 0x5d, 0x1f, 0xeb, 0x16, 0xfb, 0xda, 0x31, 0x2b, 0xc1, 0x02, 0x82, 0xde, 0x0f, 0x13, 0xba, + 0x43, 0xea, 0xc4, 0xf2, 0x0c, 0xcd, 0x74, 0xe9, 0xe6, 0x1c, 0xe8, 0x7e, 0x73, 0xb2, 0x4e, 0x94, + 0x22, 0xcd, 0x71, 0x0c, 0x9d, 0xfa, 0xa5, 0x02, 0x8c, 0xf1, 0x89, 0x5a, 0x68, 0xeb, 0xb7, 0x89, + 0x87, 0xbe, 0x03, 0x86, 0xa9, 0x7c, 0x57, 0xd7, 0x3c, 0x4d, 0x7c, 0x08, 0xdf, 0x92, 0x39, 0x74, + 0xf6, 0x0d, 0xd2, 0xda, 0x21, 0xf5, 0x15, 0xe2, 0x69, 0xe1, 0xfe, 0x0b, 0xcb, 0x70, 0x80, 0x15, + 0x6d, 0x42, 0xbf, 0xdb, 0x22, 0xba, 0x98, 0xd8, 0x72, 0x9e, 0x4f, 0x5d, 0xee, 0x71, 0xad, 0x45, + 0xf4, 0x70, 0xbb, 0xd2, 0x5f, 0x98, 0xe1, 0x47, 0x16, 0x0c, 0xba, 0x9e, 0xe6, 0xb5, 0x5d, 0x36, + 0xbf, 0xa3, 0x0f, 0x2f, 0xf5, 0x4c, 0x89, 0x61, 0x5b, 0x98, 0x10, 0xb4, 0x06, 0xf9, 0x6f, 0x2c, + 0xa8, 0xa8, 0xff, 0x56, 0x81, 0x49, 0xb9, 0xfa, 0xb2, 0xe1, 0x7a, 0xe8, 0xdb, 0x12, 0xd3, 0x39, + 0xd7, 0xdd, 0x74, 0xd2, 0xd6, 0x6c, 0x32, 0x83, 0xdd, 0xea, 0x97, 0x48, 0x53, 0x49, 0x60, 0xc0, + 0xf0, 0x48, 0xd3, 0x97, 0x52, 0x9e, 0xee, 0x75, 0x84, 0x0b, 0xe3, 0x82, 0xd8, 0x40, 0x85, 0xa2, + 0xc5, 0x1c, 0xbb, 0xfa, 0x1d, 0x30, 0x2d, 0xd7, 0xf2, 0xf7, 0x35, 0x65, 0x19, 0xde, 0x6e, 0x2b, + 0xc1, 0x32, 0xe8, 0x27, 0x88, 0x19, 0x04, 0xbd, 0x35, 0xd8, 0xe3, 0x9c, 0x33, 0x05, 0x73, 0x17, + 0xdd, 0xe7, 0xea, 0x87, 0xfb, 0xa2, 0x73, 0x47, 0x97, 0x11, 0x6d, 0xc7, 0xbe, 0xdc, 0xd1, 0x87, + 0x6f, 0xf4, 0x3a, 0x40, 0xbf, 0xeb, 0xaf, 0x17, 0x1e, 0x70, 0x15, 0x86, 0x5d, 0x42, 0xea, 0xd2, + 0xb9, 0xcc, 0x4e, 0xf3, 0x9a, 0x28, 0xc3, 0x01, 0xf4, 0xf8, 0x39, 0xc1, 0x6b, 0xfd, 0x80, 0x92, + 0xbb, 0x5d, 0x9e, 0x0c, 0x5e, 0x22, 0x96, 0xa2, 0x97, 0xc9, 0x10, 0x1f, 0x4e, 0x0c, 0x31, 0xba, + 0x0b, 0xe3, 0xa6, 0xe6, 0x7a, 0xb7, 0x5a, 0x54, 0x6f, 0xf3, 0xf7, 0xcc, 0xe8, 0xc3, 0xf3, 0x79, + 0x16, 0x7d, 0x59, 0x46, 0xb4, 0x30, 0xb5, 0xbf, 0x57, 0x1c, 0x8f, 0x14, 0xe1, 0x28, 0x29, 0xf4, + 0x32, 0x8c, 0xd0, 0x82, 0x45, 0xc7, 0xb1, 0x1d, 0xc1, 0x2f, 0x9e, 0xc8, 0x4b, 0x97, 0x21, 0xe1, + 0x7a, 0x64, 0xf0, 0x13, 0x87, 0xe8, 0xd1, 0x7b, 0x01, 0xd9, 0x1b, 0x4c, 0x93, 0xaf, 0x5f, 0xe7, + 0x4a, 0x2a, 0x1d, 0x2c, 0x5d, 0xfe, 0xbe, 0x85, 0x59, 0xb1, 0x2f, 0xd1, 0xad, 0x44, 0x0d, 0x9c, + 0xd2, 0x0a, 0xdd, 0x06, 0x14, 0x28, 0xba, 0xfc, 0xb4, 0x3f, 0x60, 0x6b, 0x04, 0x95, 0xc4, 0xd6, + 0x38, 0x4b, 0x89, 0x5d, 0x4f, 0xa0, 0xc0, 0x29, 0x68, 0xd5, 0xdf, 0x2c, 0xc0, 0x28, 0xdf, 0x22, + 0x5c, 0x19, 0x39, 0xfe, 0xb3, 0x82, 0x44, 0xce, 0x8a, 0x52, 0xfe, 0xcf, 0x9f, 0x75, 0x38, 0xf3, + 0xa8, 0x68, 0xc6, 0x8e, 0x8a, 0xc5, 0x5e, 0x09, 0x75, 0x3e, 0x29, 0xfe, 0x58, 0x81, 0x53, 0x52, + 0xed, 0x13, 0x38, 0x28, 0xea, 0xd1, 0x83, 0xe2, 0xa9, 0x1e, 0xc7, 0x97, 0x71, 0x4e, 0xd8, 0x91, + 0x61, 0x31, 0x1e, 0xfe, 0x30, 0xc0, 0x06, 0x63, 0x27, 0x92, 0x68, 0x1b, 0x2c, 0xf9, 0x42, 0x00, + 0xc1, 0x52, 0xad, 0x08, 0x53, 0x2c, 0x74, 0x62, 0x8a, 0xea, 0x7f, 0xe8, 0x83, 0xa9, 0xc4, 0xb4, + 0x27, 0xf9, 0x88, 0xf2, 0x75, 0xe2, 0x23, 0x85, 0xaf, 0x07, 0x1f, 0xe9, 0xcb, 0xc5, 0x47, 0xba, + 0x3f, 0x88, 0x1c, 0x40, 0x4d, 0xa3, 0xc1, 0x9b, 0xd5, 0x3c, 0xcd, 0xf1, 0xd6, 0x8c, 0x26, 0x11, + 0x1c, 0xe7, 0x9b, 0xbb, 0xdb, 0xb2, 0xb4, 0x05, 0x67, 0x3c, 0x2b, 0x09, 0x4c, 0x38, 0x05, 0xbb, + 0xfa, 0x5d, 0x05, 0x18, 0x5a, 0xd0, 0x5c, 0xd6, 0xd3, 0x0f, 0xc2, 0x98, 0x40, 0x5d, 0x69, 0x6a, + 0x0d, 0xd2, 0x8b, 0x39, 0x42, 0xa0, 0x5c, 0x91, 0xd0, 0x71, 0x8d, 0x4e, 0x2e, 0xc1, 0x11, 0x72, + 0x68, 0x17, 0x46, 0x9b, 0xa1, 0xf6, 0x22, 0x96, 0x78, 0xa9, 0x77, 0xea, 0x14, 0x1b, 0x57, 0x5b, + 0xa5, 0x02, 0x2c, 0xd3, 0x52, 0x5f, 0x82, 0xd3, 0x29, 0x3d, 0xee, 0x42, 0x71, 0x7b, 0x0b, 0x0c, + 0x51, 0xdd, 0x3b, 0x14, 0xc3, 0x46, 0xf7, 0xf7, 0x8a, 0x43, 0xcf, 0xf2, 0x22, 0xec, 0xc3, 0xd4, + 0x77, 0x51, 0x01, 0x20, 0xde, 0xa7, 0x6e, 0x0c, 0x7c, 0x03, 0x00, 0xa5, 0x79, 0x6c, 0x7b, 0x7c, + 0x2b, 0x3d, 0x05, 0x03, 0xad, 0x2d, 0xcd, 0xf5, 0x5b, 0x3c, 0xe0, 0xb3, 0x8a, 0x2a, 0x2d, 0xbc, + 0xb7, 0x57, 0x9c, 0x91, 0x05, 0x11, 0xd1, 0x88, 0xc1, 0x30, 0x6f, 0x47, 0x77, 0x18, 0xdd, 0xe4, + 0x25, 0xbb, 0xd9, 0x32, 0x09, 0x85, 0xb2, 0x1d, 0x56, 0xc8, 0xb7, 0xc3, 0x96, 0x13, 0x98, 0x70, + 0x0a, 0x76, 0x9f, 0x66, 0xc5, 0x32, 0x3c, 0x43, 0x0b, 0x68, 0xf6, 0xe5, 0xa7, 0x19, 0xc5, 0x84, + 0x53, 0xb0, 0xa3, 0x4f, 0x2a, 0x30, 0x1b, 0x2d, 0x5e, 0x32, 0x2c, 0xc3, 0xdd, 0x22, 0x75, 0x46, + 0xbc, 0xff, 0xd0, 0xc4, 0x2f, 0xef, 0xef, 0x15, 0x67, 0x97, 0x33, 0x31, 0xe2, 0x0e, 0xd4, 0xd0, + 0xf7, 0x29, 0x70, 0x21, 0x36, 0x2f, 0x8e, 0xd1, 0x68, 0x10, 0x47, 0xf4, 0xe6, 0xf0, 0x1f, 0x78, + 0x71, 0x7f, 0xaf, 0x78, 0x61, 0x39, 0x1b, 0x25, 0xee, 0x44, 0x0f, 0xfd, 0x84, 0x02, 0x67, 0x5b, + 0xc4, 0xaa, 0x1b, 0x56, 0xe3, 0x39, 0xdb, 0xb9, 0x4d, 0x1c, 0x17, 0xdb, 0xa6, 0x69, 0xb7, 0x3d, + 0x77, 0x66, 0x90, 0x9d, 0x61, 0x95, 0x3c, 0xdf, 0x5c, 0x35, 0x0d, 0xe3, 0xc2, 0x65, 0xb1, 0x45, + 0xcf, 0xa6, 0x82, 0x5d, 0x9c, 0xd1, 0x11, 0xf5, 0x37, 0x14, 0xe8, 0x2b, 0xe1, 0x0a, 0x7a, 0x30, + 0xf2, 0x89, 0x9c, 0x93, 0x3f, 0x91, 0x7b, 0x7b, 0xc5, 0xa1, 0x12, 0xae, 0x48, 0x1f, 0xe3, 0xf7, + 0x29, 0x30, 0xa5, 0xdb, 0x96, 0xa7, 0xd1, 0xb9, 0xc3, 0x5c, 0x56, 0xf6, 0xcf, 0xe5, 0x5c, 0xca, + 0x70, 0x29, 0x86, 0x2c, 0x34, 0x76, 0xc7, 0x21, 0x2e, 0x4e, 0x52, 0x56, 0x3f, 0xaf, 0xc0, 0x74, + 0x49, 0x6b, 0x09, 0x53, 0x50, 0x99, 0x6c, 0x1a, 0x74, 0x83, 0x74, 0x63, 0xd9, 0x47, 0x5b, 0x30, + 0xc8, 0xac, 0xcd, 0x6e, 0x2f, 0xba, 0x7c, 0x48, 0xfb, 0x59, 0x86, 0x8b, 0xdb, 0x41, 0xf8, 0xff, + 0x58, 0xe0, 0x57, 0x9f, 0x80, 0xc9, 0x78, 0x3d, 0x54, 0xf4, 0x65, 0x1a, 0x6e, 0x7c, 0x1c, 0x89, + 0x8b, 0x23, 0x8f, 0x0f, 0xff, 0xff, 0xaf, 0x15, 0xdf, 0xf4, 0xe1, 0x3f, 0xbb, 0xf2, 0x26, 0xf5, + 0x2b, 0x0a, 0x8c, 0x95, 0x4c, 0xbb, 0x5d, 0xaf, 0x3a, 0xf6, 0xa6, 0x61, 0x92, 0x37, 0x86, 0x95, + 0x43, 0xee, 0x71, 0x96, 0xe8, 0xca, 0xac, 0x0e, 0x72, 0xc5, 0x37, 0x88, 0xd5, 0x41, 0xee, 0x72, + 0x86, 0x34, 0xf9, 0x22, 0x9c, 0x91, 0x6b, 0x85, 0x26, 0xd3, 0x2b, 0xd0, 0x7f, 0xdb, 0xb0, 0xea, + 0xf1, 0x8d, 0x79, 0xd3, 0xb0, 0xea, 0x98, 0x41, 0x82, 0xad, 0x5b, 0xc8, 0x3c, 0xb3, 0xfe, 0x66, + 0x24, 0x3a, 0x6d, 0x4c, 0x58, 0xbd, 0x0a, 0xc3, 0xba, 0xb6, 0xd0, 0xb6, 0xea, 0x66, 0xb0, 0xeb, + 0xe9, 0x14, 0x94, 0xe6, 0x79, 0x19, 0x0e, 0xa0, 0xe8, 0x2e, 0x40, 0x78, 0x3b, 0xd1, 0x8b, 0x10, + 0x10, 0x5e, 0x7c, 0xd4, 0x88, 0xe7, 0x19, 0x56, 0xc3, 0x0d, 0xf7, 0x55, 0x08, 0xc3, 0x12, 0x35, + 0xf4, 0x41, 0x18, 0x97, 0x25, 0x12, 0x6e, 0x26, 0xcd, 0xb9, 0x0c, 0x11, 0xd1, 0xe7, 0x8c, 0x20, + 0x3c, 0x2e, 0x97, 0xba, 0x38, 0x4a, 0x0d, 0xed, 0x06, 0xf2, 0x17, 0x37, 0xd2, 0xf6, 0xe7, 0xd7, + 0x28, 0x64, 0xd1, 0x67, 0x5a, 0x10, 0x1f, 0x8b, 0x18, 0x8d, 0x23, 0xa4, 0x52, 0x0c, 0x33, 0x03, + 0xc7, 0x65, 0x98, 0x21, 0x30, 0xc4, 0x4d, 0x53, 0xfe, 0x71, 0xf3, 0x78, 0x9e, 0x01, 0x72, 0x2b, + 0x57, 0x78, 0xdd, 0xc6, 0x7f, 0xbb, 0xd8, 0xc7, 0x8d, 0xb6, 0x61, 0x8c, 0x0a, 0xd6, 0x35, 0x62, + 0x12, 0xdd, 0xb3, 0x9d, 0x99, 0xa1, 0xfc, 0xd7, 0x59, 0x35, 0x09, 0x0f, 0x97, 0x62, 0xe5, 0x12, + 0x1c, 0xa1, 0x13, 0x58, 0xee, 0x86, 0x33, 0x2d, 0x77, 0x6d, 0x18, 0xdd, 0x96, 0x4c, 0xf1, 0x23, + 0x6c, 0x12, 0x9e, 0xcc, 0xd3, 0xb1, 0xd0, 0x2e, 0xbf, 0x70, 0x5a, 0x10, 0x1a, 0x95, 0x6d, 0xf8, + 0x32, 0x1d, 0xb4, 0x01, 0x43, 0x1b, 0x5c, 0x06, 0x9d, 0x01, 0x36, 0x17, 0xef, 0xe9, 0x41, 0xb4, + 0xe6, 0x72, 0xae, 0xf8, 0x81, 0x7d, 0xc4, 0xe8, 0x25, 0x18, 0x34, 0x8d, 0xa6, 0xe1, 0xb9, 0x33, + 0xa3, 0x8c, 0x44, 0xae, 0xa5, 0x5d, 0x66, 0x18, 0xf8, 0x61, 0xc5, 0xff, 0xc7, 0x02, 0x2b, 0xfa, + 0xb4, 0x02, 0xa7, 0xc5, 0x36, 0x0c, 0x0e, 0x2d, 0x83, 0xb8, 0x33, 0x63, 0x6c, 0x0e, 0x6f, 0xf4, + 0x76, 0x48, 0x86, 0x07, 0xf4, 0xc2, 0x05, 0x31, 0x9b, 0xa7, 0x57, 0x92, 0xc4, 0x70, 0x5a, 0x0f, + 0xd4, 0x7f, 0x39, 0x0e, 0x53, 0x25, 0xb3, 0xed, 0x7a, 0xc4, 0x99, 0x17, 0x9e, 0x2c, 0xc4, 0x41, + 0x1f, 0x55, 0xe0, 0x2c, 0xfb, 0xb7, 0x6c, 0xdf, 0xb1, 0xca, 0xc4, 0xd4, 0x76, 0xe7, 0x37, 0x69, + 0x8d, 0x7a, 0xfd, 0x70, 0x87, 0x47, 0xb9, 0x2d, 0xd4, 0x64, 0x76, 0x63, 0x53, 0x4b, 0xc5, 0x88, + 0x33, 0x28, 0xa1, 0xef, 0x55, 0xe0, 0x7c, 0x0a, 0xa8, 0x4c, 0x4c, 0xe2, 0xf9, 0xc2, 0xff, 0x61, + 0xfb, 0x71, 0x69, 0x7f, 0xaf, 0x78, 0xbe, 0x96, 0x85, 0x14, 0x67, 0xd3, 0x43, 0xdf, 0xaf, 0xc0, + 0x6c, 0x0a, 0x74, 0x49, 0x33, 0xcc, 0xb6, 0xe3, 0xeb, 0x05, 0x87, 0xed, 0x0e, 0x13, 0xcf, 0x6b, + 0x99, 0x58, 0x71, 0x07, 0x8a, 0xe8, 0x43, 0x70, 0x26, 0x80, 0xae, 0x5b, 0x16, 0x21, 0xf5, 0x88, + 0x96, 0x70, 0xd8, 0xae, 0x9c, 0xdf, 0xdf, 0x2b, 0x9e, 0xa9, 0xa5, 0x21, 0xc4, 0xe9, 0x74, 0x50, + 0x03, 0x2e, 0x85, 0x00, 0xcf, 0x30, 0x8d, 0xbb, 0x5c, 0x91, 0xd9, 0x72, 0x88, 0xbb, 0x65, 0x9b, + 0x75, 0xc6, 0x8a, 0x95, 0x85, 0x37, 0xef, 0xef, 0x15, 0x2f, 0xd5, 0x3a, 0x55, 0xc4, 0x9d, 0xf1, + 0xa0, 0x3a, 0x8c, 0xb9, 0xba, 0x66, 0x55, 0x2c, 0x8f, 0x38, 0xdb, 0x9a, 0x39, 0x33, 0x98, 0x6b, + 0x80, 0x9c, 0x01, 0x4a, 0x78, 0x70, 0x04, 0x2b, 0x7a, 0x37, 0x0c, 0x93, 0x9d, 0x96, 0x66, 0xd5, + 0x09, 0x67, 0xba, 0x23, 0x0b, 0x17, 0xe9, 0x51, 0xbf, 0x28, 0xca, 0xee, 0xed, 0x15, 0xc7, 0xfc, + 0xff, 0x57, 0xec, 0x3a, 0xc1, 0x41, 0x6d, 0xf4, 0x01, 0x98, 0x66, 0xae, 0x36, 0x75, 0xc2, 0x8e, + 0x10, 0xd7, 0xd7, 0x15, 0x87, 0x73, 0xf5, 0x93, 0x5d, 0xc3, 0xaf, 0xa4, 0xe0, 0xc3, 0xa9, 0x54, + 0xe8, 0x32, 0x34, 0xb5, 0x9d, 0xeb, 0x8e, 0xa6, 0x93, 0xcd, 0xb6, 0xb9, 0x46, 0x9c, 0xa6, 0x61, + 0x71, 0x63, 0x09, 0xd1, 0x6d, 0xab, 0x4e, 0x19, 0xb5, 0x72, 0x75, 0x80, 0x2f, 0xc3, 0x4a, 0xa7, + 0x8a, 0xb8, 0x33, 0x1e, 0xf4, 0x08, 0x8c, 0x19, 0x0d, 0xcb, 0x76, 0xc8, 0x9a, 0x66, 0x58, 0x9e, + 0x3b, 0x03, 0x4c, 0xc8, 0x66, 0xd3, 0x5a, 0x91, 0xca, 0x71, 0xa4, 0x16, 0xda, 0x06, 0x64, 0x91, + 0x3b, 0x55, 0xbb, 0xce, 0xb6, 0xc0, 0x7a, 0x8b, 0x6d, 0x64, 0xc1, 0x66, 0x0f, 0x3b, 0x35, 0x4c, + 0x95, 0x5e, 0x4d, 0x60, 0xc3, 0x29, 0x14, 0xd0, 0x12, 0xa0, 0xa6, 0xb6, 0xb3, 0xd8, 0x6c, 0x79, + 0xbb, 0x0b, 0x6d, 0xf3, 0xb6, 0xe0, 0x1a, 0x63, 0x6c, 0x2e, 0xb8, 0xa1, 0x29, 0x01, 0xc5, 0x29, + 0x2d, 0x90, 0x06, 0x17, 0xf8, 0x78, 0xca, 0x1a, 0x69, 0xda, 0x96, 0x4b, 0x3c, 0x57, 0xda, 0xa4, + 0x33, 0xe3, 0xcc, 0xe1, 0x82, 0x29, 0xb6, 0x95, 0xec, 0x6a, 0xb8, 0x13, 0x8e, 0xa8, 0xcb, 0xd9, + 0xc4, 0x01, 0x2e, 0x67, 0x8f, 0xc2, 0xb8, 0xeb, 0x69, 0x8e, 0xd7, 0x6e, 0x89, 0x65, 0x38, 0xc5, + 0x96, 0x81, 0xd9, 0x21, 0x6b, 0x32, 0x00, 0x47, 0xeb, 0xd1, 0xe5, 0xe3, 0xc6, 0x66, 0xd1, 0x6e, + 0x32, 0x5c, 0xbe, 0x9a, 0x54, 0x8e, 0x23, 0xb5, 0xd0, 0x3a, 0x9c, 0x6b, 0x6a, 0x3b, 0xc1, 0xe7, + 0x5b, 0xd5, 0x1c, 0xcd, 0x34, 0x89, 0x69, 0xb8, 0xcd, 0x99, 0x29, 0xd6, 0x53, 0x76, 0xc3, 0xbf, + 0x92, 0x5e, 0x05, 0x67, 0xb5, 0x15, 0x3e, 0x68, 0x65, 0x47, 0x33, 0x22, 0x28, 0x51, 0xc4, 0x07, + 0x2d, 0x0e, 0xc6, 0x69, 0x6d, 0xd4, 0xff, 0xd1, 0x0f, 0x33, 0x89, 0x13, 0xcc, 0x77, 0x24, 0x3b, + 0x90, 0x47, 0x29, 0x47, 0xc4, 0xa3, 0x5a, 0x70, 0x25, 0xa8, 0x70, 0xbd, 0xd5, 0x4e, 0xa5, 0x55, + 0x60, 0xb4, 0xee, 0xdf, 0xdf, 0x2b, 0x5e, 0xa9, 0x1d, 0x50, 0x17, 0x1f, 0x88, 0x2d, 0x9b, 0xff, + 0xf7, 0x9d, 0x10, 0xff, 0xff, 0x00, 0x4c, 0x4b, 0x00, 0x87, 0x68, 0xf5, 0xdd, 0x1e, 0xce, 0x1f, + 0xc6, 0xf6, 0x6a, 0x29, 0xf8, 0x70, 0x2a, 0x95, 0x4c, 0xa6, 0x3b, 0x70, 0x12, 0x4c, 0x57, 0xdd, + 0xeb, 0x83, 0x91, 0x92, 0x6d, 0xd5, 0xb9, 0x5d, 0xe4, 0xa1, 0xc8, 0xad, 0xf7, 0x25, 0x59, 0x76, + 0xbe, 0xb7, 0x57, 0x1c, 0x0f, 0x2a, 0x4a, 0xc2, 0xf4, 0x63, 0xc1, 0xfd, 0x12, 0xd7, 0x48, 0xdf, + 0x1c, 0xbd, 0x18, 0xba, 0xb7, 0x57, 0x3c, 0x15, 0x34, 0x8b, 0xde, 0x15, 0x51, 0x8e, 0x6a, 0x6a, + 0xae, 0xb7, 0xe6, 0x68, 0x96, 0x6b, 0xf4, 0x60, 0x98, 0x0c, 0x2e, 0x04, 0x96, 0x13, 0xd8, 0x70, + 0x0a, 0x05, 0xf4, 0x32, 0x4c, 0xd0, 0xd2, 0xf5, 0x56, 0x5d, 0xf3, 0x48, 0x4e, 0x7b, 0x64, 0xe0, + 0xc3, 0xb4, 0x1c, 0xc1, 0x84, 0x63, 0x98, 0xb9, 0x97, 0x80, 0xe6, 0xda, 0x16, 0x5b, 0xcf, 0x88, + 0x97, 0x00, 0x2d, 0xc5, 0x02, 0x8a, 0x1e, 0x80, 0xa1, 0x26, 0x71, 0x5d, 0xad, 0x41, 0x98, 0x54, + 0x30, 0x12, 0x2a, 0x56, 0x2b, 0xbc, 0x18, 0xfb, 0x70, 0xf4, 0x36, 0x18, 0xd0, 0xed, 0x3a, 0x71, + 0x67, 0x86, 0x18, 0xe3, 0xa3, 0x67, 0xc0, 0x40, 0x89, 0x16, 0xdc, 0xdb, 0x2b, 0x8e, 0xb0, 0xeb, + 0x13, 0xfa, 0x0b, 0xf3, 0x4a, 0xea, 0x8f, 0x2b, 0x30, 0x19, 0x37, 0x96, 0x75, 0xe1, 0xdd, 0x70, + 0x72, 0x8e, 0x02, 0xea, 0xa7, 0x15, 0x18, 0xa3, 0x3d, 0x74, 0x6c, 0xb3, 0x6a, 0x6a, 0x16, 0x41, + 0xdf, 0xa3, 0xc0, 0xe4, 0x96, 0xd1, 0xd8, 0x92, 0xfd, 0xb8, 0x84, 0xb8, 0x9e, 0xcb, 0xd8, 0x74, + 0x23, 0x86, 0x6b, 0x61, 0x7a, 0x7f, 0xaf, 0x38, 0x19, 0x2f, 0xc5, 0x09, 0x9a, 0xea, 0x5f, 0x16, + 0xe0, 0x9c, 0xdc, 0xb3, 0xf9, 0xd0, 0x45, 0x1e, 0xfd, 0xb1, 0x02, 0xd0, 0x34, 0xac, 0x79, 0xd3, + 0xb4, 0xef, 0x30, 0xe7, 0x53, 0xaa, 0x00, 0xbd, 0x98, 0xd7, 0xc8, 0x99, 0x42, 0x61, 0x6e, 0x25, + 0xc0, 0xce, 0x2f, 0x26, 0x9f, 0xf7, 0x8d, 0x27, 0x21, 0xe0, 0xde, 0x5e, 0xb1, 0x98, 0xf4, 0xcb, + 0x9f, 0xc3, 0xc2, 0xf9, 0x7d, 0xd9, 0x70, 0xbd, 0x8f, 0xfe, 0x45, 0xc7, 0x2a, 0xfc, 0x86, 0x32, + 0x1c, 0xc8, 0x6c, 0x13, 0x4e, 0xc5, 0x08, 0xa7, 0x38, 0xe6, 0x96, 0x65, 0xc7, 0xdc, 0x03, 0x98, + 0xd4, 0x9c, 0xef, 0x8a, 0x3f, 0xf7, 0x4c, 0x5b, 0xb3, 0x3c, 0x3a, 0xd3, 0x92, 0x23, 0xef, 0x9f, + 0x17, 0x60, 0x5a, 0x4c, 0x80, 0x49, 0x55, 0x94, 0x96, 0x69, 0xef, 0x36, 0x89, 0x75, 0x12, 0xce, + 0x5a, 0xfe, 0x47, 0x50, 0xc8, 0xfc, 0x08, 0x9a, 0x89, 0x8f, 0xa0, 0x2f, 0xcf, 0x47, 0x10, 0xf0, + 0x8a, 0x03, 0x0c, 0x33, 0x18, 0xce, 0x1a, 0x16, 0xed, 0xe8, 0x75, 0xb6, 0x61, 0x42, 0x97, 0x4a, + 0xc6, 0x9f, 0x86, 0xb9, 0xee, 0x59, 0x49, 0xad, 0x81, 0x33, 0x5a, 0xaa, 0x7f, 0xad, 0xc0, 0x4c, + 0xda, 0xfc, 0x9e, 0x80, 0x2d, 0xb5, 0x19, 0xb5, 0xa5, 0xde, 0xe8, 0xe1, 0xdb, 0x88, 0x74, 0x3d, + 0xc3, 0xa6, 0xfa, 0x57, 0x05, 0x38, 0x1b, 0x56, 0xaf, 0x58, 0xae, 0xa7, 0x99, 0x26, 0x97, 0x4b, + 0x8f, 0x7f, 0x2f, 0xb5, 0x22, 0x26, 0xf1, 0xd5, 0xde, 0x86, 0x2a, 0xf7, 0x3d, 0xd3, 0xaf, 0x63, + 0x27, 0xe6, 0xd7, 0x51, 0x3d, 0x42, 0x9a, 0x9d, 0x5d, 0x3c, 0xfe, 0x93, 0x02, 0xb3, 0xe9, 0x0d, + 0x4f, 0x60, 0x53, 0xd9, 0xd1, 0x4d, 0xf5, 0xde, 0xa3, 0x1b, 0x75, 0xc6, 0xb6, 0xfa, 0xf9, 0x42, + 0xd6, 0x68, 0x99, 0x5d, 0x7d, 0x13, 0x4e, 0x39, 0xa4, 0x61, 0xb8, 0x9e, 0x70, 0x40, 0x38, 0x9c, + 0x8f, 0xb5, 0x7f, 0x9f, 0x76, 0x0a, 0x47, 0x71, 0xe0, 0x38, 0x52, 0xb4, 0x0a, 0x43, 0x2e, 0x21, + 0x75, 0x8a, 0xbf, 0xd0, 0x3d, 0xfe, 0x40, 0x88, 0xa8, 0xf1, 0xb6, 0xd8, 0x47, 0x82, 0xbe, 0x0d, + 0xc6, 0xeb, 0xc1, 0x17, 0x45, 0xb1, 0xf6, 0x75, 0x8f, 0x95, 0xa9, 0x68, 0x65, 0xb9, 0x35, 0x8e, + 0x22, 0x53, 0xff, 0x4e, 0x81, 0x8b, 0x9d, 0xf6, 0x16, 0x7a, 0x05, 0x40, 0xf7, 0xa5, 0x42, 0x57, + 0x1c, 0x9e, 0x4f, 0xe4, 0x5c, 0x4b, 0x8e, 0x25, 0xfc, 0x40, 0x83, 0x22, 0x17, 0x4b, 0x44, 0x52, + 0xbc, 0xfd, 0x0a, 0xc7, 0xe4, 0xed, 0xa7, 0xfe, 0x67, 0x45, 0x66, 0x45, 0xf2, 0xda, 0xbe, 0xd1, + 0x58, 0x91, 0xdc, 0xf7, 0xcc, 0x7b, 0xba, 0x2f, 0x17, 0xe0, 0x4a, 0x7a, 0x13, 0xe9, 0x3c, 0x7f, + 0x1a, 0x06, 0x5b, 0xfc, 0x1d, 0x04, 0x77, 0x09, 0xbf, 0x4a, 0x39, 0x0b, 0x7f, 0xa5, 0x70, 0x6f, + 0xaf, 0x38, 0x9b, 0xc6, 0xe8, 0xc5, 0xfb, 0x06, 0xd1, 0x0e, 0x19, 0xb1, 0x0b, 0x05, 0x2e, 0xb4, + 0xbf, 0xa3, 0x4b, 0xe6, 0xa2, 0x6d, 0x10, 0xb3, 0xeb, 0x3b, 0x84, 0x8f, 0x28, 0x30, 0x11, 0xd9, + 0xd1, 0xee, 0xcc, 0x00, 0xdb, 0xa3, 0xb9, 0x1c, 0xad, 0x22, 0x9f, 0x4a, 0x28, 0x0d, 0x44, 0x8a, + 0x5d, 0x1c, 0x23, 0x18, 0x63, 0xb3, 0xf2, 0xac, 0xbe, 0xe1, 0xd8, 0xac, 0xdc, 0xf9, 0x0c, 0x36, + 0xfb, 0x63, 0x85, 0xac, 0xd1, 0x32, 0x36, 0x7b, 0x07, 0x46, 0x7c, 0x31, 0xd2, 0x67, 0x17, 0x4b, + 0xbd, 0xf6, 0x89, 0xa3, 0x5b, 0x98, 0x12, 0xfd, 0x19, 0xf1, 0x4b, 0x5c, 0x1c, 0xd2, 0x42, 0xdf, + 0xad, 0x00, 0x84, 0x0b, 0x23, 0x3e, 0xaa, 0xb5, 0xa3, 0x9b, 0x0e, 0x49, 0xac, 0x99, 0xa0, 0x9f, + 0xb4, 0xb4, 0x29, 0x24, 0xba, 0xea, 0xbf, 0xe9, 0x07, 0x94, 0xec, 0x7b, 0x77, 0xd7, 0xc5, 0x07, + 0x08, 0xb9, 0x2d, 0x98, 0x74, 0xa8, 0xb4, 0xa8, 0x1b, 0x26, 0x53, 0x59, 0xed, 0xb6, 0x97, 0xd3, + 0xf2, 0xc1, 0xd4, 0x2a, 0x1c, 0xc3, 0x85, 0x13, 0xd8, 0xd1, 0x5b, 0x60, 0xa8, 0xe5, 0x18, 0x4d, + 0xcd, 0xd9, 0x65, 0x4a, 0xf1, 0x30, 0xbf, 0xcb, 0xaa, 0xf2, 0x22, 0xec, 0xc3, 0xd0, 0x07, 0x60, + 0xc4, 0x34, 0x36, 0x89, 0xbe, 0xab, 0x9b, 0x44, 0x98, 0xca, 0x6f, 0x1d, 0xcd, 0x9a, 0x2f, 0xfb, + 0x68, 0x85, 0x07, 0xa2, 0xff, 0x13, 0x87, 0x04, 0x51, 0x05, 0x4e, 0xdf, 0x61, 0x3e, 0x31, 0x26, + 0x71, 0xdd, 0x5a, 0xbb, 0xd5, 0xb2, 0x1d, 0x8f, 0xd4, 0x99, 0x41, 0x7d, 0x98, 0x1b, 0xf6, 0x9e, + 0x4b, 0x82, 0x71, 0x5a, 0x1b, 0xf4, 0x04, 0x80, 0xd6, 0xf6, 0x6c, 0xfe, 0x20, 0x71, 0x66, 0x98, + 0x69, 0xed, 0x97, 0xe8, 0x62, 0xcf, 0x07, 0xa5, 0xf7, 0xf6, 0x8a, 0xa3, 0xc2, 0xf6, 0xc7, 0x96, + 0x46, 0x6a, 0x80, 0x5e, 0x84, 0x69, 0x9d, 0x83, 0x4a, 0x76, 0xb3, 0xa5, 0x79, 0x86, 0xd0, 0x88, + 0x47, 0x18, 0xa2, 0x6f, 0xda, 0xdf, 0x2b, 0x4e, 0x97, 0x52, 0xe0, 0x71, 0x94, 0xa9, 0x48, 0xd4, + 0x4f, 0x16, 0xe0, 0x42, 0x87, 0x09, 0x42, 0x98, 0x7e, 0x78, 0x62, 0xfd, 0xc4, 0x36, 0x7b, 0x84, + 0x7f, 0x2c, 0xa2, 0xf0, 0xde, 0x5e, 0xf1, 0xbe, 0x0e, 0x08, 0x6a, 0x74, 0x9f, 0x93, 0xc6, 0x2e, + 0x0e, 0xd1, 0xa0, 0x0a, 0x0c, 0xd6, 0xc3, 0xbb, 0xaf, 0x91, 0x85, 0x87, 0xe8, 0x51, 0xc0, 0xad, + 0xd4, 0xdd, 0x62, 0x13, 0x08, 0xd0, 0x32, 0x0c, 0x71, 0x9f, 0x4a, 0x22, 0x8e, 0x95, 0x87, 0x99, + 0xc9, 0x84, 0x17, 0x75, 0x8b, 0xcc, 0x47, 0xa1, 0xfe, 0xad, 0x02, 0x43, 0x25, 0xdb, 0x21, 0xe5, + 0xd5, 0x1a, 0xda, 0x85, 0x51, 0xe9, 0x45, 0xbc, 0x60, 0xb1, 0x39, 0x79, 0x0e, 0xc3, 0x28, 0xa9, + 0xf6, 0xfe, 0x1b, 0xbe, 0xa0, 0x00, 0xcb, 0xb4, 0xd0, 0x2b, 0x74, 0xce, 0xef, 0x38, 0x86, 0x47, + 0x09, 0xf7, 0xe2, 0x64, 0xc3, 0x09, 0x63, 0x1f, 0x17, 0xdf, 0xed, 0xc1, 0x4f, 0x1c, 0x52, 0x51, + 0xab, 0x94, 0xbd, 0xc4, 0xbb, 0x89, 0x1e, 0x87, 0xfe, 0xa6, 0x5d, 0xf7, 0xd7, 0xfd, 0xad, 0x3e, + 0xf3, 0x58, 0xb1, 0xeb, 0x74, 0x6e, 0xcf, 0x26, 0x5b, 0xb0, 0xfb, 0x24, 0xd6, 0x46, 0x5d, 0x85, + 0xc9, 0x38, 0x7d, 0xf4, 0x38, 0x4c, 0xe8, 0x76, 0xb3, 0x69, 0x5b, 0xb5, 0xf6, 0xe6, 0xa6, 0xb1, + 0x43, 0x22, 0x8f, 0x2b, 0x4b, 0x11, 0x08, 0x8e, 0xd5, 0x54, 0x3f, 0xa7, 0x40, 0x1f, 0x5d, 0x17, + 0x15, 0x06, 0xeb, 0x76, 0x53, 0x33, 0x2c, 0xd1, 0x2b, 0x76, 0x4b, 0x5d, 0x66, 0x25, 0x58, 0x40, + 0x50, 0x0b, 0x46, 0x7c, 0x89, 0xac, 0x27, 0xb7, 0xf0, 0xf2, 0x6a, 0x2d, 0x78, 0x55, 0x13, 0x1c, + 0x13, 0x7e, 0x89, 0x8b, 0x43, 0x22, 0xaa, 0x06, 0x53, 0xe5, 0xd5, 0x5a, 0xc5, 0xd2, 0xcd, 0x76, + 0x9d, 0x2c, 0xee, 0xb0, 0x3f, 0x94, 0xcf, 0x19, 0xbc, 0x44, 0x8c, 0x93, 0xf1, 0x39, 0x51, 0x09, + 0xfb, 0x30, 0x5a, 0x8d, 0xf0, 0x16, 0xe2, 0xad, 0x21, 0xab, 0x26, 0x90, 0x60, 0x1f, 0xa6, 0x7e, + 0xa5, 0x00, 0xa3, 0x52, 0x87, 0x90, 0x09, 0x43, 0x7c, 0xb8, 0x6e, 0x2f, 0xef, 0xc9, 0x13, 0xbd, + 0xe6, 0xd4, 0xf9, 0x84, 0xba, 0xd8, 0x27, 0x21, 0xf3, 0xec, 0x42, 0x07, 0x9e, 0x3d, 0x17, 0x79, + 0xb2, 0xc9, 0x3f, 0xc9, 0x89, 0xec, 0xe7, 0x9a, 0xe8, 0xa2, 0x38, 0x9e, 0xb8, 0x5f, 0xf6, 0x70, + 0xec, 0x68, 0xda, 0x84, 0x81, 0xbb, 0xb6, 0x45, 0x5c, 0x61, 0x0b, 0x3f, 0xa2, 0x01, 0x32, 0x6f, + 0xba, 0x17, 0x28, 0x5e, 0xcc, 0xd1, 0xab, 0x3f, 0xa1, 0x00, 0x94, 0x35, 0x4f, 0xe3, 0xae, 0x1b, + 0x5d, 0x78, 0x07, 0x5e, 0x8c, 0x9c, 0xaa, 0xc3, 0x89, 0x97, 0x61, 0xfd, 0xae, 0x71, 0xd7, 0x1f, + 0x7e, 0x20, 0xad, 0x73, 0xec, 0x35, 0xe3, 0x2e, 0xc1, 0x0c, 0x8e, 0x1e, 0x84, 0x11, 0x62, 0xe9, + 0xce, 0x6e, 0x8b, 0x1e, 0x2c, 0xdc, 0xc4, 0xc3, 0xbe, 0xd0, 0x45, 0xbf, 0x10, 0x87, 0x70, 0xf5, + 0x21, 0x88, 0xaa, 0x5c, 0x5d, 0x38, 0x2f, 0xff, 0xbd, 0x02, 0xe7, 0xca, 0x6d, 0xcd, 0x9c, 0x6f, + 0xd1, 0x8d, 0xaa, 0x99, 0x4b, 0x36, 0xf7, 0x01, 0xa0, 0x7a, 0xc8, 0xdb, 0x60, 0xd8, 0x17, 0x72, + 0xe2, 0x4f, 0x47, 0x7d, 0x46, 0x89, 0x83, 0x1a, 0x48, 0x83, 0x61, 0xd7, 0x17, 0xbb, 0x0b, 0x3d, + 0x88, 0xdd, 0x3e, 0x89, 0x40, 0xec, 0x0e, 0xd0, 0x72, 0xe3, 0x17, 0x5b, 0xa0, 0x1a, 0x71, 0xb6, + 0x0d, 0x9d, 0xcc, 0xeb, 0xba, 0xdd, 0xb6, 0x3c, 0x57, 0x3c, 0x05, 0x17, 0xc6, 0xaf, 0xb4, 0x1a, + 0x38, 0xa3, 0xa5, 0xfa, 0x25, 0x05, 0xfa, 0x17, 0xd7, 0x4a, 0x65, 0xf4, 0x6d, 0xd0, 0x1f, 0xb0, + 0x8c, 0x9c, 0xae, 0x3e, 0x14, 0x0f, 0xb7, 0xd3, 0xf1, 0xf5, 0x5e, 0xa1, 0x0c, 0x87, 0x61, 0x45, + 0x1b, 0x30, 0x48, 0xb6, 0x09, 0xed, 0x6a, 0xe1, 0x48, 0xf0, 0x33, 0x96, 0xb6, 0xc8, 0x30, 0x62, + 0x81, 0x59, 0xfd, 0x94, 0x02, 0x10, 0x56, 0x41, 0xdf, 0x99, 0x76, 0x3a, 0xdd, 0x3c, 0x42, 0xeb, + 0x73, 0xe7, 0x23, 0x4a, 0xfd, 0xb5, 0x01, 0x38, 0x4f, 0xbb, 0x23, 0xb6, 0xaa, 0x61, 0x5b, 0x37, + 0xc9, 0xee, 0x37, 0xdc, 0xe4, 0xbf, 0xe1, 0x26, 0x7f, 0x84, 0x6e, 0xf2, 0x2f, 0xc2, 0x79, 0xba, + 0xe5, 0x04, 0x98, 0x30, 0x9f, 0xa1, 0xaa, 0x43, 0x5a, 0x9a, 0x43, 0xea, 0x4c, 0x1f, 0x18, 0xe6, + 0x5e, 0x50, 0xf3, 0x59, 0x95, 0x70, 0x76, 0x7b, 0xf5, 0x29, 0x98, 0x0c, 0xf7, 0xae, 0xf8, 0xa8, + 0x1e, 0x8c, 0x2b, 0x99, 0x23, 0xbe, 0xc4, 0x94, 0x54, 0x0c, 0xd5, 0x7b, 0x0a, 0x4c, 0x2e, 0xee, + 0xb4, 0x0c, 0x87, 0xbd, 0xcf, 0xe7, 0xcf, 0x4c, 0xd0, 0x03, 0xe1, 0x6b, 0x14, 0x25, 0x7a, 0x8b, + 0x17, 0x7f, 0x91, 0x82, 0x36, 0x61, 0x82, 0xb0, 0xe6, 0x4c, 0x0b, 0xd4, 0xbc, 0x3c, 0xdb, 0x9b, + 0x07, 0xa5, 0x88, 0x60, 0xc1, 0x31, 0xac, 0xa8, 0x06, 0x13, 0xba, 0xa9, 0xb9, 0xae, 0xb1, 0x69, + 0xe8, 0xe1, 0x2b, 0xaa, 0x91, 0x85, 0x07, 0x99, 0xcc, 0x15, 0x81, 0xdc, 0xdb, 0x2b, 0x9e, 0x11, + 0xfd, 0x8c, 0x02, 0x70, 0x0c, 0x85, 0xfa, 0x99, 0x02, 0x8c, 0x2f, 0xee, 0xb4, 0x6c, 0xb7, 0xed, + 0x10, 0x56, 0xf5, 0x04, 0xec, 0x5a, 0x0f, 0xc0, 0xd0, 0x96, 0x66, 0xd5, 0x4d, 0xe2, 0x88, 0x63, + 0x37, 0x98, 0xdb, 0x1b, 0xbc, 0x18, 0xfb, 0x70, 0xf4, 0x2a, 0x80, 0xab, 0x6f, 0x91, 0x7a, 0x9b, + 0x31, 0xc7, 0xbe, 0xfc, 0xcc, 0x31, 0x32, 0xc6, 0x5a, 0x80, 0x52, 0x88, 0x34, 0xc1, 0x6f, 0x2c, + 0x91, 0x53, 0xff, 0x44, 0x81, 0xa9, 0x48, 0xbb, 0x13, 0x30, 0xd7, 0x6c, 0x46, 0xcd, 0x35, 0xf3, + 0x3d, 0x8f, 0x35, 0xc3, 0x4a, 0xf3, 0xf1, 0x02, 0x9c, 0xcb, 0x98, 0x93, 0x84, 0xbf, 0xaf, 0x72, + 0x42, 0xfe, 0xbe, 0x6d, 0x18, 0xf5, 0x6c, 0x53, 0x3c, 0xf6, 0xf3, 0x67, 0x20, 0xd7, 0x11, 0xbc, + 0x16, 0xa0, 0x09, 0xbd, 0x79, 0xc3, 0x32, 0x17, 0xcb, 0x74, 0xd4, 0xdf, 0x50, 0x60, 0x24, 0xb0, + 0x0a, 0xbf, 0xae, 0x2e, 0xd4, 0xbb, 0x8f, 0xa3, 0xa3, 0xfe, 0x6e, 0x01, 0xce, 0x06, 0xb8, 0x7d, + 0x36, 0x57, 0xf3, 0x28, 0xdf, 0x38, 0xd8, 0xb4, 0x74, 0x31, 0xf2, 0x12, 0x61, 0x38, 0xf9, 0x30, + 0xaf, 0xd5, 0x76, 0x5a, 0xb6, 0xeb, 0xcb, 0xc1, 0x5c, 0x61, 0xe0, 0x45, 0xd8, 0x87, 0xa1, 0x55, + 0x18, 0x70, 0x29, 0x3d, 0x71, 0xd6, 0x1d, 0x72, 0x36, 0x98, 0x28, 0xcf, 0xfa, 0x8b, 0x39, 0x1a, + 0xf4, 0xaa, 0xcc, 0xc3, 0x07, 0xf2, 0x1b, 0x2f, 0xe9, 0x48, 0xea, 0x81, 0x24, 0x1c, 0x5e, 0x7d, + 0x74, 0x32, 0x16, 0xaa, 0xcb, 0x30, 0x29, 0x9c, 0x5a, 0xf9, 0xb6, 0xb1, 0x74, 0x82, 0xde, 0x1d, + 0xd9, 0x19, 0xf7, 0xc7, 0x5c, 0x6a, 0xa6, 0xe3, 0xf5, 0xc3, 0x1d, 0xa3, 0xba, 0x30, 0x7c, 0x5d, + 0x74, 0x12, 0xcd, 0x42, 0xc1, 0xf0, 0xd7, 0x02, 0x04, 0x8e, 0x42, 0xa5, 0x8c, 0x0b, 0x46, 0x17, + 0x2f, 0x42, 0xe4, 0x63, 0xa9, 0xaf, 0xf3, 0xb1, 0xa4, 0x7e, 0xad, 0x00, 0xd3, 0x3e, 0x55, 0x7f, + 0x8c, 0x65, 0x71, 0x5b, 0x7e, 0x80, 0x52, 0x74, 0xb0, 0xa9, 0xf1, 0x16, 0xf4, 0x33, 0x06, 0x98, + 0xeb, 0x16, 0x3d, 0x40, 0x48, 0xbb, 0x83, 0x19, 0x22, 0xf4, 0x01, 0x18, 0x34, 0xa9, 0x86, 0xe1, + 0x3f, 0xd5, 0xc8, 0x65, 0x98, 0x4d, 0x1b, 0x2e, 0x57, 0x5c, 0x44, 0x08, 0xb3, 0xe0, 0x22, 0x94, + 0x17, 0x62, 0x41, 0x73, 0xf6, 0x31, 0x18, 0x95, 0xaa, 0x1d, 0x2a, 0x7e, 0xd9, 0xe7, 0x0a, 0x30, + 0x73, 0x83, 0x98, 0xcd, 0x54, 0xd7, 0x87, 0x22, 0x0c, 0xe8, 0x5b, 0x9a, 0xc3, 0x43, 0xe3, 0x8d, + 0xf1, 0x4d, 0x5e, 0xa2, 0x05, 0x98, 0x97, 0x53, 0x85, 0x23, 0xf2, 0x38, 0xed, 0x49, 0x69, 0x26, + 0xc3, 0x98, 0x89, 0xef, 0x0f, 0x82, 0x2a, 0x86, 0x03, 0x8f, 0x54, 0xa0, 0xc7, 0xcb, 0x7b, 0x6b, + 0xb7, 0x56, 0xd3, 0x9e, 0xa5, 0xa1, 0xbb, 0x30, 0x6e, 0xeb, 0x06, 0x26, 0x2d, 0xdb, 0x35, 0x3c, + 0xdb, 0xd9, 0x15, 0x8b, 0x96, 0xeb, 0x68, 0xb9, 0x55, 0xaa, 0x84, 0x88, 0xf8, 0xf5, 0x61, 0xa4, + 0x08, 0x47, 0x49, 0xa9, 0x5f, 0x54, 0x60, 0xf4, 0x86, 0xb1, 0x41, 0x1c, 0xee, 0xb7, 0xcb, 0x2c, + 0x24, 0x91, 0x20, 0x6f, 0xa3, 0x69, 0x01, 0xde, 0xd0, 0x0e, 0x8c, 0x88, 0x73, 0x38, 0x78, 0x75, + 0x78, 0x3d, 0x9f, 0xbf, 0x50, 0x40, 0x5a, 0x9c, 0x6f, 0xd2, 0x87, 0xef, 0x97, 0xb8, 0x38, 0x24, + 0xa6, 0xbe, 0x0a, 0xa7, 0x53, 0x1a, 0xd1, 0x85, 0x64, 0xae, 0xab, 0xe2, 0xa3, 0xf1, 0xb9, 0x15, + 0x5d, 0x48, 0x56, 0x8e, 0xce, 0x43, 0x1f, 0xb1, 0xea, 0xe2, 0x8b, 0x19, 0xda, 0xdf, 0x2b, 0xf6, + 0x2d, 0x5a, 0x75, 0x4c, 0xcb, 0x28, 0x13, 0x37, 0xed, 0x88, 0xc4, 0xc6, 0x98, 0xf8, 0xb2, 0x28, + 0xc3, 0x01, 0x94, 0x79, 0x78, 0xc5, 0x9d, 0x99, 0xa8, 0x66, 0x31, 0xb9, 0x19, 0xe3, 0x2d, 0xbd, + 0xf8, 0x50, 0xc5, 0xf9, 0xd4, 0xc2, 0x8c, 0x98, 0x90, 0x04, 0xc7, 0xc3, 0x09, 0xba, 0xea, 0x2f, + 0xf7, 0xc3, 0xa5, 0x1b, 0xb6, 0x63, 0xdc, 0xb5, 0x2d, 0x4f, 0x33, 0xab, 0x76, 0x3d, 0xf4, 0x6f, + 0x15, 0x47, 0xd6, 0xc7, 0x14, 0x38, 0xa7, 0xb7, 0xda, 0x5c, 0x33, 0xf1, 0x5d, 0x44, 0xab, 0xc4, + 0x31, 0xec, 0xbc, 0x0f, 0x35, 0x98, 0x3b, 0x6f, 0xa9, 0xba, 0x9e, 0x86, 0x12, 0x67, 0xd1, 0x62, + 0xef, 0x45, 0xea, 0xf6, 0x1d, 0x8b, 0x75, 0xae, 0xe6, 0xb1, 0xd9, 0xbc, 0x1b, 0x2e, 0x42, 0xce, + 0xf7, 0x22, 0xe5, 0x54, 0x8c, 0x38, 0x83, 0x12, 0xfa, 0x10, 0x9c, 0x31, 0x78, 0xe7, 0x30, 0xd1, + 0xea, 0x86, 0x45, 0x5c, 0x97, 0x3b, 0x9b, 0xf7, 0xf0, 0x20, 0xa2, 0x92, 0x86, 0x10, 0xa7, 0xd3, + 0x41, 0x2f, 0x01, 0xb8, 0xbb, 0x96, 0x2e, 0xe6, 0x3f, 0x9f, 0x23, 0x2a, 0x17, 0x91, 0x03, 0x2c, + 0x58, 0xc2, 0x48, 0x15, 0x2d, 0x2f, 0xd8, 0x94, 0x83, 0xcc, 0x99, 0x98, 0x29, 0x5a, 0xe1, 0x1e, + 0x0a, 0xe1, 0xea, 0xa7, 0x14, 0x98, 0xa8, 0x58, 0x55, 0x53, 0xd3, 0x09, 0xf7, 0xab, 0x74, 0xd1, + 0x35, 0x18, 0x71, 0x83, 0x1b, 0x19, 0xce, 0x11, 0xc2, 0xef, 0x33, 0xb8, 0x8b, 0x09, 0xeb, 0x30, + 0x2f, 0x6d, 0xc3, 0x12, 0x87, 0xdd, 0x92, 0xed, 0x70, 0x44, 0xe2, 0xbb, 0xe3, 0x5e, 0xda, 0x49, + 0x30, 0x4e, 0x6b, 0xa3, 0xfe, 0x9c, 0x02, 0xd3, 0xd1, 0xee, 0x08, 0x97, 0x86, 0x1f, 0x55, 0x60, + 0x3a, 0xf2, 0x98, 0x5a, 0x80, 0x7b, 0x89, 0xef, 0x54, 0x4d, 0xc1, 0xc7, 0x5d, 0x7c, 0xd3, 0x20, + 0x38, 0x95, 0xbe, 0xfa, 0xb3, 0x0a, 0x0c, 0x89, 0x58, 0x8f, 0xe8, 0xad, 0x31, 0xeb, 0x79, 0x70, + 0xb4, 0xc5, 0x2c, 0xe8, 0xbb, 0xcc, 0x3f, 0x43, 0x1c, 0x4d, 0xe2, 0x94, 0xc9, 0x65, 0x7e, 0x15, + 0x84, 0xc3, 0x73, 0x2e, 0xe2, 0xa7, 0xe1, 0x5f, 0xcd, 0x48, 0xc4, 0xd4, 0xd7, 0x14, 0x98, 0x4a, + 0xb4, 0xea, 0x42, 0x1c, 0x3d, 0x41, 0x8f, 0xd5, 0x2f, 0xf7, 0xd3, 0x2d, 0xe9, 0x51, 0x7e, 0x6f, + 0x72, 0xc3, 0xf6, 0x09, 0xe8, 0xbf, 0x0f, 0xc2, 0x88, 0xd1, 0x6c, 0xb6, 0x3d, 0x76, 0x89, 0x38, + 0x10, 0x5a, 0x8b, 0x2b, 0x7e, 0x21, 0x0e, 0xe1, 0xc8, 0x12, 0x92, 0x16, 0x3f, 0x05, 0x97, 0xf3, + 0xad, 0x9c, 0x3c, 0xc0, 0x39, 0x2a, 0x15, 0x71, 0x71, 0x28, 0x4d, 0x10, 0xfb, 0x1e, 0x05, 0xc0, + 0xf5, 0x1c, 0xc3, 0x6a, 0xd0, 0x42, 0x21, 0x8d, 0xe1, 0x23, 0x20, 0x5b, 0x0b, 0x90, 0x72, 0xe2, + 0x61, 0xfc, 0xc7, 0x00, 0x80, 0x25, 0xca, 0x68, 0x5e, 0x08, 0xa1, 0xfc, 0xc8, 0x7c, 0x7b, 0x4c, + 0xdc, 0xbe, 0x94, 0xe2, 0x09, 0xcb, 0x09, 0x85, 0x52, 0xea, 0xec, 0xa3, 0x30, 0x12, 0xd0, 0x3b, + 0x48, 0xa8, 0x1b, 0x93, 0x84, 0xba, 0xd9, 0x27, 0xe0, 0x54, 0xac, 0xbb, 0x87, 0x92, 0x09, 0xff, + 0x54, 0x01, 0x14, 0x1d, 0xfd, 0x09, 0x58, 0x0e, 0x1a, 0x51, 0xcb, 0xc1, 0x42, 0xef, 0x4b, 0x96, + 0x61, 0x3a, 0xf8, 0x69, 0x04, 0x2c, 0x14, 0x6e, 0x10, 0x1a, 0x5a, 0x9c, 0xfc, 0x54, 0x50, 0x09, + 0x5f, 0x80, 0x8b, 0x2f, 0xb7, 0x07, 0x41, 0xe5, 0x66, 0x0c, 0x57, 0x28, 0xa8, 0xc4, 0x21, 0x38, + 0x41, 0x17, 0x7d, 0x42, 0x81, 0x49, 0x2d, 0x1a, 0x0a, 0xd7, 0x9f, 0x99, 0x5c, 0x01, 0xba, 0x62, + 0x61, 0x75, 0xc3, 0xbe, 0xc4, 0x00, 0x2e, 0x4e, 0x90, 0x45, 0x8f, 0xc0, 0x98, 0xd6, 0x32, 0xe6, + 0xdb, 0x75, 0x83, 0x6a, 0x9e, 0x7e, 0xc4, 0x50, 0x66, 0x0d, 0x99, 0xaf, 0x56, 0x82, 0x72, 0x1c, + 0xa9, 0x15, 0xc4, 0x9c, 0x2d, 0x85, 0x8e, 0xc3, 0xbd, 0xc4, 0x9c, 0x15, 0x73, 0x18, 0xc6, 0x9c, + 0x15, 0x53, 0x27, 0x13, 0x41, 0x16, 0x80, 0x6d, 0xd4, 0x75, 0x41, 0x72, 0x30, 0xff, 0x1d, 0xc8, + 0xad, 0x4a, 0xb9, 0x24, 0x28, 0x32, 0xf1, 0x21, 0xfc, 0x8d, 0x25, 0x0a, 0xe8, 0xd3, 0x0a, 0x8c, + 0x0b, 0xde, 0x2d, 0x68, 0x0e, 0xb1, 0x25, 0x7a, 0x21, 0xef, 0x7e, 0x89, 0xed, 0xc9, 0x39, 0x2c, + 0x23, 0xe7, 0x7c, 0x27, 0x08, 0x20, 0x10, 0x81, 0xe1, 0x68, 0x3f, 0x98, 0x0c, 0xe0, 0x46, 0x2e, + 0xa1, 0x44, 0x07, 0x87, 0xf3, 0xcb, 0x00, 0xb5, 0x14, 0x7c, 0xe2, 0x91, 0x51, 0x0a, 0x04, 0xa7, + 0xd2, 0xa7, 0x72, 0xed, 0xa9, 0x3b, 0x9a, 0xa7, 0x6f, 0x95, 0x34, 0x7d, 0x8b, 0xdd, 0x41, 0xf2, + 0xe7, 0x94, 0x39, 0xf7, 0xf5, 0x73, 0x51, 0x54, 0x0b, 0xa7, 0xf7, 0xf7, 0x8a, 0xa7, 0x62, 0x85, + 0x38, 0x4e, 0x10, 0xd9, 0x30, 0xec, 0x88, 0x78, 0xf0, 0xe2, 0x05, 0x7c, 0xbe, 0x10, 0xe8, 0xf1, + 0xe0, 0xf2, 0x5c, 0x33, 0xf2, 0x7f, 0xe1, 0x80, 0x08, 0x6a, 0xc0, 0x25, 0xae, 0x1b, 0xce, 0x5b, + 0xb6, 0xb5, 0xdb, 0xb4, 0xdb, 0xee, 0x7c, 0xdb, 0xdb, 0x22, 0x96, 0xe7, 0x9b, 0xc2, 0x47, 0xd9, + 0x31, 0xca, 0x1e, 0xcd, 0x2d, 0x76, 0xaa, 0x88, 0x3b, 0xe3, 0x41, 0xcf, 0xc3, 0x30, 0xbb, 0xa7, + 0x5b, 0x5b, 0x5b, 0x66, 0x2f, 0x33, 0x0f, 0x2f, 0x2e, 0xb3, 0x21, 0x2c, 0x0a, 0x1c, 0x38, 0xc0, + 0x86, 0x6e, 0xc3, 0x90, 0xc9, 0x03, 0xfa, 0xb3, 0x17, 0x9a, 0x39, 0x99, 0x62, 0x3c, 0x39, 0x00, + 0x57, 0xa0, 0xc5, 0x0f, 0xec, 0x53, 0x40, 0x2d, 0xb8, 0x52, 0x27, 0x9b, 0x5a, 0xdb, 0xf4, 0x56, + 0x6d, 0x0f, 0xb3, 0x17, 0x6a, 0x81, 0xc5, 0xd3, 0x7f, 0x84, 0x3b, 0xc1, 0x62, 0xb0, 0xb1, 0xb7, + 0x7f, 0xe5, 0x03, 0xea, 0xe2, 0x03, 0xb1, 0xa1, 0x5d, 0xb8, 0x4f, 0xd4, 0x61, 0x4f, 0xe2, 0xf4, + 0x2d, 0x3a, 0xcb, 0x49, 0xa2, 0xa7, 0x18, 0xd1, 0x6f, 0xda, 0xdf, 0x2b, 0xde, 0x57, 0x3e, 0xb8, + 0x3a, 0xee, 0x06, 0x27, 0x7b, 0x65, 0x44, 0x62, 0x57, 0x40, 0x33, 0x93, 0xf9, 0xe7, 0x38, 0x7e, + 0x9d, 0xc4, 0xdd, 0xe1, 0xe2, 0xa5, 0x38, 0x41, 0x13, 0x7d, 0x41, 0x81, 0x19, 0xd7, 0x73, 0xda, + 0xba, 0xd7, 0x76, 0x48, 0x3d, 0xb6, 0x43, 0xa7, 0x58, 0x87, 0x72, 0x09, 0x70, 0xb5, 0x0c, 0x9c, + 0xec, 0x39, 0xf8, 0x4c, 0x16, 0x14, 0x67, 0xf6, 0x05, 0x7d, 0x5e, 0x81, 0x73, 0x51, 0x20, 0xd5, + 0xe9, 0x79, 0x3f, 0x51, 0xfe, 0x4b, 0x96, 0x5a, 0x3a, 0x4a, 0xae, 0xc1, 0x67, 0x00, 0x71, 0x56, + 0x47, 0xe2, 0x37, 0xe3, 0xa7, 0x4f, 0xf8, 0x66, 0x7c, 0xf6, 0x69, 0x40, 0xc9, 0xe3, 0xe3, 0x20, + 0x39, 0x70, 0x58, 0x96, 0x03, 0x3f, 0x3b, 0x00, 0x17, 0xe8, 0xa9, 0x14, 0x6a, 0x3f, 0x2b, 0x9a, + 0xa5, 0x35, 0x5e, 0x9f, 0x12, 0xd3, 0x17, 0x15, 0x38, 0xb7, 0x95, 0x6e, 0xda, 0x11, 0xfa, 0xd7, + 0x33, 0xb9, 0x4c, 0x70, 0x9d, 0xac, 0x45, 0x9c, 0x61, 0x77, 0xac, 0x82, 0xb3, 0x3a, 0x85, 0x9e, + 0x86, 0x49, 0xcb, 0xae, 0x93, 0x52, 0xa5, 0x8c, 0x57, 0x34, 0xf7, 0x76, 0xcd, 0x77, 0xd4, 0x19, + 0xe0, 0xdf, 0xeb, 0x6a, 0x0c, 0x86, 0x13, 0xb5, 0xd1, 0x36, 0xa0, 0x96, 0x5d, 0x5f, 0xdc, 0xe6, + 0x89, 0x2f, 0x7a, 0x73, 0x99, 0x65, 0xce, 0x02, 0xd5, 0x04, 0x36, 0x9c, 0x42, 0x81, 0xd9, 0xa6, + 0x68, 0x67, 0x56, 0x6c, 0xcb, 0xf0, 0x6c, 0x87, 0x05, 0x38, 0xe8, 0xc9, 0x44, 0xc3, 0x6c, 0x53, + 0xab, 0xa9, 0x18, 0x71, 0x06, 0x25, 0xf5, 0xbf, 0x2b, 0x70, 0x8a, 0x6e, 0x8b, 0xaa, 0x63, 0xef, + 0xec, 0xbe, 0x1e, 0x37, 0xe4, 0x03, 0xc2, 0x67, 0x91, 0xdb, 0x76, 0xce, 0x48, 0xfe, 0x8a, 0x23, + 0xac, 0xcf, 0xa1, 0x8b, 0xa2, 0x6c, 0x56, 0xee, 0xcb, 0x36, 0x2b, 0xab, 0x9f, 0x2e, 0x70, 0xcd, + 0xc5, 0x37, 0xeb, 0xbe, 0x2e, 0xbf, 0xc3, 0x47, 0x61, 0x9c, 0x96, 0xad, 0x68, 0x3b, 0xd5, 0xf2, + 0xb3, 0xb6, 0xe9, 0xbf, 0xc6, 0x66, 0xb6, 0xf6, 0x9b, 0x32, 0x00, 0x47, 0xeb, 0xa1, 0xc7, 0x61, + 0xa8, 0xc5, 0xe3, 0x84, 0x09, 0x9d, 0xf9, 0x0a, 0x77, 0xec, 0x63, 0x45, 0xf7, 0xf6, 0x8a, 0x53, + 0xe1, 0x15, 0xaf, 0x1f, 0xad, 0xcc, 0x6f, 0xa0, 0xfe, 0xf2, 0x59, 0x60, 0xc8, 0x4d, 0xe2, 0xbd, + 0x1e, 0xe7, 0xe4, 0x21, 0x18, 0xd5, 0x5b, 0xed, 0xd2, 0x52, 0xed, 0x99, 0xb6, 0xcd, 0x6c, 0x21, + 0x2c, 0x3f, 0x0c, 0xe5, 0xde, 0xa5, 0xea, 0xba, 0x5f, 0x8c, 0xe5, 0x3a, 0x94, 0x3b, 0xe8, 0xad, + 0xb6, 0xe0, 0xb7, 0x55, 0xf9, 0xbd, 0x0a, 0xe3, 0x0e, 0xa5, 0xea, 0x7a, 0x04, 0x86, 0x13, 0xb5, + 0xd1, 0x87, 0x60, 0x8c, 0x88, 0x0f, 0xf7, 0x86, 0xe6, 0xd4, 0x05, 0x5f, 0xa8, 0xe4, 0x1d, 0x7c, + 0x30, 0xb5, 0x3e, 0x37, 0xe0, 0x1a, 0xe0, 0xa2, 0x44, 0x02, 0x47, 0x08, 0xa2, 0x17, 0xe1, 0xbc, + 0xff, 0x9b, 0xae, 0xb2, 0x5d, 0x8f, 0x33, 0x8a, 0x01, 0xee, 0x36, 0xb3, 0x98, 0x55, 0x09, 0x67, + 0xb7, 0x47, 0x3f, 0xa3, 0xc0, 0xd9, 0x00, 0x6a, 0x58, 0x46, 0xb3, 0xdd, 0xc4, 0x44, 0x37, 0x35, + 0xa3, 0x29, 0xf4, 0xbe, 0xe7, 0x8e, 0x6c, 0xa0, 0x51, 0xf4, 0x9c, 0x59, 0xa5, 0xc3, 0x70, 0x46, + 0x97, 0xd0, 0x6b, 0x0a, 0x5c, 0xf1, 0x41, 0x55, 0x87, 0xb8, 0x6e, 0xdb, 0x21, 0x61, 0x2c, 0x00, + 0x31, 0x25, 0x43, 0xb9, 0x78, 0x27, 0x13, 0x80, 0x17, 0x0f, 0xc0, 0x8d, 0x0f, 0xa4, 0x2e, 0x6f, + 0x97, 0x9a, 0xbd, 0xe9, 0x09, 0x45, 0xf1, 0xb8, 0xb6, 0x0b, 0x25, 0x81, 0x23, 0x04, 0xd1, 0xcf, + 0x29, 0x70, 0x4e, 0x2e, 0x90, 0x77, 0x0b, 0xd7, 0x10, 0x9f, 0x3f, 0xb2, 0xce, 0xc4, 0xf0, 0x73, + 0x09, 0x2f, 0x03, 0x88, 0xb3, 0x7a, 0x45, 0xd9, 0x76, 0x93, 0x6d, 0x4c, 0xae, 0x45, 0x0e, 0x70, + 0xb6, 0xcd, 0xf7, 0xaa, 0x8b, 0x7d, 0x18, 0x7a, 0x04, 0xc6, 0x5a, 0x76, 0xbd, 0x6a, 0xd4, 0x5d, + 0x16, 0xc3, 0x8c, 0xe9, 0x7a, 0x7d, 0x7c, 0x3a, 0xaa, 0x76, 0xbd, 0x5a, 0x29, 0xf3, 0x72, 0x1c, + 0xa9, 0x85, 0xe6, 0x00, 0x36, 0x35, 0xc3, 0xac, 0xdd, 0xd1, 0x5a, 0xb7, 0xfc, 0xa0, 0x38, 0xcc, + 0x16, 0xb1, 0x14, 0x94, 0x62, 0xa9, 0x06, 0x5d, 0x3f, 0xca, 0x77, 0x30, 0xe1, 0x61, 0xa7, 0x99, + 0x7a, 0x74, 0x14, 0xeb, 0xe7, 0x23, 0xe4, 0x1d, 0xbe, 0x29, 0x91, 0xc0, 0x11, 0x82, 0xe8, 0x63, + 0x0a, 0x4c, 0xb8, 0xbb, 0xae, 0x47, 0x9a, 0x41, 0x1f, 0x4e, 0x1d, 0x75, 0x1f, 0x98, 0x4d, 0xbc, + 0x16, 0x21, 0x82, 0x63, 0x44, 0x59, 0x78, 0xa1, 0xa6, 0xd6, 0x20, 0xd7, 0x4b, 0x37, 0x8c, 0xc6, + 0x56, 0x10, 0xdd, 0xa5, 0x4a, 0x1c, 0x9d, 0x58, 0x1e, 0x53, 0xac, 0x06, 0x44, 0x78, 0xa1, 0xec, + 0x6a, 0xb8, 0x13, 0x0e, 0xf4, 0x12, 0xcc, 0x0a, 0xf0, 0xb2, 0x7d, 0x27, 0x41, 0x81, 0x47, 0xf1, + 0x61, 0x0e, 0x90, 0x95, 0xcc, 0x5a, 0xb8, 0x03, 0x06, 0x54, 0x81, 0xd3, 0x2e, 0x71, 0xd8, 0x9d, + 0x20, 0x8f, 0x08, 0x59, 0x6d, 0x9b, 0xa6, 0xcb, 0x54, 0x1b, 0xf1, 0xe4, 0xa7, 0x96, 0x04, 0xe3, + 0xb4, 0x36, 0xe8, 0x89, 0xe0, 0x59, 0xf0, 0x2e, 0x2d, 0x78, 0xa6, 0x5a, 0x63, 0x9a, 0xc8, 0x00, + 0xb7, 0x94, 0xe0, 0x28, 0x08, 0xc7, 0xeb, 0xd2, 0xd3, 0xdc, 0x2f, 0x5a, 0x68, 0x3b, 0xae, 0x37, 0x33, 0xcd, 0x1a, 0xb3, 0xd3, 0x1c, 0xcb, 0x00, 0x1c, 0xad, 0x87, 0x1e, 0x87, 0x09, 0x97, 0xe8, - 0xba, 0xdd, 0x68, 0x0a, 0x3d, 0x79, 0xe6, 0x0c, 0xeb, 0x3d, 0x5f, 0xc1, 0x08, 0x04, 0xc7, 0x6a, - 0xa2, 0x3d, 0x38, 0x1d, 0x84, 0xd0, 0x5d, 0xb6, 0xeb, 0x2b, 0xda, 0x2e, 0x13, 0x8e, 0xcf, 0xe6, + 0xba, 0xdd, 0x6c, 0x09, 0x3d, 0x79, 0xe6, 0x0c, 0xeb, 0x3d, 0x5f, 0xc1, 0x08, 0x04, 0xc7, 0x6a, + 0xa2, 0x5d, 0x38, 0x1d, 0x84, 0xd0, 0x5d, 0xb6, 0x1b, 0x2b, 0xda, 0x0e, 0x13, 0x8e, 0xcf, 0xe6, 0x09, 0xf1, 0xc0, 0xa7, 0xab, 0x94, 0x44, 0x87, 0xd3, 0x68, 0xa0, 0x65, 0x98, 0x8e, 0x15, 0x2f, 0x19, 0x26, 0x71, 0x67, 0xce, 0xb1, 0x61, 0x33, 0x63, 0x57, 0x29, 0x05, 0x8e, 0x53, 0x5b, 0xa1, - 0x5b, 0x70, 0xa6, 0xe9, 0xd8, 0x1e, 0xd1, 0xbd, 0x9b, 0x54, 0x20, 0x30, 0xc5, 0x00, 0xdd, 0x99, - 0x19, 0x36, 0x17, 0xec, 0x3e, 0x74, 0x2d, 0xad, 0x02, 0x4e, 0x6f, 0x87, 0x3e, 0xa7, 0xc0, 0x65, - 0xd7, 0x73, 0x88, 0xd6, 0x30, 0xac, 0x7a, 0xc9, 0xb6, 0x2c, 0xc2, 0x18, 0x53, 0xa5, 0x16, 0xbe, - 0x98, 0x3b, 0x9f, 0xeb, 0x14, 0x51, 0xf7, 0xdb, 0xc5, 0xcb, 0xd5, 0x8e, 0x98, 0xf1, 0x01, 0x94, - 0xd1, 0xab, 0x00, 0x0d, 0xd2, 0xb0, 0x9d, 0x3d, 0xca, 0x91, 0x66, 0x66, 0xf3, 0xeb, 0xbb, 0x2b, + 0x5b, 0x70, 0xa6, 0xe5, 0xd8, 0x1e, 0xd1, 0xbd, 0x9b, 0x54, 0x20, 0x30, 0xc5, 0x00, 0xdd, 0x99, + 0x19, 0x36, 0x17, 0xec, 0x3e, 0xb4, 0x9a, 0x56, 0x01, 0xa7, 0xb7, 0x43, 0x9f, 0x55, 0xe0, 0xb2, + 0xeb, 0x39, 0x44, 0x6b, 0x1a, 0x56, 0xa3, 0x64, 0x5b, 0x16, 0x61, 0x8c, 0xa9, 0x52, 0x0f, 0x5f, + 0xcc, 0x9d, 0xcf, 0x75, 0x8a, 0xa8, 0xfb, 0x7b, 0xc5, 0xcb, 0xb5, 0x8e, 0x98, 0xf1, 0x01, 0x94, + 0xd1, 0xab, 0x00, 0x4d, 0xd2, 0xb4, 0x9d, 0x5d, 0xca, 0x91, 0x66, 0x66, 0xf3, 0xeb, 0xbb, 0x2b, 0x01, 0x16, 0xfe, 0xf9, 0x47, 0x6e, 0x72, 0x43, 0x20, 0x96, 0xc8, 0xd1, 0xa9, 0xa6, 0xfc, 0x56, 0x44, 0xb1, 0x92, 0x3e, 0x9a, 0x0b, 0x6c, 0xe5, 0xd8, 0x54, 0xaf, 0xa4, 0x55, 0xc0, 0xe9, 0xed, - 0x90, 0x0b, 0x53, 0xec, 0x0b, 0x15, 0x27, 0xf9, 0xf5, 0xd2, 0x7c, 0x9d, 0xcc, 0x5c, 0xcc, 0x35, - 0xb9, 0x54, 0xf6, 0x9f, 0xaa, 0xc4, 0x91, 0xe1, 0x24, 0xfe, 0x90, 0xa8, 0xb6, 0x1b, 0x12, 0xbd, - 0xd4, 0x2b, 0x51, 0x09, 0x19, 0x4e, 0xe2, 0x57, 0xdb, 0x05, 0x38, 0x93, 0x7a, 0x4a, 0x52, 0xe6, - 0xc1, 0xa7, 0x78, 0xde, 0xcf, 0xff, 0x25, 0xae, 0x3d, 0x19, 0xf3, 0x58, 0x89, 0x82, 0x70, 0xbc, - 0x2e, 0x95, 0x61, 0x19, 0xb5, 0xa5, 0x6a, 0xd8, 0xbe, 0x10, 0xca, 0xb0, 0x95, 0x18, 0x0c, 0x27, - 0x6a, 0xa3, 0x92, 0x98, 0x8f, 0xa5, 0x6a, 0x85, 0xaa, 0x81, 0xee, 0x92, 0x43, 0x7c, 0xed, 0x20, - 0x1c, 0x9f, 0x0c, 0xc4, 0xc9, 0xfa, 0x74, 0x14, 0xf4, 0x87, 0xdc, 0x8b, 0xfe, 0x70, 0x14, 0xab, - 0x51, 0x10, 0x8e, 0xd7, 0xf5, 0xf5, 0xf4, 0x48, 0x17, 0x06, 0xc2, 0x51, 0xac, 0xc6, 0x60, 0x38, - 0x51, 0x5b, 0xfd, 0xd3, 0x7e, 0xb8, 0xaf, 0x0b, 0xc9, 0x12, 0x35, 0xd2, 0xa7, 0xfb, 0xf0, 0x3c, - 0xaf, 0xbb, 0xe5, 0x69, 0x66, 0x2c, 0xcf, 0xe1, 0xe9, 0x75, 0xbb, 0x9c, 0x6e, 0xd6, 0x72, 0x1e, - 0x9e, 0x64, 0xf7, 0xcb, 0xdf, 0x48, 0x5f, 0xfe, 0x9c, 0xb3, 0x7a, 0xe0, 0x76, 0x69, 0x66, 0x6c, - 0x97, 0x9c, 0xb3, 0xda, 0xc5, 0xf6, 0xfa, 0xb3, 0x7e, 0xb8, 0xbf, 0x1b, 0x29, 0x37, 0xe7, 0xfe, - 0x4a, 0xe1, 0x2d, 0xc7, 0xba, 0xbf, 0xb2, 0xde, 0x73, 0x1f, 0xe3, 0xfe, 0xea, 0xc8, 0x3e, 0x8f, - 0x67, 0x7f, 0x65, 0xcd, 0xea, 0x71, 0xed, 0xaf, 0xac, 0x59, 0xed, 0x62, 0x7f, 0xfd, 0x75, 0xfc, - 0x7c, 0x08, 0x44, 0xed, 0x0a, 0xf4, 0xe9, 0xcd, 0x56, 0x4e, 0x26, 0xc5, 0xbc, 0x0c, 0x4b, 0x6b, - 0x1b, 0x98, 0xe2, 0x40, 0x18, 0x06, 0xf9, 0xfe, 0xc9, 0xc9, 0x82, 0x98, 0xe7, 0x28, 0xdf, 0x92, - 0x58, 0x60, 0xa2, 0x53, 0x45, 0x9a, 0xdb, 0xa4, 0x41, 0x1c, 0xcd, 0xac, 0x7a, 0xb6, 0xa3, 0xd5, - 0xf3, 0x72, 0x1b, 0x7e, 0x83, 0x12, 0xc3, 0x85, 0x13, 0xd8, 0xe9, 0x84, 0x34, 0x8d, 0x5a, 0x4e, - 0xfe, 0xc2, 0x26, 0x64, 0xad, 0x52, 0xc6, 0x14, 0x87, 0xfa, 0xab, 0xc3, 0x20, 0x05, 0xa7, 0x47, - 0x9f, 0x54, 0x60, 0x4a, 0x8f, 0xc7, 0xe4, 0xec, 0xc5, 0x1f, 0x2a, 0x11, 0xe0, 0x93, 0x6f, 0xf9, - 0x44, 0x31, 0x4e, 0x92, 0x45, 0x1f, 0x56, 0xb8, 0x91, 0x2f, 0xb8, 0xcd, 0x13, 0xd3, 0x7a, 0xfd, - 0x88, 0xee, 0xbd, 0x43, 0x6b, 0x61, 0x78, 0xc5, 0x1a, 0x25, 0x88, 0x5e, 0x53, 0xe0, 0xcc, 0xed, - 0xb4, 0xbb, 0x09, 0x31, 0xf9, 0xb7, 0xf2, 0x76, 0x25, 0xe3, 0xb2, 0x83, 0x4b, 0x90, 0xa9, 0x15, - 0x70, 0x7a, 0x47, 0x82, 0x59, 0x0a, 0xcc, 0xb5, 0xe2, 0x3b, 0xcd, 0x3d, 0x4b, 0x31, 0xbb, 0x6f, - 0x38, 0x4b, 0x01, 0x00, 0x47, 0x09, 0xa2, 0x26, 0x8c, 0xdc, 0xf6, 0x6d, 0xe4, 0xc2, 0x2e, 0x56, - 0xca, 0x4b, 0x5d, 0x32, 0xb4, 0x73, 0x7f, 0xaf, 0xa0, 0x10, 0x87, 0x44, 0xd0, 0x36, 0x0c, 0xdd, - 0xe6, 0xbc, 0x42, 0xd8, 0xb3, 0xe6, 0x7b, 0xd6, 0xfe, 0xb9, 0x59, 0x45, 0x14, 0x61, 0x1f, 0xbd, - 0xfc, 0x96, 0x60, 0xf8, 0x80, 0x27, 0x6e, 0x9f, 0x53, 0xe0, 0xcc, 0x0e, 0x71, 0x3c, 0x43, 0x8f, - 0xdf, 0x0c, 0x8d, 0xe4, 0xb7, 0x50, 0x3c, 0x9b, 0x86, 0x90, 0x6f, 0x93, 0x54, 0x10, 0x4e, 0xef, - 0x02, 0x7a, 0x16, 0xfa, 0x89, 0xa7, 0xd7, 0x44, 0x00, 0xe7, 0x77, 0xe7, 0x7d, 0xb2, 0xcb, 0xdf, - 0xbd, 0xd0, 0xff, 0x30, 0xc3, 0xa7, 0xfe, 0xa5, 0x02, 0x09, 0xdb, 0x34, 0xfa, 0x41, 0x05, 0xc6, - 0xb6, 0x88, 0xe6, 0xb5, 0x1c, 0x72, 0x5d, 0xf8, 0x84, 0xf6, 0x5d, 0x1d, 0x7d, 0xf8, 0xd9, 0xa3, - 0x30, 0x89, 0xcf, 0x2d, 0x49, 0x88, 0xb9, 0xb3, 0x4a, 0x90, 0x70, 0x42, 0x06, 0xe1, 0x48, 0x0f, - 0x66, 0x9f, 0x82, 0xa9, 0x44, 0xc3, 0x43, 0x5d, 0x53, 0xfe, 0x0b, 0x05, 0xd2, 0x72, 0x9b, 0xa3, - 0x97, 0x60, 0x40, 0xab, 0xd5, 0x82, 0x14, 0x97, 0x8f, 0xe5, 0xf3, 0x9b, 0xaa, 0xc9, 0x91, 0x82, - 0xd8, 0x4f, 0xcc, 0xd1, 0xa2, 0x25, 0x40, 0x5a, 0xe4, 0x5e, 0x7a, 0x25, 0x0c, 0x51, 0xc1, 0xae, - 0xd3, 0xe6, 0x13, 0x50, 0x9c, 0xd2, 0x42, 0xfd, 0xb8, 0x02, 0x28, 0x99, 0xa2, 0x04, 0x39, 0x30, - 0x2c, 0xf6, 0xaf, 0xbf, 0x4a, 0xe5, 0x9c, 0xaf, 0xe9, 0x22, 0x4f, 0x43, 0x43, 0x27, 0x3c, 0x51, - 0xe0, 0xe2, 0x80, 0x8e, 0xfa, 0xb7, 0x0a, 0x84, 0x69, 0xf0, 0xd0, 0x3b, 0x61, 0xb4, 0x46, 0x5c, - 0xdd, 0x31, 0x9a, 0x5e, 0xf8, 0x90, 0x34, 0x78, 0x90, 0x56, 0x0e, 0x41, 0x58, 0xae, 0x87, 0x54, - 0x18, 0xf4, 0x34, 0xf7, 0x76, 0xa5, 0x2c, 0x94, 0x3d, 0x76, 0x34, 0xaf, 0xb3, 0x12, 0x2c, 0x20, - 0x61, 0xe8, 0xd8, 0xbe, 0x2e, 0x42, 0xc7, 0xa2, 0xad, 0x23, 0x88, 0x93, 0x8b, 0x0e, 0x8e, 0x91, - 0xab, 0xfe, 0x74, 0x01, 0x4e, 0xd1, 0x2a, 0x2b, 0x9a, 0x61, 0x79, 0xc4, 0x62, 0xcf, 0xa6, 0x72, - 0x4e, 0x42, 0x1d, 0xc6, 0xbd, 0xc8, 0xa3, 0xe5, 0xc3, 0x3f, 0xaa, 0x0d, 0x3c, 0xbd, 0xa2, 0x4f, - 0x95, 0xa3, 0x78, 0xd1, 0x63, 0xfe, 0xbb, 0x35, 0xae, 0x16, 0xdf, 0xe7, 0x6f, 0x55, 0xf6, 0x18, - 0xed, 0x9e, 0x78, 0x01, 0x1e, 0xe4, 0x4e, 0x8c, 0x3c, 0x51, 0x7b, 0x14, 0xc6, 0xc5, 0x0b, 0x09, - 0x1e, 0x03, 0x58, 0xa8, 0xc5, 0xec, 0x58, 0x59, 0x92, 0x01, 0x38, 0x5a, 0x4f, 0xfd, 0x83, 0x02, - 0x44, 0x33, 0x34, 0xe6, 0x9d, 0xa5, 0x64, 0x00, 0xe4, 0xc2, 0xb1, 0x05, 0x40, 0xe6, 0x39, 0xca, - 0x99, 0x3b, 0xb7, 0xb8, 0x67, 0x97, 0xf3, 0x13, 0xb3, 0x72, 0x1c, 0xd4, 0x08, 0xa7, 0xb5, 0xff, - 0xd0, 0xd3, 0xfa, 0x4e, 0xe1, 0xf9, 0x3b, 0x10, 0x09, 0x43, 0xed, 0x7b, 0xfe, 0x4e, 0x45, 0x1a, - 0x4a, 0xaf, 0xec, 0xe6, 0x41, 0xe4, 0x38, 0xa1, 0xeb, 0x22, 0x42, 0x64, 0xbb, 0xeb, 0xb6, 0xa7, - 0x99, 0x6c, 0x3a, 0x85, 0xd1, 0x75, 0x45, 0x06, 0xe0, 0x68, 0x3d, 0x75, 0x15, 0xde, 0xbc, 0x6c, - 0x6b, 0xb5, 0x05, 0xcd, 0xa4, 0x5b, 0xd7, 0x11, 0x6e, 0x79, 0x2e, 0x3b, 0x99, 0xd7, 0x1c, 0xdb, - 0xb3, 0x75, 0xdb, 0xa4, 0xe7, 0xa6, 0x16, 0xc4, 0x0a, 0x66, 0x17, 0x91, 0xfe, 0xb9, 0x29, 0x62, - 0xed, 0x62, 0x1f, 0xae, 0xfe, 0x96, 0x02, 0x43, 0x22, 0xef, 0x49, 0x17, 0x0f, 0x4b, 0xb7, 0x60, - 0x80, 0x69, 0x47, 0xbd, 0x48, 0xa5, 0xd5, 0x6d, 0xdb, 0xf6, 0x22, 0x09, 0x93, 0x78, 0xca, 0x31, - 0x96, 0x24, 0x92, 0xa3, 0x67, 0xfe, 0xa8, 0x8e, 0xbe, 0x6d, 0x78, 0x84, 0xb9, 0xdd, 0x88, 0x8d, - 0xcf, 0xfd, 0x51, 0xa5, 0x72, 0x1c, 0xa9, 0xa5, 0xfe, 0xb7, 0x01, 0xb8, 0xe2, 0xe7, 0x70, 0x89, - 0x8b, 0x6a, 0x01, 0xcf, 0xdd, 0x0b, 0x92, 0xca, 0xb0, 0xa0, 0xf8, 0xbe, 0x4d, 0x35, 0x9f, 0x96, - 0x7c, 0x4e, 0x4a, 0x1b, 0x23, 0xa3, 0xc3, 0x69, 0x34, 0x78, 0xf4, 0x75, 0x56, 0x7c, 0x83, 0x68, - 0xa6, 0xb7, 0xed, 0xd3, 0x2e, 0xf4, 0x12, 0x7d, 0x3d, 0x89, 0x0f, 0xa7, 0x52, 0x61, 0x2e, 0x1d, - 0x7e, 0x32, 0x1b, 0x87, 0x68, 0xb2, 0x3f, 0x49, 0x0f, 0xcf, 0x8d, 0x56, 0x52, 0x31, 0xe2, 0x0c, - 0x4a, 0xcc, 0xdc, 0xa8, 0xed, 0x32, 0xeb, 0x05, 0x26, 0x9e, 0x63, 0xb0, 0xc4, 0x57, 0xc1, 0x5d, - 0xc5, 0x4a, 0x14, 0x84, 0xe3, 0x75, 0xd1, 0xe3, 0x30, 0xc1, 0x5c, 0x64, 0xc2, 0x70, 0x9e, 0x03, - 0x61, 0x50, 0xa7, 0xd5, 0x08, 0x04, 0xc7, 0x6a, 0xa2, 0xef, 0x51, 0x60, 0xda, 0x90, 0x1f, 0xd3, - 0xf8, 0xa3, 0xcf, 0x97, 0x19, 0x85, 0x5d, 0x2f, 0xf9, 0xdb, 0x38, 0x05, 0x2d, 0x4e, 0x25, 0x86, - 0x96, 0x61, 0x5a, 0xbc, 0x88, 0x8e, 0xee, 0x01, 0x1e, 0xeb, 0x8d, 0xad, 0x69, 0x39, 0x05, 0x8e, - 0x53, 0x5b, 0xa9, 0x1f, 0x29, 0xc0, 0xd8, 0x21, 0x93, 0x98, 0xb6, 0x24, 0x99, 0xa3, 0x87, 0x77, - 0x8b, 0x32, 0xd5, 0x2e, 0xc4, 0x0e, 0xf4, 0x3c, 0x4c, 0xb4, 0xd8, 0x44, 0xf8, 0x91, 0xd0, 0xc4, - 0x37, 0xfd, 0x6d, 0x74, 0xe5, 0x36, 0x22, 0x90, 0x7b, 0xed, 0xe2, 0xac, 0x8c, 0x3e, 0x0a, 0xc5, - 0x31, 0x3c, 0xea, 0xaf, 0x15, 0x00, 0xc9, 0xd5, 0x97, 0x4c, 0x6d, 0xc7, 0x76, 0xd0, 0x4f, 0x2a, - 0x30, 0xa6, 0xcb, 0x69, 0xa3, 0xb8, 0x80, 0xf5, 0x7c, 0xaf, 0x83, 0xe5, 0xe8, 0xe7, 0xe4, 0x7c, - 0x50, 0x5c, 0x10, 0xf6, 0x9f, 0x5b, 0x8f, 0xc9, 0xa0, 0x7b, 0xb1, 0xdf, 0x38, 0xd2, 0xa7, 0xd9, - 0x8f, 0x29, 0x30, 0x95, 0xc0, 0x94, 0x22, 0x19, 0xbf, 0x10, 0x8d, 0x81, 0x7e, 0x24, 0x09, 0x22, - 0x65, 0xf9, 0xfa, 0x27, 0x06, 0xe0, 0x74, 0xca, 0x8a, 0x32, 0x17, 0x1b, 0x12, 0x93, 0x2e, 0x7b, - 0x71, 0xb1, 0x49, 0x48, 0xaa, 0x81, 0x8b, 0x4d, 0x1c, 0x82, 0x13, 0x74, 0xd1, 0xb3, 0xd0, 0xa7, - 0x3b, 0x86, 0xd8, 0xb4, 0x8f, 0xe6, 0x9a, 0x02, 0x5c, 0x59, 0x18, 0x15, 0x14, 0xfb, 0x4a, 0xb8, - 0x82, 0x29, 0x42, 0x7a, 0x16, 0xcb, 0xc7, 0x88, 0x2f, 0xb0, 0xb2, 0xb3, 0x58, 0x3e, 0x6d, 0x5c, - 0x1c, 0xad, 0x87, 0x9e, 0x87, 0x19, 0xa1, 0xa9, 0xfa, 0x91, 0x4c, 0x6c, 0xcb, 0xf5, 0x28, 0xc7, - 0xf7, 0x84, 0x4c, 0xc1, 0xbc, 0x5f, 0x6f, 0x66, 0xd4, 0xc1, 0x99, 0xad, 0xd1, 0x77, 0xc3, 0x44, - 0x84, 0x7b, 0xf8, 0x51, 0xc9, 0x72, 0x3e, 0x99, 0x91, 0x31, 0x71, 0x5e, 0x19, 0x2d, 0xc3, 0x31, - 0x6a, 0xe8, 0xff, 0x55, 0x60, 0x2a, 0xd8, 0xa8, 0x7b, 0x7c, 0xcb, 0xfb, 0x19, 0xfc, 0x96, 0x8e, - 0xe6, 0x0b, 0x92, 0xd2, 0xab, 0xc6, 0x09, 0xe1, 0x24, 0x6d, 0xf5, 0x47, 0x06, 0x41, 0xce, 0xe8, - 0x8c, 0x56, 0x7a, 0xb1, 0x4b, 0x86, 0x7b, 0xc0, 0xb7, 0x4d, 0xae, 0x40, 0x5f, 0xbd, 0xd9, 0xca, - 0x69, 0x98, 0x0c, 0xd0, 0x5d, 0xa7, 0xe8, 0xea, 0xcd, 0x16, 0x7a, 0x36, 0x30, 0x75, 0xe6, 0x33, - 0x46, 0x06, 0x4f, 0x25, 0x63, 0xe6, 0x4e, 0x9f, 0xbd, 0xf7, 0x67, 0xb2, 0xf7, 0x06, 0x0c, 0xb9, - 0xc2, 0x0e, 0x3a, 0x90, 0x3f, 0x8c, 0xa4, 0x34, 0xd3, 0xc2, 0xee, 0xc9, 0x2d, 0x34, 0xbe, 0x59, - 0xd4, 0xa7, 0x41, 0x15, 0xc1, 0x16, 0x3b, 0x97, 0x44, 0x90, 0x24, 0xa6, 0x08, 0x6e, 0xb0, 0x12, - 0x2c, 0x20, 0x09, 0x61, 0x6e, 0xa8, 0x1b, 0x61, 0x0e, 0x7d, 0x3e, 0xce, 0xbf, 0x87, 0xd9, 0xee, - 0x7b, 0xa6, 0xc7, 0xe1, 0xfc, 0x5f, 0xc6, 0xb8, 0xff, 0x9f, 0xf0, 0xf0, 0x93, 0x16, 0x0b, 0xdd, - 0x07, 0x03, 0x2c, 0x8a, 0x92, 0x90, 0x03, 0x02, 0xe3, 0x06, 0x8b, 0xa3, 0x83, 0x39, 0x0c, 0x55, - 0x45, 0xe8, 0xc0, 0x7c, 0x9b, 0x9e, 0xf9, 0x36, 0x0a, 0x7a, 0x52, 0x9c, 0xc1, 0x2b, 0x91, 0x37, - 0x91, 0x69, 0x3a, 0xc4, 0x06, 0x0c, 0x35, 0x0c, 0x8b, 0xb9, 0x7b, 0xe4, 0x33, 0xa2, 0x73, 0x17, - 0x2c, 0x8e, 0x02, 0xfb, 0xb8, 0xd4, 0x3f, 0x2b, 0x50, 0x06, 0x11, 0x2a, 0xf5, 0x7b, 0x3c, 0x10, - 0xae, 0x78, 0x7d, 0xad, 0xe4, 0x37, 0x02, 0x4a, 0x48, 0xe7, 0x03, 0x84, 0xdc, 0x51, 0x21, 0xfc, - 0x8d, 0x25, 0x62, 0x94, 0xb4, 0x67, 0x34, 0xc8, 0x73, 0x86, 0x55, 0xb3, 0xef, 0x88, 0xe9, 0xed, - 0x95, 0xf4, 0x7a, 0x80, 0x90, 0x93, 0x0e, 0x7f, 0x63, 0x89, 0x18, 0x3d, 0x92, 0x58, 0x32, 0x0e, - 0x8b, 0x65, 0xe2, 0x15, 0x7d, 0xe3, 0x79, 0xb6, 0x85, 0xdf, 0x31, 0x3b, 0x92, 0x4a, 0x19, 0x75, - 0x70, 0x66, 0x6b, 0xf5, 0x67, 0x15, 0x38, 0x93, 0x3a, 0x15, 0xe8, 0x3a, 0x4c, 0x85, 0xee, 0xb0, - 0xb2, 0x90, 0x30, 0x1c, 0xf2, 0xf8, 0x9b, 0xf1, 0x0a, 0x38, 0xd9, 0x86, 0xe7, 0x37, 0x4b, 0x08, - 0x21, 0xc2, 0x97, 0x56, 0x56, 0xb5, 0x64, 0x30, 0x4e, 0x6b, 0xa3, 0xbe, 0x18, 0xe9, 0x6c, 0x38, - 0x59, 0xf4, 0xcb, 0xd8, 0x24, 0xf5, 0xe0, 0x4d, 0x7a, 0xf0, 0x65, 0x2c, 0xd0, 0x42, 0xcc, 0x61, - 0xe8, 0x92, 0x1c, 0x2a, 0x23, 0xe0, 0xee, 0x7e, 0xb8, 0x0c, 0xf5, 0xfd, 0x70, 0x2e, 0xc3, 0x7f, - 0x05, 0x95, 0x61, 0xcc, 0xbd, 0xa3, 0x35, 0x17, 0xc8, 0xb6, 0xb6, 0x63, 0x88, 0xc0, 0x54, 0xdc, - 0xcd, 0x79, 0xac, 0x2a, 0x95, 0xdf, 0x8b, 0xfd, 0xc6, 0x91, 0x56, 0xaa, 0x07, 0x20, 0xdc, 0xe1, - 0x0d, 0xab, 0x8e, 0xb6, 0x60, 0x58, 0x33, 0x89, 0xe3, 0x85, 0xc1, 0x17, 0xbf, 0x3d, 0x97, 0x9d, - 0x53, 0xe0, 0xe0, 0xcf, 0xbf, 0xfc, 0x5f, 0x38, 0xc0, 0xad, 0xfe, 0x23, 0x05, 0xce, 0xa6, 0x87, - 0x22, 0xea, 0x42, 0xad, 0x68, 0xc0, 0xa8, 0x13, 0x36, 0x13, 0x9b, 0xfe, 0x5d, 0x72, 0x86, 0x07, - 0xe9, 0xe1, 0x0a, 0x55, 0xa4, 0x4a, 0x8e, 0xed, 0xfa, 0x2b, 0x1f, 0x4f, 0xfa, 0x10, 0x58, 0x95, - 0xa4, 0x9e, 0x60, 0x19, 0x3f, 0x4b, 0xc0, 0x42, 0xa9, 0xbb, 0x4d, 0x4d, 0x27, 0xb5, 0x13, 0xce, - 0x49, 0x7e, 0x04, 0x59, 0x0f, 0xd2, 0xfb, 0x7e, 0xbc, 0x09, 0x58, 0x32, 0x68, 0x1e, 0x9c, 0x80, - 0x25, 0xbd, 0xe1, 0x1b, 0x24, 0x33, 0x40, 0x7a, 0xe7, 0x33, 0x1e, 0x8e, 0xff, 0x97, 0xc1, 0xac, - 0xd1, 0x1e, 0x32, 0xb1, 0xf9, 0xce, 0x31, 0x26, 0x36, 0x9f, 0xf8, 0x66, 0x52, 0xf3, 0x94, 0xa4, - 0xe6, 0xb1, 0x44, 0xdb, 0x83, 0x27, 0x94, 0x68, 0xfb, 0x15, 0x18, 0x6c, 0x6a, 0x0e, 0xb1, 0xfc, - 0xbb, 0xd0, 0x4a, 0xaf, 0x59, 0xfc, 0x43, 0x2e, 0x18, 0x7c, 0x92, 0x6b, 0x8c, 0x00, 0x16, 0x84, - 0x52, 0x82, 0x8f, 0x0c, 0x1f, 0x57, 0x74, 0xbf, 0x30, 0xc5, 0xf7, 0xc8, 0x71, 0xa4, 0xf8, 0x56, - 0xff, 0x46, 0x81, 0x8b, 0x9d, 0xd8, 0x12, 0x33, 0x40, 0xe8, 0xb1, 0xcf, 0xb0, 0x17, 0x03, 0x44, - 0x82, 0xdb, 0x06, 0x06, 0x88, 0x38, 0x04, 0x27, 0xe8, 0xa2, 0xf7, 0x02, 0xb2, 0x37, 0xb9, 0x5f, - 0xcc, 0x75, 0x4a, 0x83, 0xbf, 0x46, 0x2d, 0x30, 0x5f, 0xff, 0x20, 0x2f, 0xe4, 0xad, 0x44, 0x0d, - 0x9c, 0xd2, 0x4a, 0xfd, 0xe5, 0x02, 0xc0, 0x2a, 0xf1, 0xee, 0xd8, 0xce, 0x6d, 0x7a, 0xc6, 0x5f, - 0x8c, 0x98, 0xde, 0x87, 0xbf, 0x71, 0xf1, 0x1c, 0x2f, 0x42, 0x7f, 0xd3, 0xae, 0xb9, 0x42, 0x7e, - 0x67, 0x1d, 0x61, 0x4f, 0x1d, 0x58, 0x29, 0x2a, 0xc2, 0x00, 0x73, 0x1a, 0x12, 0x0a, 0x28, 0x33, - 0xdc, 0xb3, 0xcb, 0x09, 0xcc, 0xcb, 0x29, 0x87, 0x14, 0x31, 0x01, 0x5c, 0x71, 0x39, 0x32, 0xc6, - 0x63, 0x70, 0xf3, 0x32, 0x1c, 0x40, 0xd1, 0xe3, 0x00, 0x46, 0x73, 0x49, 0x6b, 0x18, 0xa6, 0x21, - 0x3e, 0xd7, 0x11, 0x66, 0x51, 0x86, 0xca, 0x9a, 0x5f, 0x7a, 0xaf, 0x5d, 0x1c, 0x16, 0xbf, 0xf6, - 0xb0, 0x54, 0x5b, 0xfd, 0x92, 0x02, 0x93, 0xe1, 0xe4, 0x89, 0xad, 0xe2, 0xf7, 0x9c, 0x07, 0xd3, - 0xcd, 0xec, 0x39, 0x0f, 0x7b, 0xdf, 0xb9, 0xe7, 0xdc, 0x00, 0x94, 0xd5, 0xf3, 0x87, 0x60, 0x94, - 0xf0, 0x90, 0x41, 0x95, 0x32, 0xe6, 0x3c, 0x6e, 0x84, 0xab, 0x43, 0x8b, 0x61, 0x31, 0x96, 0xeb, - 0xa8, 0x7f, 0xd7, 0x07, 0x63, 0xab, 0x75, 0xc3, 0xda, 0xf5, 0x63, 0x23, 0x05, 0x17, 0xd7, 0xca, - 0xf1, 0x5c, 0x5c, 0x3f, 0x0f, 0x33, 0xa6, 0x7c, 0x4d, 0xc4, 0x05, 0x27, 0xcd, 0xaa, 0x07, 0x33, - 0xc0, 0xf4, 0x80, 0xe5, 0x8c, 0x3a, 0x38, 0xb3, 0x35, 0xf2, 0x60, 0x50, 0xf7, 0xf3, 0x13, 0xe6, - 0x8e, 0xf7, 0x23, 0xcf, 0xc5, 0x9c, 0x1c, 0xfa, 0x22, 0xe0, 0x79, 0x62, 0x7b, 0x0a, 0x5a, 0xe8, - 0xa3, 0x0a, 0x9c, 0x21, 0xbb, 0x3c, 0xf4, 0xcb, 0xba, 0xa3, 0x6d, 0x6d, 0x19, 0xba, 0x78, 0x31, - 0xc7, 0x77, 0xe2, 0xf2, 0x7e, 0xbb, 0x78, 0x66, 0x31, 0xad, 0xc2, 0xbd, 0x76, 0xf1, 0x5a, 0x6a, - 0x24, 0x1e, 0xb6, 0x9a, 0xa9, 0x4d, 0x70, 0x3a, 0xa9, 0xd9, 0xc7, 0x60, 0xf4, 0x10, 0xef, 0xac, - 0x23, 0xf1, 0x76, 0x7e, 0xa5, 0x00, 0x63, 0x74, 0xbb, 0x2d, 0xdb, 0xba, 0x66, 0x96, 0x57, 0xab, - 0xe8, 0x81, 0x78, 0x98, 0xc1, 0xe0, 0x8a, 0x2e, 0x11, 0x6a, 0x70, 0x19, 0xa6, 0xb7, 0x6c, 0x47, - 0x27, 0xeb, 0xa5, 0xb5, 0x75, 0x5b, 0x38, 0x6f, 0x95, 0x57, 0xab, 0x42, 0x2f, 0x62, 0x57, 0x06, - 0x4b, 0x29, 0x70, 0x9c, 0xda, 0x0a, 0xdd, 0x82, 0x33, 0x61, 0xf9, 0x46, 0x93, 0x3b, 0xfc, 0x53, - 0x74, 0x7d, 0xe1, 0x83, 0x85, 0xa5, 0xb4, 0x0a, 0x38, 0xbd, 0x1d, 0xd2, 0xe0, 0x82, 0xb8, 0x9b, - 0x58, 0xb2, 0x9d, 0x3b, 0x9a, 0x53, 0x8b, 0xa2, 0xed, 0x0f, 0x73, 0x7d, 0x97, 0xb3, 0xab, 0xe1, - 0x4e, 0x38, 0xd4, 0xbf, 0x55, 0x20, 0x1a, 0xc4, 0x11, 0x9d, 0x87, 0x3e, 0x47, 0xa4, 0xbf, 0x13, - 0xc1, 0x0c, 0xa9, 0x8a, 0x40, 0xcb, 0xd0, 0x1c, 0x80, 0x13, 0x46, 0x92, 0x2c, 0x84, 0x69, 0x21, - 0xa4, 0x18, 0x90, 0x52, 0x0d, 0x8a, 0xca, 0xd3, 0xea, 0x82, 0xe1, 0x31, 0x54, 0xeb, 0x5a, 0x1d, - 0xd3, 0x32, 0x96, 0xff, 0xc3, 0xa8, 0x13, 0xd7, 0x37, 0xe7, 0xf2, 0xfc, 0x1f, 0xac, 0x04, 0x0b, - 0x08, 0xd2, 0x60, 0xbc, 0xd9, 0x32, 0x45, 0x3c, 0x22, 0xaa, 0xfa, 0x70, 0xb3, 0xdb, 0xd5, 0xb4, - 0xe4, 0x76, 0x6c, 0xf5, 0x53, 0x33, 0xdc, 0xad, 0xc9, 0x28, 0x70, 0x14, 0xa3, 0xfa, 0xe3, 0x83, - 0x20, 0x85, 0xa7, 0x39, 0x84, 0x14, 0xfa, 0x53, 0x0a, 0x4c, 0xeb, 0xa6, 0x41, 0x2c, 0x2f, 0x16, - 0xe9, 0x81, 0x1f, 0x1f, 0x1b, 0xb9, 0xe2, 0xe6, 0x34, 0x89, 0x55, 0x29, 0x8b, 0xe7, 0x21, 0xa5, - 0x14, 0xe4, 0xe2, 0x09, 0x4d, 0x0a, 0x04, 0xa7, 0x76, 0x86, 0x8d, 0x87, 0x95, 0x57, 0xca, 0x72, - 0xf4, 0xc9, 0x92, 0x28, 0xc3, 0x01, 0x94, 0x72, 0xde, 0xba, 0x63, 0xb7, 0x9a, 0x6e, 0x89, 0xbd, - 0x02, 0xe5, 0x8b, 0xc2, 0x38, 0xef, 0xf5, 0xb0, 0x18, 0xcb, 0x75, 0xd0, 0x23, 0x30, 0xc6, 0x7f, - 0xae, 0x39, 0x64, 0xcb, 0xd8, 0x15, 0x87, 0x12, 0x33, 0x3e, 0x5e, 0x97, 0xca, 0x71, 0xa4, 0x16, - 0x8b, 0x7f, 0xe6, 0xba, 0x2d, 0xe2, 0x6c, 0xe0, 0x65, 0x91, 0x23, 0x99, 0xc7, 0x3f, 0xf3, 0x0b, - 0x71, 0x08, 0x47, 0x3f, 0xac, 0xc0, 0x84, 0x43, 0x5e, 0x69, 0x19, 0x0e, 0x15, 0x61, 0x34, 0xa3, - 0xe1, 0x8a, 0x18, 0x41, 0xb8, 0xb7, 0xb8, 0x44, 0x73, 0x38, 0x82, 0x94, 0x33, 0xc8, 0xc0, 0x15, - 0x22, 0x0a, 0xc4, 0xb1, 0x1e, 0xd0, 0xa9, 0x72, 0x8d, 0xba, 0x65, 0x58, 0xf5, 0x79, 0xb3, 0xee, - 0x8a, 0x44, 0x50, 0xdc, 0x66, 0x17, 0x16, 0x63, 0xb9, 0x0e, 0x7a, 0x14, 0xc6, 0x5b, 0x2e, 0x65, - 0x7b, 0x0d, 0xc2, 0xe7, 0x77, 0x24, 0xf4, 0x15, 0xd9, 0x90, 0x01, 0x38, 0x5a, 0x0f, 0x3d, 0x0e, - 0x13, 0x7e, 0x81, 0x98, 0x65, 0xe0, 0xd9, 0x48, 0xd8, 0xdd, 0x5e, 0x04, 0x82, 0x63, 0x35, 0x67, - 0xe7, 0xe1, 0x74, 0xca, 0x30, 0x0f, 0xc5, 0x5b, 0xff, 0xb7, 0x02, 0x67, 0xb8, 0xd4, 0xe5, 0x67, - 0x57, 0xf6, 0x73, 0x43, 0xa4, 0xa7, 0x59, 0x50, 0x8e, 0x35, 0xcd, 0xc2, 0x37, 0x20, 0x9d, 0x84, - 0xfa, 0x33, 0x05, 0x78, 0xf3, 0x81, 0xdf, 0x25, 0xfa, 0x07, 0x0a, 0x8c, 0x92, 0x5d, 0xcf, 0xd1, - 0x82, 0xa7, 0xf2, 0x74, 0x93, 0x6e, 0x1d, 0x0b, 0x13, 0x98, 0x5b, 0x0c, 0x09, 0xf1, 0x8d, 0x1b, - 0xa8, 0x52, 0x12, 0x04, 0xcb, 0xfd, 0xa1, 0xdc, 0x96, 0x67, 0xeb, 0x91, 0x9d, 0xca, 0x04, 0x17, - 0x14, 0x90, 0xd9, 0x27, 0x61, 0x32, 0x8e, 0xf9, 0x50, 0x7b, 0xe5, 0x8b, 0x0a, 0xa4, 0x86, 0xb3, - 0x44, 0x25, 0x98, 0xd2, 0x5a, 0x9e, 0x1d, 0xb9, 0x17, 0x13, 0xa2, 0x24, 0x73, 0x9a, 0x9e, 0x8f, - 0x03, 0x71, 0xb2, 0x3e, 0x37, 0x60, 0x5a, 0x2d, 0xcd, 0x8c, 0xa2, 0xe1, 0x02, 0x97, 0x30, 0x60, - 0x26, 0xc0, 0x38, 0xad, 0x8d, 0xfa, 0x4f, 0x14, 0x38, 0x13, 0xe9, 0xa8, 0x2b, 0x0c, 0xb1, 0x5d, - 0xd8, 0xe2, 0xd2, 0xb7, 0x7d, 0xe1, 0x38, 0xb7, 0xbd, 0xfa, 0x4b, 0x05, 0x18, 0x5a, 0x73, 0x6c, - 0x7a, 0xb4, 0x9d, 0x80, 0x05, 0x4e, 0x8b, 0x58, 0xe0, 0x72, 0xd9, 0x17, 0x44, 0x67, 0x33, 0x4d, - 0x6e, 0x46, 0xcc, 0xe4, 0x36, 0xdf, 0x0b, 0x91, 0xce, 0x36, 0xb6, 0xdf, 0x55, 0x60, 0x54, 0xd4, - 0x3c, 0x01, 0xa3, 0xda, 0x77, 0x45, 0x8d, 0x6a, 0xef, 0xe9, 0x61, 0x5c, 0x19, 0x56, 0xb4, 0xcf, - 0x29, 0x30, 0x2e, 0x6a, 0xac, 0x90, 0xc6, 0x26, 0x71, 0xd0, 0x12, 0x0c, 0xb9, 0x2d, 0xb6, 0x90, - 0x62, 0x40, 0x17, 0x64, 0xf1, 0xc8, 0xd9, 0xd4, 0x74, 0xda, 0xfd, 0x2a, 0xaf, 0x22, 0x65, 0x12, - 0xe6, 0x05, 0xd8, 0x6f, 0x4c, 0xf7, 0xbe, 0x63, 0x9b, 0x89, 0xb0, 0xef, 0xd8, 0x36, 0x09, 0x66, - 0x10, 0xaa, 0xe7, 0xd1, 0xbf, 0xbe, 0x0e, 0xc7, 0xf4, 0x3c, 0x0a, 0x76, 0x31, 0x2f, 0x57, 0xbf, - 0x34, 0x10, 0x4c, 0x36, 0x53, 0xea, 0x6f, 0xc0, 0x88, 0xee, 0x10, 0xcd, 0x23, 0xb5, 0x85, 0xbd, - 0x6e, 0x3a, 0xc7, 0xe4, 0x80, 0x92, 0xdf, 0x02, 0x87, 0x8d, 0xe9, 0x91, 0x2b, 0x3b, 0x48, 0x16, - 0x42, 0xe9, 0x24, 0xd3, 0x39, 0xf2, 0xdb, 0x61, 0xc0, 0xbe, 0x63, 0x05, 0x8f, 0x2b, 0x3a, 0x12, - 0x66, 0x43, 0xb9, 0x45, 0x6b, 0x63, 0xde, 0x48, 0x4e, 0x7b, 0xd0, 0xdf, 0x21, 0xed, 0x81, 0x09, - 0x43, 0x0d, 0xb6, 0x0c, 0x3d, 0x25, 0x96, 0x8d, 0x2c, 0x68, 0xb8, 0x44, 0xfc, 0xb7, 0x8b, 0x7d, - 0x12, 0x54, 0x74, 0xb2, 0x7c, 0x8b, 0x8e, 0x2c, 0x3a, 0x05, 0x66, 0x1e, 0x1c, 0xc2, 0xd1, 0x5e, - 0x34, 0x9f, 0xc6, 0x50, 0x7e, 0x3b, 0xa9, 0xe8, 0x9e, 0x94, 0x42, 0x83, 0x4f, 0x7d, 0x56, 0x4e, - 0x0d, 0xf4, 0x45, 0x05, 0xce, 0xd5, 0xd2, 0x13, 0x96, 0x89, 0xab, 0xe6, 0x5c, 0x0f, 0x9b, 0x33, - 0x72, 0xa0, 0x2d, 0x14, 0xc5, 0x84, 0x65, 0x25, 0x49, 0xc3, 0x59, 0x9d, 0x51, 0xbf, 0xbf, 0x3f, - 0xf8, 0x9a, 0x84, 0xa5, 0x23, 0xdd, 0x0e, 0xa5, 0xe4, 0xb1, 0x43, 0xa1, 0x77, 0xf8, 0xe9, 0xb3, - 0xf8, 0x76, 0xbd, 0x14, 0x4f, 0x9f, 0x35, 0x26, 0x48, 0x47, 0x52, 0x66, 0xb5, 0xe0, 0xb4, 0xeb, - 0x69, 0x26, 0xa9, 0x1a, 0xe2, 0x72, 0xcd, 0xf5, 0xb4, 0x46, 0x33, 0x47, 0xfe, 0x2a, 0x1e, 0xe8, - 0x20, 0x89, 0x0a, 0xa7, 0xe1, 0x47, 0xdf, 0xcb, 0x82, 0xd7, 0x69, 0x26, 0xbb, 0x7c, 0xe4, 0x39, - 0x3c, 0x43, 0xe2, 0x87, 0x77, 0x17, 0x17, 0xa1, 0xe9, 0xd2, 0xf1, 0xe1, 0x4c, 0x4a, 0xe8, 0x55, - 0x38, 0x43, 0x0f, 0xc0, 0x79, 0xdd, 0x33, 0x76, 0x0c, 0x6f, 0x2f, 0xec, 0xc2, 0xe1, 0x93, 0x56, - 0x31, 0x6d, 0x7b, 0x39, 0x0d, 0x19, 0x4e, 0xa7, 0xa1, 0xfe, 0xb5, 0x02, 0x28, 0xb9, 0xd7, 0x91, - 0x09, 0xc3, 0x35, 0x3f, 0xf2, 0x80, 0x72, 0x24, 0x59, 0x69, 0x82, 0x23, 0x24, 0x08, 0x58, 0x10, - 0x50, 0x40, 0x36, 0x8c, 0xdc, 0xd9, 0x36, 0x3c, 0x62, 0x1a, 0xae, 0x77, 0x44, 0x49, 0x70, 0x82, - 0x98, 0xea, 0xcf, 0xf9, 0x88, 0x71, 0x48, 0x43, 0xfd, 0x54, 0x3f, 0x0c, 0x07, 0xc9, 0x28, 0x0f, - 0x76, 0x53, 0x6e, 0x01, 0xd2, 0xa5, 0x80, 0x7a, 0xbd, 0xd8, 0x4c, 0x99, 0x0c, 0x54, 0x4a, 0x20, - 0xc3, 0x29, 0x04, 0xd0, 0xab, 0x30, 0x6d, 0x58, 0x5b, 0x8e, 0x16, 0x84, 0x0b, 0x2c, 0xf9, 0x86, - 0xb2, 0x1c, 0x84, 0x99, 0x16, 0x5d, 0x49, 0x41, 0x87, 0x53, 0x89, 0x20, 0x02, 0x43, 0x3c, 0x1f, - 0xb0, 0x7f, 0xeb, 0x92, 0xcb, 0x56, 0xcf, 0x65, 0xcd, 0x90, 0xbd, 0xfb, 0xb2, 0xa7, 0x8f, 0x9b, - 0x07, 0x77, 0xe5, 0xff, 0xfb, 0x17, 0x52, 0x62, 0xdf, 0x97, 0xf2, 0xd3, 0x0b, 0xef, 0xb6, 0x78, - 0x70, 0xd7, 0x68, 0x21, 0x8e, 0x13, 0x54, 0x7f, 0x5b, 0x81, 0x01, 0x1e, 0x43, 0xeb, 0xf8, 0x45, - 0xcd, 0xf7, 0x47, 0x44, 0xcd, 0x27, 0xf2, 0x0c, 0x92, 0x75, 0x35, 0x33, 0xa3, 0xfd, 0x6f, 0x29, - 0x30, 0xc2, 0x6a, 0x9c, 0x80, 0xec, 0xf7, 0x52, 0x54, 0xf6, 0x7b, 0x2c, 0xf7, 0x68, 0x32, 0x24, - 0xbf, 0xdf, 0xee, 0x13, 0x63, 0x61, 0xa2, 0x55, 0x05, 0x4e, 0x8b, 0x87, 0xa5, 0xcb, 0xc6, 0x16, - 0xa1, 0x5b, 0xbc, 0xac, 0xed, 0xb9, 0xe2, 0xa9, 0x03, 0x0f, 0xda, 0x92, 0x04, 0xe3, 0xb4, 0x36, - 0xe8, 0x57, 0x14, 0x2a, 0xc4, 0x78, 0x8e, 0xa1, 0xf7, 0x74, 0x19, 0x1c, 0xf4, 0x6d, 0x6e, 0x85, - 0x23, 0xe3, 0xba, 0xe9, 0x46, 0x28, 0xcd, 0xb0, 0xd2, 0x7b, 0xed, 0x62, 0x31, 0xc5, 0x66, 0x1c, - 0x66, 0x75, 0x76, 0xbd, 0x8f, 0xfe, 0x79, 0xc7, 0x2a, 0x4c, 0x1b, 0xf3, 0x7b, 0x8c, 0x6e, 0xc0, - 0x80, 0xab, 0xdb, 0x4d, 0xff, 0x69, 0xf2, 0x7d, 0x69, 0xb6, 0xc1, 0xb8, 0x59, 0x30, 0x98, 0xe0, - 0x2a, 0x6d, 0x89, 0x39, 0x82, 0xd9, 0x97, 0x61, 0x4c, 0xee, 0x79, 0x8a, 0xee, 0x5b, 0x8e, 0x7a, - 0x9c, 0x1d, 0xd2, 0xb9, 0x4a, 0xd6, 0x95, 0xff, 0xb0, 0x0f, 0x06, 0x31, 0xa9, 0x8b, 0x94, 0x63, - 0x07, 0xe8, 0x9c, 0x86, 0x9f, 0x3e, 0xb7, 0x90, 0xff, 0x1d, 0x9b, 0x9c, 0x54, 0xe6, 0x05, 0xdb, - 0x92, 0xe6, 0x40, 0xce, 0xa0, 0x8b, 0xac, 0x20, 0x11, 0x53, 0x5f, 0x7e, 0x87, 0x54, 0x3e, 0xb0, - 0x6e, 0x52, 0x2f, 0xa1, 0x1f, 0x52, 0x00, 0x69, 0xba, 0x4e, 0x5c, 0x17, 0x13, 0x97, 0xce, 0x3d, - 0x17, 0x56, 0x39, 0x97, 0xcd, 0x17, 0x55, 0x3a, 0x8e, 0x2d, 0x14, 0xdb, 0x12, 0x20, 0x17, 0xa7, - 0x10, 0xef, 0x25, 0x1d, 0xd4, 0xef, 0x29, 0x30, 0x16, 0xc9, 0xb6, 0xd5, 0x08, 0x6d, 0xe9, 0xf9, - 0x5d, 0x76, 0xfc, 0xd7, 0x53, 0x17, 0x3a, 0x54, 0xe2, 0xf6, 0xf9, 0x5b, 0x41, 0xba, 0x88, 0xa3, - 0x49, 0xcc, 0xa5, 0x7e, 0x46, 0x81, 0xb3, 0xfe, 0x80, 0xa2, 0x71, 0xc1, 0xd1, 0x55, 0x18, 0xd6, - 0x9a, 0x06, 0x33, 0xf4, 0xca, 0xa6, 0xf2, 0xf9, 0xb5, 0x0a, 0x2b, 0xc3, 0x01, 0x34, 0x92, 0xa3, - 0xb8, 0x70, 0x60, 0x8e, 0xe2, 0xb7, 0x48, 0x59, 0x97, 0x07, 0x42, 0xd9, 0x25, 0x20, 0xcc, 0x9d, - 0x21, 0xd5, 0x77, 0xc1, 0x48, 0xb5, 0x7a, 0x83, 0x2f, 0xe9, 0x21, 0x6e, 0x7c, 0xd4, 0x4f, 0xf4, - 0xc1, 0xb8, 0x48, 0x70, 0x60, 0x30, 0x13, 0xd0, 0x09, 0x9c, 0x73, 0xeb, 0x30, 0xe2, 0x06, 0x77, - 0x18, 0x85, 0x6c, 0x3e, 0x15, 0x5c, 0x4b, 0xc4, 0xb3, 0xd4, 0x85, 0xd7, 0x17, 0x21, 0x22, 0x74, - 0x13, 0x06, 0x5f, 0xa1, 0x3c, 0xd7, 0xff, 0x56, 0xbb, 0x62, 0x7d, 0xc1, 0x87, 0xc8, 0xd8, 0xb5, - 0x8b, 0x05, 0x0a, 0xe4, 0xb2, 0xe7, 0x7d, 0x4c, 0x08, 0xec, 0x25, 0xd4, 0x65, 0x64, 0x66, 0x83, - 0x9c, 0xeb, 0x63, 0xe2, 0x95, 0x20, 0xfb, 0x85, 0x03, 0x42, 0x2c, 0xc5, 0x66, 0xa4, 0xc5, 0x1b, - 0x24, 0xc5, 0x66, 0xa4, 0xcf, 0x19, 0xc7, 0xf5, 0x63, 0x70, 0x26, 0x75, 0x32, 0x0e, 0x16, 0xb1, - 0xd5, 0x7f, 0x5a, 0x80, 0xfe, 0x2a, 0x21, 0xb5, 0x13, 0xd8, 0x99, 0x2f, 0x45, 0x24, 0xb0, 0x6f, - 0xcf, 0x9d, 0xe4, 0x33, 0xcb, 0xd2, 0xb7, 0x15, 0xb3, 0xf4, 0x3d, 0x99, 0x9b, 0x42, 0x67, 0x33, - 0xdf, 0x5f, 0x16, 0x60, 0x88, 0x56, 0x2b, 0xaf, 0x56, 0x51, 0x43, 0xda, 0xca, 0x85, 0xfc, 0xe2, - 0xb3, 0x40, 0x77, 0xd0, 0x26, 0xa6, 0x5f, 0x8e, 0x21, 0x12, 0xa7, 0x88, 0x41, 0x56, 0x8e, 0x80, - 0x9c, 0x88, 0xdd, 0xc0, 0x88, 0xfa, 0x79, 0x59, 0x70, 0x40, 0x08, 0xdd, 0x91, 0x74, 0x52, 0x7e, - 0x58, 0x1e, 0x21, 0xd1, 0x0e, 0xea, 0xa9, 0xfa, 0x09, 0x05, 0x4e, 0xc5, 0x5a, 0x75, 0xa1, 0x34, - 0x1e, 0x0b, 0x03, 0x54, 0xbf, 0xae, 0xd0, 0x6f, 0x2c, 0x65, 0x04, 0x5d, 0xf4, 0x28, 0x4c, 0xa0, - 0x55, 0xe8, 0x98, 0x40, 0xeb, 0x22, 0xf4, 0x53, 0xc9, 0x48, 0xf6, 0xd9, 0xa1, 0x02, 0x13, 0x66, - 0xa5, 0x48, 0x87, 0x09, 0x5d, 0x4a, 0x81, 0x4e, 0xb6, 0x04, 0xef, 0xec, 0x8a, 0x15, 0x07, 0xd7, - 0x8d, 0xa5, 0x08, 0x0a, 0x1c, 0x43, 0xa9, 0xfe, 0xa6, 0x02, 0xc3, 0x74, 0x98, 0x27, 0xc0, 0x1c, - 0xbf, 0x33, 0xca, 0x1c, 0xdf, 0x9d, 0x77, 0x4f, 0x65, 0xf0, 0xc4, 0xbf, 0x2a, 0x00, 0xcb, 0x00, - 0x2c, 0xfc, 0x8b, 0x24, 0xcf, 0x21, 0x25, 0xc3, 0xe7, 0xe9, 0x8a, 0x70, 0x3c, 0x8a, 0x19, 0xa5, - 0x25, 0xe7, 0xa3, 0xb7, 0x45, 0x7c, 0x8b, 0x22, 0xc2, 0x45, 0x8a, 0x7f, 0xd1, 0x5d, 0x18, 0x77, - 0xb7, 0x6d, 0xdb, 0x2b, 0x87, 0x1f, 0x4f, 0xee, 0x0b, 0x08, 0xf6, 0xd8, 0xda, 0x1f, 0x0a, 0xbf, - 0xca, 0xad, 0xca, 0xb8, 0x71, 0x94, 0x14, 0x9a, 0x03, 0xd8, 0x34, 0x6d, 0xfd, 0x36, 0x77, 0x6d, - 0xe2, 0x8f, 0x6b, 0x99, 0xf3, 0xc4, 0x42, 0x50, 0x8a, 0xa5, 0x1a, 0x3d, 0x79, 0x71, 0x7d, 0x5d, - 0xe1, 0x33, 0x7d, 0x88, 0x6f, 0xf4, 0x04, 0x1d, 0xe1, 0xde, 0x0a, 0x83, 0x0e, 0xd3, 0x11, 0xc4, - 0x8a, 0x05, 0x1f, 0x1f, 0xd7, 0x1c, 0xb0, 0x80, 0xd2, 0xed, 0xc1, 0x15, 0x9f, 0xfe, 0xf0, 0xc2, - 0x41, 0x56, 0x57, 0xd4, 0x5f, 0x12, 0xc3, 0x0c, 0x92, 0x48, 0x37, 0x61, 0x9c, 0x69, 0x16, 0xb1, - 0xec, 0xd5, 0xef, 0xe8, 0xf2, 0x1b, 0x91, 0x9b, 0x86, 0x6e, 0xb9, 0x91, 0x62, 0x1c, 0x25, 0x80, - 0x1e, 0x85, 0x71, 0x7f, 0x74, 0xdc, 0x3b, 0xb6, 0x10, 0xbe, 0x70, 0x5c, 0x93, 0x01, 0x38, 0x5a, - 0x4f, 0xfd, 0x6c, 0x01, 0x2e, 0xf1, 0xbe, 0x33, 0xcb, 0x4b, 0x99, 0x34, 0x89, 0x55, 0x23, 0x96, - 0xbe, 0xc7, 0xe4, 0xec, 0x9a, 0x5d, 0x47, 0xaf, 0xc2, 0xe0, 0x1d, 0x42, 0x6a, 0xc1, 0x15, 0xc6, - 0x73, 0xf9, 0x73, 0x70, 0x67, 0x90, 0x78, 0x8e, 0xa1, 0xe7, 0xb7, 0xc1, 0xfc, 0x7f, 0x2c, 0x48, - 0x52, 0xe2, 0x4d, 0xc7, 0xde, 0x0c, 0xc4, 0xc1, 0xa3, 0x27, 0xbe, 0xc6, 0xd0, 0x73, 0xe2, 0xfc, - 0x7f, 0x2c, 0x48, 0xaa, 0x6b, 0x70, 0x5f, 0x17, 0x4d, 0x0f, 0x23, 0xf6, 0x1f, 0x84, 0x91, 0x8f, - 0xfe, 0x30, 0x18, 0xff, 0x58, 0x81, 0xfb, 0x25, 0x94, 0x8b, 0xbb, 0x54, 0x13, 0x29, 0x69, 0x4d, - 0x4d, 0xa7, 0xba, 0x3e, 0x8b, 0xf1, 0x76, 0xa8, 0xac, 0xb7, 0x9f, 0x50, 0x60, 0x88, 0x7b, 0xe4, - 0xf9, 0xec, 0xf7, 0xa5, 0x1e, 0xa7, 0x3c, 0xb3, 0x4b, 0x7e, 0x36, 0x30, 0x7f, 0x6c, 0xfc, 0xb7, - 0x8b, 0x7d, 0xfa, 0xea, 0xbf, 0x1e, 0x80, 0x6f, 0xed, 0x1e, 0x11, 0xfa, 0xba, 0x22, 0x67, 0xeb, - 0xe6, 0x36, 0xf2, 0xc6, 0xf1, 0x76, 0x3e, 0xb0, 0x06, 0x09, 0x03, 0xc3, 0x73, 0x89, 0x84, 0xde, - 0x47, 0x64, 0x68, 0x0a, 0x07, 0x86, 0xfe, 0xb1, 0x02, 0x63, 0xf4, 0x58, 0x0a, 0x98, 0x0b, 0x5f, - 0xa6, 0xe6, 0x31, 0x8f, 0x74, 0x55, 0x22, 0x19, 0x8b, 0x0b, 0x25, 0x83, 0x70, 0xa4, 0x6f, 0x68, - 0x23, 0x7a, 0xfd, 0xc7, 0x55, 0xc4, 0xcb, 0x69, 0x72, 0xc9, 0x61, 0xd2, 0xe5, 0xcf, 0x9a, 0x30, - 0x11, 0x9d, 0xf9, 0xe3, 0x34, 0x93, 0xcd, 0x3e, 0x05, 0x53, 0x89, 0xd1, 0x1f, 0xca, 0x20, 0xf3, - 0x73, 0x83, 0x50, 0x94, 0xa6, 0x3a, 0x2d, 0xbc, 0x0b, 0xfa, 0xbc, 0x02, 0xa3, 0x9a, 0x65, 0x09, - 0xc7, 0x26, 0x7f, 0xff, 0xd6, 0x7a, 0x5c, 0xd5, 0x34, 0x52, 0x73, 0xf3, 0x21, 0x99, 0x98, 0xe7, - 0x8e, 0x04, 0xc1, 0x72, 0x6f, 0x3a, 0x78, 0xe7, 0x16, 0x4e, 0xcc, 0x3b, 0x17, 0x7d, 0xd0, 0x3f, - 0x88, 0xfb, 0xf2, 0x07, 0x7a, 0x38, 0x60, 0x6e, 0xd8, 0xb9, 0x9e, 0x61, 0x95, 0xfc, 0xb4, 0xc2, - 0x0e, 0xd9, 0x30, 0x0a, 0x8f, 0x38, 0x93, 0x72, 0x39, 0x59, 0x1e, 0x18, 0xe2, 0x27, 0x38, 0xbb, - 0xc3, 0x22, 0x1c, 0x25, 0x8f, 0x7e, 0x54, 0x81, 0xb1, 0xbb, 0xb6, 0xa5, 0x99, 0xc2, 0xcf, 0x5a, - 0x5c, 0xd3, 0xbc, 0xff, 0x78, 0xe6, 0x25, 0x20, 0xc3, 0x7d, 0x2b, 0xe5, 0x12, 0x1c, 0xe9, 0xc6, - 0xec, 0x93, 0x30, 0x19, 0xdf, 0x62, 0x87, 0xfa, 0x5c, 0xd6, 0xe1, 0xc1, 0x43, 0x74, 0xa7, 0xcb, - 0x93, 0x4d, 0xfd, 0xb5, 0xfe, 0xc8, 0x49, 0x99, 0xb9, 0xfa, 0x5d, 0x98, 0xc2, 0x5f, 0x8b, 0x7d, - 0xab, 0x9c, 0x03, 0x1b, 0xc7, 0xb5, 0x1f, 0x8f, 0xf6, 0x83, 0xed, 0x3b, 0xb9, 0x0f, 0xf6, 0xf5, - 0xf6, 0xc5, 0xf4, 0xbc, 0x33, 0x17, 0xb8, 0xaa, 0x2f, 0x16, 0x2c, 0xcc, 0x65, 0xc4, 0xe2, 0x58, - 0x1a, 0xae, 0xe1, 0x47, 0x63, 0x96, 0x24, 0xb6, 0x67, 0x79, 0x31, 0xf6, 0xe1, 0xea, 0x72, 0xe4, - 0x2c, 0x58, 0xb7, 0x9b, 0xb6, 0x69, 0xd7, 0xf7, 0xe6, 0xef, 0x68, 0x0e, 0xc1, 0x76, 0xcb, 0x13, - 0xd8, 0xba, 0x95, 0xff, 0x3e, 0x32, 0x00, 0x57, 0x24, 0x74, 0xa9, 0x41, 0x2b, 0x0f, 0xf3, 0x14, - 0xe1, 0xb5, 0x78, 0x6c, 0xc9, 0x42, 0x7e, 0x1f, 0xd2, 0x83, 0xfa, 0x95, 0x2f, 0xd6, 0x24, 0x6a, - 0x2b, 0x00, 0x0d, 0x6d, 0x57, 0x04, 0x3a, 0x13, 0x87, 0x41, 0xed, 0x58, 0x3a, 0xb8, 0x12, 0x90, - 0xe1, 0xdd, 0x7b, 0xde, 0x37, 0x63, 0x86, 0x80, 0x23, 0x12, 0xf0, 0xa4, 0x11, 0xf5, 0x1c, 0x4c, - 0x73, 0xb6, 0x01, 0xa7, 0x62, 0x3d, 0x3f, 0xd6, 0x6b, 0xc4, 0xdf, 0x1d, 0xf2, 0xf5, 0x5f, 0x11, - 0x81, 0xed, 0x17, 0x15, 0x38, 0x4f, 0xb2, 0xe4, 0x49, 0xa1, 0x0c, 0x3f, 0x7f, 0x5c, 0xf2, 0xaa, - 0xc8, 0x47, 0x95, 0x05, 0xc6, 0xd9, 0x3d, 0x43, 0x7b, 0x00, 0x6e, 0xf0, 0x4d, 0xf7, 0x12, 0xd6, - 0x21, 0x95, 0x49, 0x70, 0x33, 0x4b, 0xf8, 0x1b, 0x4b, 0xc4, 0xd0, 0x17, 0x14, 0x98, 0x36, 0x53, - 0x38, 0x9c, 0xe0, 0x98, 0xd5, 0x63, 0x38, 0x5b, 0xb8, 0x03, 0x4a, 0x1a, 0x04, 0xa7, 0x76, 0x05, - 0xfd, 0x74, 0x66, 0x04, 0x5e, 0x2e, 0x78, 0xac, 0x1f, 0xc7, 0x37, 0x98, 0x23, 0x18, 0xef, 0x67, - 0x15, 0x40, 0xb5, 0x84, 0x6e, 0x2d, 0x7c, 0x0f, 0x9f, 0x39, 0x72, 0x0b, 0x02, 0xf7, 0x20, 0x4a, - 0x96, 0xe3, 0x94, 0x4e, 0xb0, 0x75, 0xf6, 0x52, 0x78, 0xbe, 0x78, 0xe1, 0xdb, 0xeb, 0x3a, 0xa7, - 0x1d, 0x27, 0x7c, 0x9d, 0xd3, 0x20, 0x38, 0xb5, 0x2b, 0xea, 0x6f, 0x8c, 0x70, 0x53, 0x2f, 0x73, - 0xf1, 0x78, 0x09, 0x06, 0x37, 0x35, 0xfd, 0xb6, 0xb8, 0x5e, 0xcd, 0xe9, 0x74, 0xb4, 0xc0, 0x30, - 0x70, 0x23, 0x0b, 0xff, 0x1f, 0x0b, 0xac, 0xe8, 0x05, 0xe8, 0xab, 0x59, 0xfe, 0x03, 0xfa, 0xf7, - 0xf4, 0x70, 0x7d, 0x10, 0x86, 0xf1, 0x28, 0xaf, 0x56, 0x31, 0x45, 0x8a, 0x2c, 0x18, 0xb6, 0x84, - 0x65, 0x54, 0x18, 0xaf, 0x9e, 0xce, 0x4b, 0x20, 0xb0, 0xb0, 0x06, 0x76, 0x5d, 0xbf, 0x04, 0x07, - 0x34, 0x28, 0xbd, 0xd8, 0xf5, 0x65, 0x6e, 0x7a, 0xc1, 0x85, 0x8f, 0x1c, 0xdf, 0x34, 0x7e, 0xe9, - 0x43, 0x60, 0xd0, 0xd3, 0x0c, 0xcb, 0xf3, 0x1f, 0xc3, 0x3f, 0x91, 0x97, 0xda, 0x3a, 0xc5, 0x12, - 0x1a, 0x40, 0xd9, 0x4f, 0x17, 0x0b, 0xe4, 0x68, 0x13, 0x06, 0xf9, 0x83, 0x78, 0xf1, 0x09, 0xe5, - 0xbe, 0x3e, 0xe3, 0x6f, 0xec, 0xf9, 0x36, 0xe0, 0xff, 0x63, 0x81, 0x19, 0xbd, 0x0c, 0xc3, 0xae, - 0xef, 0x6d, 0x36, 0xdc, 0xdb, 0xd4, 0x05, 0xae, 0x66, 0xe2, 0x79, 0xaf, 0xf0, 0x31, 0x0b, 0xf0, - 0xa3, 0x4d, 0x18, 0x32, 0x84, 0xc6, 0x34, 0x92, 0x7f, 0xdb, 0xf9, 0xda, 0x10, 0xd3, 0x36, 0x7c, - 0x45, 0xc8, 0x47, 0x9c, 0xe5, 0x52, 0x02, 0xdf, 0x40, 0x97, 0x12, 0xf4, 0x0a, 0x00, 0xf1, 0x2d, - 0xe4, 0xee, 0xcc, 0x68, 0xfe, 0x2d, 0x23, 0xd9, 0xd9, 0x7d, 0x81, 0x28, 0x28, 0x72, 0xb1, 0x44, - 0x04, 0xbd, 0x2a, 0x9b, 0xe4, 0xc6, 0x7a, 0x8b, 0xf1, 0x91, 0x8c, 0x5a, 0x13, 0xde, 0xcc, 0x05, - 0xd6, 0x20, 0xc9, 0x52, 0xa6, 0xfe, 0x2e, 0x00, 0x84, 0xb7, 0xb6, 0x68, 0x0b, 0x86, 0x7d, 0x2a, - 0xbd, 0x44, 0xd9, 0xb9, 0x2e, 0xc0, 0x7c, 0x7b, 0xf9, 0xbf, 0x70, 0x80, 0x1b, 0x95, 0xd2, 0xa2, - 0x25, 0x85, 0x09, 0x64, 0xbb, 0x8b, 0x94, 0xf4, 0x0a, 0x80, 0x1e, 0xc6, 0x40, 0xed, 0xcb, 0xbf, - 0x56, 0x41, 0x7c, 0xd4, 0x70, 0xad, 0xa4, 0x10, 0xaa, 0x12, 0x91, 0x0c, 0xa7, 0xf3, 0xfe, 0x5c, - 0x4e, 0xe7, 0x4f, 0xc0, 0x29, 0xe1, 0xe4, 0x57, 0x61, 0x97, 0x88, 0xde, 0x9e, 0x78, 0x97, 0xc9, - 0xdc, 0x3f, 0x4b, 0x51, 0x10, 0x8e, 0xd7, 0x45, 0xff, 0x4a, 0x81, 0x61, 0x5d, 0x08, 0x68, 0x82, - 0xb7, 0x2d, 0xf7, 0x76, 0x67, 0x3f, 0xe7, 0xcb, 0x7b, 0x5c, 0x90, 0x7f, 0xd6, 0xe7, 0xaa, 0x7e, - 0xf1, 0x11, 0x89, 0xf1, 0x41, 0xaf, 0xd1, 0xef, 0x28, 0x30, 0xaa, 0x99, 0xa6, 0xad, 0x6b, 0x1e, - 0x8b, 0x9e, 0xc7, 0x1f, 0x8c, 0xde, 0xea, 0x71, 0x14, 0xf3, 0x21, 0x46, 0x3e, 0x90, 0xf7, 0x05, - 0x96, 0x80, 0x10, 0x72, 0x44, 0x63, 0x91, 0xbb, 0x8f, 0xfe, 0xa1, 0x02, 0xf7, 0xf3, 0x57, 0xba, - 0x25, 0x2a, 0x73, 0x6d, 0x19, 0xba, 0xe6, 0x11, 0x1e, 0xd2, 0xd3, 0x7f, 0x21, 0xc6, 0x5d, 0xe4, - 0x87, 0x0f, 0xed, 0x22, 0x7f, 0x75, 0xbf, 0x5d, 0xbc, 0xbf, 0xd4, 0x05, 0x6e, 0xdc, 0x55, 0x0f, - 0xd0, 0x5d, 0x18, 0x37, 0xe5, 0xf0, 0xdc, 0x82, 0xc9, 0xe7, 0xba, 0x5d, 0x8d, 0xc4, 0xf9, 0xe6, - 0x06, 0x86, 0x48, 0x11, 0x8e, 0x92, 0x9a, 0xbd, 0x0d, 0xe3, 0x91, 0x8d, 0x76, 0xac, 0x76, 0x69, - 0x0b, 0x26, 0xe3, 0xfb, 0xe1, 0x58, 0xf5, 0xbc, 0x9b, 0x30, 0x12, 0x08, 0x0b, 0xe8, 0x92, 0x44, - 0x28, 0x14, 0xbd, 0x6e, 0x92, 0x3d, 0x4e, 0xb5, 0x18, 0xb1, 0xa1, 0xf0, 0x4b, 0x53, 0x16, 0xdc, - 0x50, 0x20, 0x54, 0x7f, 0x5f, 0x5c, 0x9a, 0xae, 0x93, 0x46, 0xd3, 0xd4, 0x3c, 0xf2, 0xc6, 0x77, - 0x33, 0x52, 0xff, 0xa3, 0xc2, 0xcf, 0x1b, 0x2e, 0xda, 0x20, 0x0d, 0x46, 0x1b, 0x3c, 0x37, 0x1c, - 0x0b, 0x85, 0xa8, 0xe4, 0x0f, 0xc2, 0xb8, 0x12, 0xa2, 0xc1, 0x32, 0x4e, 0x74, 0x07, 0x46, 0x7c, - 0x61, 0xd0, 0xb7, 0xd4, 0x2c, 0xf5, 0x26, 0x9c, 0x05, 0x72, 0x67, 0x70, 0xb4, 0xfa, 0x25, 0x2e, - 0x0e, 0x69, 0xa9, 0x1a, 0xa0, 0x64, 0x1b, 0xf4, 0x40, 0xf8, 0x5c, 0x4d, 0x89, 0x66, 0x73, 0x49, - 0x3c, 0x59, 0xf3, 0x8d, 0xac, 0x85, 0x2c, 0x23, 0xab, 0xfa, 0xeb, 0x05, 0x98, 0x16, 0xaa, 0xe7, - 0xbc, 0xae, 0xdb, 0x2d, 0xcb, 0x0b, 0x5f, 0x11, 0xf3, 0xa7, 0xf9, 0x82, 0x08, 0x13, 0x27, 0xf9, - 0xbb, 0x7d, 0x2c, 0x20, 0xe8, 0x16, 0xb7, 0x7e, 0x5a, 0xb5, 0x75, 0xfb, 0x36, 0xb1, 0x42, 0x2e, - 0x21, 0xc7, 0xc0, 0x58, 0x4c, 0xab, 0x80, 0xd3, 0xdb, 0xa1, 0x1d, 0x40, 0x0d, 0x6d, 0x37, 0x8e, - 0xad, 0x87, 0x34, 0xfd, 0x2b, 0x09, 0x6c, 0x38, 0x85, 0x02, 0x3d, 0x48, 0xa9, 0x24, 0xd7, 0xf4, - 0x48, 0x8d, 0x0f, 0xd1, 0xf7, 0xd9, 0x60, 0x07, 0xe9, 0x7c, 0x14, 0x84, 0xe3, 0x75, 0xd5, 0x8f, - 0x0d, 0xc2, 0xf9, 0xe8, 0x24, 0xd2, 0x2f, 0xd4, 0x7f, 0x3d, 0xff, 0x94, 0xff, 0x34, 0x8c, 0x4f, - 0xe4, 0x03, 0xf1, 0xa7, 0x61, 0x33, 0xb2, 0x4f, 0x90, 0x68, 0x14, 0x79, 0x26, 0xf6, 0x0d, 0x78, - 0x0a, 0x9f, 0xf1, 0xf6, 0xb9, 0xef, 0x58, 0x9f, 0xfc, 0x7f, 0x52, 0x81, 0xd9, 0x68, 0xf1, 0x92, - 0x61, 0x19, 0xee, 0xb6, 0x48, 0x0b, 0x72, 0xf8, 0x97, 0x69, 0x2c, 0xb1, 0xf0, 0x72, 0x26, 0x46, - 0xdc, 0x81, 0x1a, 0xfa, 0x01, 0x05, 0x2e, 0xc4, 0xe6, 0x25, 0x92, 0xa4, 0xe4, 0xf0, 0x8f, 0xd4, - 0x58, 0xec, 0x96, 0xe5, 0x6c, 0x94, 0xb8, 0x13, 0x3d, 0xf4, 0x05, 0x05, 0xce, 0x36, 0xd3, 0x1e, - 0xb2, 0xfb, 0x6a, 0x69, 0x2e, 0x13, 0x5a, 0xea, 0xd3, 0xf8, 0x85, 0xcb, 0x62, 0x8b, 0x9e, 0x4d, - 0x05, 0xbb, 0x38, 0xa3, 0x23, 0xea, 0x3f, 0x2b, 0xc0, 0x00, 0x73, 0x8b, 0x7a, 0x63, 0xbc, 0x27, - 0x62, 0x5d, 0xcd, 0x74, 0x67, 0xad, 0xc7, 0xdc, 0x59, 0x9f, 0xca, 0x4f, 0xa2, 0xb3, 0x3f, 0xeb, - 0xfb, 0xe0, 0x2c, 0xab, 0x36, 0x5f, 0x63, 0x86, 0x36, 0x97, 0xd4, 0xe6, 0x6b, 0x35, 0xa6, 0xde, - 0x1e, 0x7c, 0x47, 0x76, 0x09, 0xfa, 0x5a, 0x8e, 0x19, 0x8f, 0xb0, 0xba, 0x81, 0x97, 0x31, 0x2d, - 0x57, 0x3f, 0xa9, 0xc0, 0x24, 0xc3, 0x2d, 0xb1, 0x18, 0xb4, 0x03, 0xc3, 0x8e, 0x60, 0x33, 0x62, - 0x6d, 0x96, 0x73, 0x0f, 0x2d, 0x85, 0x75, 0x71, 0x8d, 0xcd, 0xff, 0x85, 0x03, 0x5a, 0xea, 0x9f, - 0x0c, 0xc0, 0x4c, 0x56, 0x23, 0xf4, 0xc3, 0x0a, 0x9c, 0xd5, 0x43, 0x89, 0x73, 0xbe, 0xe5, 0x6d, - 0xdb, 0x8e, 0x9f, 0x70, 0x20, 0xb7, 0x39, 0xa4, 0x34, 0x1f, 0xf4, 0x8a, 0x65, 0xd9, 0x28, 0xa5, - 0x52, 0xc0, 0x19, 0x94, 0xd1, 0xab, 0x00, 0xae, 0xbb, 0x7d, 0x93, 0xec, 0x35, 0x35, 0xc3, 0x77, - 0xcc, 0xba, 0x99, 0x7f, 0x1b, 0x54, 0x6f, 0x08, 0x54, 0x41, 0xa7, 0xb8, 0xad, 0x3a, 0x2c, 0x97, - 0xc8, 0xa1, 0x8f, 0x2a, 0x30, 0x6e, 0xcb, 0xe1, 0x58, 0x7a, 0xf1, 0xd5, 0x4f, 0x8d, 0xeb, 0xc2, - 0x25, 0xed, 0x28, 0x28, 0x4a, 0x92, 0x2e, 0xcb, 0x94, 0x1b, 0x3f, 0xd9, 0x04, 0xef, 0x5b, 0xc9, - 0x27, 0x03, 0x65, 0x1c, 0x93, 0x5c, 0x6b, 0x4f, 0x82, 0x93, 0xe4, 0x59, 0xa7, 0x88, 0xa7, 0xd7, - 0x16, 0x2d, 0xdd, 0xd9, 0x63, 0x01, 0x00, 0x68, 0xa7, 0x06, 0xf3, 0x77, 0x6a, 0x71, 0xbd, 0x54, - 0x8e, 0x20, 0x8b, 0x76, 0x2a, 0x09, 0x4e, 0x92, 0x57, 0x3f, 0x52, 0x80, 0x73, 0x6c, 0x99, 0x6f, - 0xb6, 0x36, 0x09, 0xf7, 0xad, 0xfa, 0x7b, 0x17, 0x3f, 0xe7, 0xb7, 0x14, 0x18, 0x61, 0x73, 0xf0, - 0x06, 0x79, 0x82, 0xc9, 0xfa, 0x9a, 0xe1, 0xbf, 0xfc, 0x9b, 0x0a, 0x4c, 0x25, 0x52, 0x2c, 0x75, - 0xf5, 0x80, 0xef, 0xc4, 0x5c, 0x6b, 0xdf, 0x12, 0xa6, 0x75, 0xec, 0x0b, 0xe3, 0x56, 0xc4, 0x53, - 0x3a, 0xaa, 0xcf, 0xc1, 0x78, 0xc4, 0x7d, 0x59, 0x8a, 0xf0, 0x98, 0x16, 0x9b, 0x52, 0x0e, 0xe0, - 0x58, 0xe8, 0x14, 0x7a, 0x32, 0xdc, 0xf2, 0x49, 0xce, 0xf6, 0xf7, 0x67, 0xcb, 0x23, 0xb1, 0xe5, - 0xfd, 0x6b, 0x1c, 0x16, 0x37, 0xd2, 0xed, 0xe5, 0x1a, 0x87, 0x05, 0xa0, 0x14, 0x71, 0x5e, 0xf9, - 0xff, 0x58, 0x60, 0x45, 0x4f, 0x47, 0xa3, 0xb8, 0xae, 0x86, 0xba, 0xdd, 0x74, 0x3c, 0xf6, 0x2a, - 0xdb, 0x92, 0x89, 0xda, 0x08, 0xf3, 0x8b, 0x20, 0x7e, 0x96, 0xe5, 0x4a, 0xfe, 0x52, 0x5e, 0xad, - 0xf2, 0xf0, 0x7e, 0xc1, 0x05, 0x50, 0xd4, 0xe2, 0xdd, 0x7f, 0x12, 0x16, 0x6f, 0x07, 0x46, 0xb7, - 0x8d, 0x4d, 0xe2, 0x58, 0x5c, 0x8c, 0x19, 0xc8, 0x2f, 0xa1, 0xdd, 0x08, 0xd1, 0x70, 0x33, 0x80, - 0x54, 0x80, 0x65, 0x22, 0xc8, 0x89, 0xc4, 0xa2, 0x1e, 0xcc, 0x2f, 0x95, 0x84, 0xa6, 0xe9, 0x70, - 0x9c, 0x19, 0x71, 0xa8, 0x2d, 0x00, 0x2b, 0x08, 0xd0, 0xda, 0xcb, 0xc5, 0x50, 0x18, 0xe6, 0x95, - 0x0b, 0x1d, 0xe1, 0x6f, 0x2c, 0x51, 0xa0, 0xf3, 0xda, 0x08, 0xe3, 0xfd, 0x0b, 0x33, 0xe3, 0x53, - 0x3d, 0xe6, 0x5c, 0x10, 0xe6, 0x95, 0xb0, 0x00, 0xcb, 0x44, 0xe8, 0x18, 0x1b, 0x41, 0x94, 0x7e, - 0x61, 0x46, 0xcc, 0x35, 0xc6, 0x30, 0xd6, 0x3f, 0x1f, 0x63, 0xf8, 0x1b, 0x4b, 0x14, 0xd0, 0xcb, - 0xd2, 0xfd, 0x21, 0xe4, 0x37, 0x52, 0x75, 0x75, 0x77, 0xf8, 0xce, 0xd0, 0x56, 0x33, 0xca, 0xbe, - 0xd3, 0x0b, 0x92, 0x9d, 0x86, 0x65, 0x2f, 0xa0, 0xbc, 0x23, 0x61, 0xb7, 0x09, 0x1f, 0x4d, 0x8c, - 0x75, 0x7c, 0x34, 0x51, 0xa2, 0xd2, 0x99, 0xf4, 0xf0, 0x90, 0x31, 0x84, 0xf1, 0xf0, 0x12, 0xa4, - 0x1a, 0x07, 0xe2, 0x64, 0x7d, 0xce, 0xf0, 0x49, 0x8d, 0xb5, 0x9d, 0x90, 0x19, 0x3e, 0x2f, 0xc3, - 0x01, 0x14, 0xed, 0xc0, 0x98, 0x2b, 0xbd, 0xc0, 0x98, 0x39, 0xd5, 0xeb, 0x15, 0xa2, 0x78, 0x7d, - 0xc1, 0x5c, 0x1d, 0xe5, 0x12, 0x1c, 0xa1, 0x13, 0xbd, 0xdf, 0x9a, 0x3c, 0xd9, 0xfb, 0x2d, 0xd4, - 0x8a, 0x3a, 0x57, 0x4f, 0x1d, 0x49, 0x98, 0x96, 0x03, 0x9d, 0xaf, 0xe9, 0xd2, 0x92, 0xdd, 0xa6, - 0xed, 0xb6, 0x1c, 0xc2, 0xb2, 0xcd, 0xb0, 0xe5, 0x41, 0xe1, 0xd2, 0x2e, 0xc6, 0x81, 0x38, 0x59, - 0x1f, 0x7d, 0x9f, 0x02, 0x93, 0xee, 0x9e, 0xeb, 0x91, 0x06, 0x3d, 0xb6, 0x6c, 0x8b, 0x58, 0x9e, - 0x3b, 0x73, 0x3a, 0x7f, 0xd8, 0xef, 0x6a, 0x0c, 0x17, 0x3f, 0x76, 0xe2, 0xa5, 0x38, 0x41, 0x93, - 0xee, 0x1c, 0x39, 0xd0, 0xcb, 0xcc, 0x74, 0xfe, 0x9d, 0x23, 0x07, 0x91, 0xe1, 0x3b, 0x47, 0x2e, - 0xc1, 0x11, 0x3a, 0xe8, 0x51, 0x18, 0x77, 0xfd, 0x14, 0xdf, 0x6c, 0x06, 0xcf, 0x84, 0xb1, 0x38, - 0xab, 0x32, 0x00, 0x47, 0xeb, 0xa1, 0x0f, 0xc1, 0x98, 0x7c, 0x76, 0xce, 0x9c, 0x3d, 0xea, 0xa8, - 0xf4, 0xbc, 0xe7, 0x32, 0x28, 0x42, 0x10, 0x61, 0x38, 0x2b, 0xbd, 0x0d, 0x94, 0xbf, 0xef, 0x73, - 0x6c, 0x08, 0x5c, 0x9f, 0x4d, 0xad, 0x81, 0x33, 0x5a, 0xa2, 0x1f, 0x4f, 0xbf, 0x2e, 0x9f, 0x61, - 0x5b, 0x7a, 0xed, 0x48, 0xae, 0xcb, 0x9f, 0x33, 0xbc, 0xed, 0x5b, 0x4d, 0x1e, 0x38, 0xec, 0x90, - 0x37, 0xe7, 0xea, 0x1f, 0x2a, 0x00, 0x81, 0xc1, 0xe4, 0x24, 0xae, 0x2a, 0x6a, 0x11, 0x1b, 0xd2, - 0x42, 0x4f, 0x06, 0x9e, 0xcc, 0xa4, 0x23, 0xea, 0x57, 0x14, 0x98, 0x08, 0xab, 0x9d, 0x80, 0x6a, - 0xa4, 0x47, 0x55, 0xa3, 0x27, 0x7b, 0x1b, 0x57, 0x86, 0x7e, 0xf4, 0xbf, 0x0a, 0xf2, 0xa8, 0x98, - 0xf4, 0xbb, 0x13, 0xb9, 0xfa, 0xa7, 0xa4, 0x6f, 0xf4, 0x72, 0xf5, 0x2f, 0x07, 0xd7, 0x08, 0xc7, - 0x9b, 0xe2, 0x0a, 0xf0, 0xdd, 0x11, 0xf9, 0xb3, 0x87, 0xb0, 0x36, 0x81, 0xb0, 0xe9, 0x93, 0xe6, - 0x13, 0x70, 0x90, 0x30, 0xfa, 0x8a, 0x7c, 0x3c, 0xf5, 0x90, 0x28, 0x24, 0x32, 0xe0, 0xce, 0x4e, - 0x17, 0x9f, 0x9e, 0x84, 0x51, 0xc9, 0xb6, 0x18, 0x73, 0x64, 0x50, 0x4e, 0xc2, 0x91, 0xc1, 0x83, - 0x51, 0x3d, 0xc8, 0xb4, 0xe8, 0x4f, 0x7b, 0x8f, 0x34, 0x83, 0x63, 0x31, 0xcc, 0xe1, 0xe8, 0x62, - 0x99, 0x0c, 0x15, 0xde, 0x82, 0x3d, 0xd6, 0x77, 0x04, 0xee, 0x25, 0x9d, 0xf6, 0xd5, 0x23, 0x00, - 0xbe, 0xfc, 0x4f, 0x6a, 0x22, 0x00, 0x7b, 0xe0, 0x3a, 0x5d, 0x71, 0x6f, 0x04, 0x30, 0x2c, 0xd5, - 0x4b, 0x5e, 0x8c, 0x0f, 0x9c, 0xd8, 0xc5, 0x38, 0xdd, 0x06, 0xa6, 0x9f, 0x93, 0xbe, 0x27, 0x77, - 0xb5, 0x20, 0xb3, 0x7d, 0xb8, 0x0d, 0x82, 0x22, 0x17, 0x4b, 0x44, 0x32, 0xfc, 0x59, 0x86, 0x72, - 0xf9, 0xb3, 0xb4, 0xe0, 0xb4, 0x43, 0x3c, 0x67, 0xaf, 0xb4, 0xa7, 0xb3, 0xcc, 0x25, 0x8e, 0xc7, - 0x34, 0xf8, 0xe1, 0x7c, 0xf1, 0x10, 0x71, 0x12, 0x15, 0x4e, 0xc3, 0x1f, 0x11, 0x80, 0x47, 0x3a, - 0x0a, 0xc0, 0xef, 0x84, 0x51, 0x8f, 0xe8, 0xdb, 0x96, 0xa1, 0x6b, 0x66, 0xa5, 0x2c, 0xc2, 0x73, - 0x87, 0xb2, 0x5c, 0x08, 0xc2, 0x72, 0x3d, 0xb4, 0x00, 0x7d, 0x2d, 0xa3, 0x26, 0x34, 0x80, 0x6f, - 0x0b, 0xac, 0xf4, 0x95, 0xf2, 0xbd, 0x76, 0xf1, 0xcd, 0xa1, 0x83, 0x48, 0x30, 0xaa, 0x6b, 0xcd, - 0xdb, 0xf5, 0x6b, 0xde, 0x5e, 0x93, 0xb8, 0x73, 0x1b, 0x95, 0x32, 0xa6, 0x8d, 0xd3, 0x7c, 0x7d, - 0xc6, 0x0e, 0xe1, 0xeb, 0xf3, 0x59, 0x05, 0x4e, 0x6b, 0xf1, 0x0b, 0x06, 0xe2, 0xce, 0x8c, 0xe7, - 0xe7, 0x96, 0xe9, 0x97, 0x16, 0x0b, 0x17, 0xc4, 0xf8, 0x4e, 0xcf, 0x27, 0xc9, 0xe1, 0xb4, 0x3e, - 0x20, 0x07, 0x50, 0xc3, 0xa8, 0x07, 0xe9, 0xe1, 0xc5, 0xaa, 0x4f, 0xe4, 0xb3, 0xdb, 0xac, 0x24, - 0x30, 0xe1, 0x14, 0xec, 0xe8, 0x0e, 0x8c, 0x4a, 0x42, 0x92, 0xd0, 0x64, 0xca, 0x47, 0x71, 0x0f, - 0xc2, 0xb5, 0x5d, 0xf9, 0x8e, 0x43, 0xa6, 0x14, 0x5c, 0x72, 0x4a, 0x66, 0x06, 0x71, 0xd1, 0xc7, - 0x46, 0x3d, 0x99, 0xff, 0x92, 0x33, 0x1d, 0x23, 0xee, 0x40, 0x8d, 0x45, 0x21, 0xa4, 0x60, 0x49, - 0x37, 0x9f, 0x99, 0xca, 0x1f, 0x46, 0x65, 0x39, 0x8a, 0x8a, 0x6f, 0xcd, 0x58, 0x21, 0x8e, 0x13, - 0x44, 0x4b, 0x80, 0x08, 0x37, 0xa5, 0x87, 0xca, 0x99, 0x3b, 0x83, 0xd8, 0xfd, 0x3b, 0x5b, 0xd2, - 0xc5, 0x04, 0x14, 0xa7, 0xb4, 0x40, 0x5e, 0xc4, 0x56, 0xd2, 0x83, 0x96, 0x13, 0x4f, 0x89, 0xd3, - 0xd1, 0x62, 0xf2, 0xbd, 0x4a, 0x22, 0xc7, 0x30, 0x57, 0x6e, 0x6e, 0xf4, 0x9e, 0x63, 0x58, 0x90, - 0xef, 0x22, 0xd3, 0xb0, 0xfa, 0x07, 0x8a, 0xb0, 0xf2, 0x9e, 0xa0, 0xa7, 0xcf, 0x71, 0x5f, 0xc1, - 0xaa, 0xcf, 0xc1, 0x4c, 0xd5, 0x0f, 0xcf, 0x59, 0x8b, 0x45, 0xe1, 0x7f, 0x0f, 0x8c, 0xf3, 0x5b, - 0x96, 0x15, 0xad, 0xb9, 0x1a, 0x9a, 0xe4, 0x83, 0x40, 0x0e, 0x25, 0x19, 0x88, 0xa3, 0x75, 0xd5, - 0xaf, 0x29, 0x70, 0x2e, 0x8a, 0xd9, 0x76, 0x8c, 0xbb, 0xbd, 0x23, 0x46, 0x1f, 0x57, 0x60, 0xf4, - 0x76, 0x70, 0x01, 0xe4, 0x4b, 0x45, 0xb9, 0x5e, 0x43, 0xf8, 0xbd, 0x22, 0x8e, 0x74, 0xa3, 0x94, - 0x4c, 0xed, 0x18, 0x02, 0x5d, 0x2c, 0x93, 0x56, 0xff, 0xab, 0x02, 0x09, 0xcd, 0x1c, 0x6d, 0xc2, - 0x10, 0x25, 0x52, 0x5e, 0xad, 0x8a, 0x3d, 0xf1, 0x9e, 0x7c, 0x02, 0x1b, 0x43, 0xc1, 0xef, 0x1b, - 0xc4, 0x0f, 0xec, 0x23, 0xa6, 0xba, 0xbe, 0x25, 0xa5, 0xe8, 0x11, 0xdb, 0x23, 0x97, 0x44, 0x2c, - 0xa7, 0xfa, 0xe1, 0x1a, 0xb3, 0x5c, 0x82, 0x23, 0x74, 0xd4, 0x65, 0x80, 0xd0, 0x9a, 0xd2, 0xb3, - 0xe7, 0xdc, 0xbf, 0x3b, 0x0d, 0x67, 0x7a, 0x7e, 0xe7, 0xf7, 0x51, 0x05, 0xce, 0x92, 0x1d, 0x43, - 0xf7, 0xe6, 0xb7, 0x3c, 0xe2, 0xdc, 0xba, 0xb5, 0xb2, 0xbe, 0xed, 0x10, 0x77, 0xdb, 0x36, 0x6b, - 0xdd, 0xf8, 0x09, 0xa6, 0x38, 0x35, 0x31, 0xad, 0x7f, 0x31, 0x15, 0x23, 0xce, 0xa0, 0xc4, 0x2c, - 0x49, 0x3b, 0x5c, 0xc7, 0xc6, 0x54, 0x9d, 0x69, 0x39, 0xae, 0x27, 0x22, 0xee, 0x71, 0x4b, 0x52, - 0x1c, 0x88, 0x93, 0xf5, 0xe3, 0x48, 0x58, 0xfe, 0x39, 0x26, 0x12, 0x2b, 0x49, 0x24, 0x0c, 0x88, - 0x93, 0xf5, 0x65, 0x24, 0x7c, 0xa5, 0xe8, 0x79, 0x33, 0x90, 0x44, 0x12, 0x00, 0x71, 0xb2, 0x3e, - 0xaa, 0xc1, 0x45, 0x87, 0xe8, 0x76, 0xa3, 0x41, 0xac, 0x1a, 0x9b, 0x94, 0x15, 0xcd, 0xa9, 0x1b, - 0xd6, 0x92, 0xa3, 0xb1, 0x8a, 0xcc, 0x30, 0xaf, 0xb0, 0x54, 0xb1, 0x17, 0x71, 0x87, 0x7a, 0xb8, - 0x23, 0x16, 0xd4, 0x80, 0x53, 0x2d, 0xc6, 0x5a, 0x1d, 0x16, 0x91, 0x6b, 0x47, 0x33, 0x85, 0xf5, - 0xfd, 0xb0, 0x2b, 0xc6, 0xce, 0xc0, 0x8d, 0x28, 0x2a, 0x1c, 0xc7, 0x8d, 0xf6, 0xa8, 0xe4, 0x2b, - 0xba, 0x23, 0x91, 0x1c, 0xce, 0x45, 0x52, 0x48, 0xbf, 0x09, 0x74, 0x38, 0x8d, 0x06, 0xaa, 0xc0, - 0x69, 0x4f, 0x73, 0xea, 0xc4, 0x2b, 0xad, 0x6d, 0xac, 0x11, 0x47, 0xa7, 0x3c, 0xd6, 0xe4, 0x82, - 0xb0, 0xc2, 0x51, 0xad, 0x27, 0xc1, 0x38, 0xad, 0x0d, 0xfa, 0x10, 0xbc, 0x25, 0x3a, 0xa9, 0xcb, - 0xf6, 0x1d, 0xe2, 0x2c, 0xd8, 0x2d, 0xab, 0x16, 0x45, 0x0e, 0x0c, 0xf9, 0x03, 0xfb, 0xed, 0xe2, - 0x5b, 0x70, 0x37, 0x0d, 0x70, 0x77, 0x78, 0x93, 0x1d, 0xd8, 0x68, 0x36, 0x53, 0x3b, 0x30, 0x9a, - 0xd5, 0x81, 0x8c, 0x06, 0xb8, 0x3b, 0xbc, 0x08, 0xc3, 0x59, 0x3e, 0x31, 0x3c, 0xb1, 0xb1, 0x44, - 0x71, 0x8c, 0x51, 0x64, 0xdf, 0xef, 0x7a, 0x6a, 0x0d, 0x9c, 0xd1, 0x92, 0x9e, 0x29, 0x57, 0xb3, - 0x86, 0x9f, 0x20, 0x33, 0xce, 0xc8, 0xbc, 0x6d, 0xbf, 0x5d, 0xbc, 0x8a, 0xbb, 0x6c, 0x83, 0xbb, - 0xc6, 0x9e, 0xd2, 0x95, 0x70, 0x22, 0x12, 0x5d, 0x99, 0xc8, 0xea, 0x4a, 0x76, 0x1b, 0xdc, 0x35, - 0x76, 0xf4, 0xfd, 0x0a, 0x9c, 0xd7, 0x9b, 0xad, 0x1b, 0x86, 0xeb, 0xd9, 0x75, 0x47, 0x6b, 0x94, - 0x89, 0xae, 0xed, 0xdd, 0xd0, 0xcc, 0xad, 0x65, 0x63, 0x8b, 0x08, 0x79, 0xfe, 0xb0, 0x1f, 0x0e, - 0x7b, 0xd3, 0x5a, 0x5a, 0xdb, 0x48, 0x47, 0x8a, 0xb3, 0xe9, 0xa1, 0x1f, 0x51, 0xe0, 0x62, 0x83, - 0x75, 0x31, 0xa3, 0x43, 0x93, 0xb9, 0x3a, 0xc4, 0xb8, 0xd8, 0x4a, 0x07, 0xbc, 0xb8, 0x23, 0x55, - 0x36, 0x49, 0xbc, 0xc2, 0x7c, 0xbd, 0xee, 0x90, 0x3a, 0xc3, 0x1a, 0x70, 0x97, 0xa9, 0xfc, 0x93, - 0xb4, 0x92, 0x85, 0x14, 0x67, 0xd3, 0x43, 0x2f, 0xc3, 0xe5, 0x4c, 0x60, 0xc9, 0x6e, 0x59, 0x1e, - 0xbb, 0xdf, 0xe8, 0x5b, 0x50, 0xf7, 0xdb, 0xc5, 0xcb, 0x2b, 0x1d, 0x6b, 0xe2, 0x03, 0x30, 0xb1, - 0xa8, 0x1d, 0x91, 0x17, 0xf6, 0xa7, 0x99, 0x24, 0xf6, 0x62, 0xae, 0x44, 0xb6, 0x47, 0xf8, 0xac, - 0xfe, 0x2b, 0xd1, 0x67, 0xf5, 0xd3, 0xac, 0x57, 0xef, 0x3b, 0xba, 0x5e, 0x7d, 0xf3, 0x2d, 0x7d, - 0x37, 0x6f, 0x2c, 0x3e, 0xab, 0x80, 0x78, 0x1a, 0x89, 0x2e, 0x46, 0x3c, 0x7a, 0x86, 0x63, 0xde, - 0x3c, 0x7e, 0x3e, 0xd9, 0x42, 0x6a, 0x3e, 0xd9, 0xb7, 0x4a, 0xe1, 0x8b, 0x47, 0x42, 0x65, 0x89, - 0x63, 0x0e, 0xe3, 0x17, 0xa3, 0x07, 0x61, 0x24, 0xd0, 0x57, 0x85, 0x1d, 0x91, 0xe5, 0x72, 0x09, - 0x15, 0xdb, 0x10, 0xae, 0xfe, 0x9e, 0x02, 0x10, 0xa6, 0x49, 0x46, 0xf7, 0xc1, 0x80, 0x6e, 0x6a, - 0xae, 0x1b, 0x4f, 0xb4, 0xcf, 0x2e, 0xf9, 0x30, 0x87, 0x1d, 0xec, 0xe7, 0x8f, 0x54, 0x18, 0x6c, - 0xb1, 0xc4, 0x90, 0xc2, 0x37, 0x9f, 0x79, 0x97, 0x6c, 0xb0, 0x12, 0x2c, 0x20, 0x68, 0x03, 0x86, - 0x1a, 0x86, 0xc5, 0x9e, 0x51, 0xf4, 0xe7, 0x7a, 0x46, 0xc1, 0xf4, 0x81, 0x15, 0x8e, 0x02, 0xfb, - 0xb8, 0xd4, 0x5f, 0x54, 0xe0, 0x54, 0x34, 0x9e, 0x34, 0x8b, 0x26, 0x23, 0xc2, 0x8c, 0x8a, 0x30, - 0xf6, 0xac, 0xa9, 0x08, 0x9f, 0x88, 0x7d, 0x58, 0xf4, 0xe2, 0xb7, 0x07, 0xc3, 0x7e, 0x7a, 0x58, - 0xeb, 0x03, 0x6c, 0xec, 0x3f, 0x76, 0x1a, 0x06, 0xb9, 0x87, 0x33, 0x95, 0xe3, 0x53, 0xc2, 0x46, - 0xdd, 0xcc, 0x9f, 0xa9, 0x21, 0x4f, 0xb0, 0x19, 0x39, 0x5f, 0x65, 0xa1, 0x63, 0xbe, 0x4a, 0x0c, - 0x7d, 0xba, 0x63, 0xf4, 0xe2, 0xe4, 0x53, 0xc2, 0x15, 0xee, 0xe4, 0x53, 0xc2, 0x15, 0x4c, 0x91, - 0x21, 0x2f, 0xe2, 0xfd, 0xd2, 0x9f, 0xdf, 0xba, 0xc2, 0x27, 0x40, 0xf2, 0x81, 0x99, 0xe8, 0xe8, - 0xff, 0xe2, 0xc7, 0xa8, 0x1f, 0xc8, 0xff, 0xee, 0x46, 0x4c, 0x79, 0x37, 0x31, 0xea, 0xfd, 0x0f, - 0x69, 0x30, 0xf3, 0x43, 0xda, 0x82, 0x21, 0xf1, 0x29, 0x08, 0x85, 0xe0, 0x3d, 0x3d, 0x64, 0x65, - 0x97, 0xf2, 0x3f, 0xf1, 0x02, 0xec, 0x23, 0xa7, 0x5a, 0x66, 0x43, 0xdb, 0x35, 0x1a, 0xad, 0x06, - 0xd3, 0x02, 0x06, 0xe4, 0xaa, 0xac, 0x18, 0xfb, 0x70, 0x56, 0x95, 0x3f, 0x57, 0x62, 0x52, 0xbb, - 0x5c, 0x95, 0x17, 0x63, 0x1f, 0x8e, 0x5e, 0x80, 0xe1, 0x86, 0xb6, 0x5b, 0x6d, 0x39, 0x75, 0x22, - 0x7c, 0x5f, 0xb2, 0x8d, 0x42, 0x2d, 0xcf, 0x30, 0xe7, 0x0c, 0xcb, 0x73, 0x3d, 0x67, 0xae, 0x62, - 0x79, 0xb7, 0x9c, 0xaa, 0xc7, 0x7c, 0x6b, 0xd8, 0xae, 0x5b, 0x11, 0x58, 0x70, 0x80, 0x0f, 0x99, - 0x30, 0xd1, 0xd0, 0x76, 0x37, 0x2c, 0x8d, 0xa7, 0x1f, 0x10, 0x52, 0x76, 0x1e, 0x0a, 0xcc, 0xe0, - 0xb5, 0x12, 0xc1, 0x85, 0x63, 0xb8, 0x53, 0xfc, 0x2c, 0xc7, 0x8e, 0xcb, 0xcf, 0x72, 0x3e, 0x08, - 0x00, 0xc0, 0xad, 0xe5, 0xe7, 0x53, 0x23, 0xeb, 0x75, 0x7c, 0xdc, 0xff, 0x52, 0xf0, 0xb8, 0x7f, - 0x22, 0xbf, 0x63, 0x60, 0x87, 0x87, 0xfd, 0x2d, 0x18, 0xad, 0x69, 0x9e, 0xc6, 0x4b, 0xdd, 0x99, - 0x53, 0xf9, 0x2f, 0x7e, 0xcb, 0x01, 0x9a, 0x90, 0x25, 0x85, 0x65, 0x2e, 0x96, 0xe9, 0xa0, 0x5b, - 0x70, 0x86, 0x7e, 0xac, 0x26, 0xf1, 0xc2, 0x2a, 0xcc, 0x66, 0x36, 0xc9, 0xbe, 0x1f, 0xf6, 0x00, - 0xec, 0x66, 0x5a, 0x05, 0x9c, 0xde, 0x2e, 0x8c, 0x02, 0x3b, 0x95, 0x1e, 0x05, 0x16, 0x7d, 0x2a, - 0xcd, 0xa3, 0x05, 0xb1, 0x39, 0x7d, 0x6f, 0x7e, 0xde, 0x90, 0xdb, 0xaf, 0xe5, 0x9f, 0x2b, 0x30, - 0x23, 0x76, 0x99, 0xf0, 0x42, 0x31, 0x89, 0xb3, 0xa2, 0x59, 0x5a, 0x9d, 0x38, 0xc2, 0x04, 0xbd, - 0xde, 0x03, 0x7f, 0x48, 0xe0, 0x0c, 0xa2, 0x2e, 0xdc, 0xbf, 0xdf, 0x2e, 0x5e, 0x39, 0xa8, 0x16, - 0xce, 0xec, 0x1b, 0x72, 0x60, 0xc8, 0xdd, 0x73, 0x75, 0xcf, 0x74, 0x85, 0x0c, 0x7a, 0xbd, 0x07, - 0xce, 0x5a, 0xe5, 0x98, 0x38, 0x6b, 0x0d, 0xb3, 0x0e, 0xf2, 0x52, 0xec, 0x13, 0x42, 0x3f, 0xa4, - 0xc0, 0x94, 0xb8, 0x97, 0x92, 0xa2, 0xda, 0x9c, 0xc9, 0xff, 0x04, 0xa5, 0x14, 0x47, 0xe6, 0x7b, - 0x9e, 0x30, 0x6b, 0x52, 0x02, 0x8a, 0x93, 0xd4, 0xe9, 0xa1, 0xda, 0x74, 0x0c, 0xdb, 0x31, 0xbc, - 0x3d, 0xe6, 0xe3, 0x33, 0xe0, 0xc7, 0x7e, 0xe7, 0x65, 0x38, 0x80, 0xa2, 0x2a, 0x4c, 0x70, 0xab, - 0x4d, 0xd5, 0x73, 0x34, 0x8f, 0xd4, 0xf7, 0x84, 0x23, 0xce, 0x83, 0x2c, 0x3b, 0x6f, 0x04, 0x72, - 0xaf, 0x5d, 0x3c, 0x23, 0xd6, 0x26, 0x0a, 0xc0, 0x31, 0x14, 0xe8, 0x03, 0x31, 0xbf, 0xa8, 0x99, - 0xfc, 0x99, 0xfb, 0xf8, 0x5a, 0x1c, 0xc6, 0x3b, 0xaa, 0xd7, 0x40, 0x6d, 0x3d, 0x64, 0x4f, 0x99, - 0x7d, 0x1c, 0xc6, 0xe4, 0x5d, 0x73, 0xa8, 0xf8, 0x70, 0x1e, 0xa0, 0xe4, 0x60, 0x8f, 0x3b, 0x84, - 0x8e, 0xfa, 0x53, 0x0a, 0x4c, 0xc6, 0x65, 0x17, 0xb4, 0x0d, 0x43, 0x82, 0x91, 0x09, 0xaa, 0xf3, - 0x79, 0x1d, 0x82, 0x4d, 0x22, 0x5e, 0xde, 0x72, 0x51, 0x58, 0x14, 0x61, 0x1f, 0xbd, 0xec, 0xec, - 0x5f, 0xe8, 0xe0, 0xec, 0xff, 0x04, 0x9c, 0x4d, 0x67, 0x69, 0x54, 0x91, 0xd0, 0xa8, 0x7e, 0x25, - 0x8c, 0xd6, 0x81, 0x22, 0xc1, 0x94, 0x2e, 0xcc, 0x61, 0xea, 0x07, 0x21, 0x9e, 0x35, 0x0c, 0xbd, - 0x0c, 0x23, 0xae, 0xbb, 0xcd, 0x5d, 0xb8, 0xc4, 0x20, 0xf3, 0x5d, 0xf5, 0xf8, 0x19, 0x5c, 0xb8, - 0xee, 0x13, 0xfc, 0xc4, 0x21, 0xfa, 0x85, 0xe7, 0xbf, 0xfc, 0xb5, 0xcb, 0x6f, 0xfa, 0xfd, 0xaf, - 0x5d, 0x7e, 0xd3, 0x57, 0xbf, 0x76, 0xf9, 0x4d, 0x1f, 0xde, 0xbf, 0xac, 0x7c, 0x79, 0xff, 0xb2, - 0xf2, 0xfb, 0xfb, 0x97, 0x95, 0xaf, 0xee, 0x5f, 0x56, 0xfe, 0x62, 0xff, 0xb2, 0xf2, 0x83, 0xff, - 0xfe, 0xf2, 0x9b, 0x5e, 0x78, 0x38, 0xa4, 0x7e, 0xcd, 0x27, 0x1a, 0xfe, 0xd3, 0xbc, 0x5d, 0xa7, - 0x4a, 0xab, 0xeb, 0x6b, 0xad, 0x8c, 0xfa, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x17, 0x2b, 0xba, - 0x40, 0xdd, 0x23, 0x01, 0x00, + 0x90, 0x0b, 0x53, 0xec, 0x0b, 0x15, 0x27, 0xf9, 0xf5, 0xd2, 0x7c, 0x83, 0xcc, 0x5c, 0xcc, 0x35, + 0xb9, 0x54, 0xf6, 0x9f, 0xaa, 0xc4, 0x91, 0xe1, 0x24, 0xfe, 0x90, 0xa8, 0xb6, 0x13, 0x12, 0xbd, + 0xd4, 0x2b, 0x51, 0x09, 0x19, 0x4e, 0xe2, 0x57, 0xf7, 0x0a, 0x70, 0x26, 0xf5, 0x94, 0xa4, 0xcc, + 0x83, 0x4f, 0xf1, 0xbc, 0x9f, 0xff, 0x4b, 0x5c, 0x7b, 0x32, 0xe6, 0xb1, 0x12, 0x05, 0xe1, 0x78, + 0x5d, 0x2a, 0xc3, 0x32, 0x6a, 0x4b, 0xb5, 0xb0, 0x7d, 0x21, 0x94, 0x61, 0x2b, 0x31, 0x18, 0x4e, + 0xd4, 0x46, 0x25, 0x31, 0x1f, 0x4b, 0xb5, 0x0a, 0x55, 0x03, 0xdd, 0x25, 0x87, 0xf8, 0xda, 0x41, + 0x38, 0x3e, 0x19, 0x88, 0x93, 0xf5, 0xe9, 0x28, 0xe8, 0x0f, 0xb9, 0x17, 0xfd, 0xe1, 0x28, 0x56, + 0xa3, 0x20, 0x1c, 0xaf, 0xeb, 0xeb, 0xe9, 0x91, 0x2e, 0x0c, 0x84, 0xa3, 0x58, 0x8d, 0xc1, 0x70, + 0xa2, 0xb6, 0xfa, 0x67, 0xfd, 0x70, 0x5f, 0x17, 0x92, 0x25, 0x6a, 0xa6, 0x4f, 0xf7, 0xe1, 0x79, + 0x5e, 0x77, 0xcb, 0xd3, 0xca, 0x58, 0x9e, 0xc3, 0xd3, 0xeb, 0x76, 0x39, 0xdd, 0xac, 0xe5, 0x3c, + 0x3c, 0xc9, 0xee, 0x97, 0xbf, 0x99, 0xbe, 0xfc, 0x39, 0x67, 0xf5, 0xc0, 0xed, 0xd2, 0xca, 0xd8, + 0x2e, 0x39, 0x67, 0xb5, 0x8b, 0xed, 0xf5, 0xe7, 0xfd, 0x70, 0x7f, 0x37, 0x52, 0x6e, 0xce, 0xfd, + 0x95, 0xc2, 0x5b, 0x8e, 0x75, 0x7f, 0x65, 0xbd, 0xe7, 0x3e, 0xc6, 0xfd, 0xd5, 0x91, 0x7d, 0x1e, + 0xcf, 0xfe, 0xca, 0x9a, 0xd5, 0xe3, 0xda, 0x5f, 0x59, 0xb3, 0xda, 0xc5, 0xfe, 0xfa, 0x9b, 0xf8, + 0xf9, 0x10, 0x88, 0xda, 0x15, 0xe8, 0xd3, 0x5b, 0xed, 0x9c, 0x4c, 0x8a, 0x79, 0x19, 0x96, 0xaa, + 0xeb, 0x98, 0xe2, 0x40, 0x18, 0x06, 0xf9, 0xfe, 0xc9, 0xc9, 0x82, 0x98, 0xe7, 0x28, 0xdf, 0x92, + 0x58, 0x60, 0xa2, 0x53, 0x45, 0x5a, 0x5b, 0xa4, 0x49, 0x1c, 0xcd, 0xac, 0x79, 0xb6, 0xa3, 0x35, + 0xf2, 0x72, 0x1b, 0x7e, 0x83, 0x12, 0xc3, 0x85, 0x13, 0xd8, 0xe9, 0x84, 0xb4, 0x8c, 0x7a, 0x4e, + 0xfe, 0xc2, 0x26, 0xa4, 0x5a, 0x29, 0x63, 0x8a, 0x43, 0xfd, 0xb5, 0x61, 0x90, 0x82, 0xd3, 0xa3, + 0x4f, 0x2a, 0x30, 0xa5, 0xc7, 0x63, 0x72, 0xf6, 0xe2, 0x0f, 0x95, 0x08, 0xf0, 0xc9, 0xb7, 0x7c, + 0xa2, 0x18, 0x27, 0xc9, 0xa2, 0x0f, 0x2b, 0xdc, 0xc8, 0x17, 0xdc, 0xe6, 0x89, 0x69, 0xbd, 0x7e, + 0x44, 0xf7, 0xde, 0xa1, 0xb5, 0x30, 0xbc, 0x62, 0x8d, 0x12, 0x44, 0xaf, 0x29, 0x70, 0xe6, 0x76, + 0xda, 0xdd, 0x84, 0x98, 0xfc, 0x5b, 0x79, 0xbb, 0x92, 0x71, 0xd9, 0xc1, 0x25, 0xc8, 0xd4, 0x0a, + 0x38, 0xbd, 0x23, 0xc1, 0x2c, 0x05, 0xe6, 0x5a, 0xf1, 0x9d, 0xe6, 0x9e, 0xa5, 0x98, 0xdd, 0x37, + 0x9c, 0xa5, 0x00, 0x80, 0xa3, 0x04, 0x51, 0x0b, 0x46, 0x6e, 0xfb, 0x36, 0x72, 0x61, 0x17, 0x2b, + 0xe5, 0xa5, 0x2e, 0x19, 0xda, 0xb9, 0xbf, 0x57, 0x50, 0x88, 0x43, 0x22, 0x68, 0x0b, 0x86, 0x6e, + 0x73, 0x5e, 0x21, 0xec, 0x59, 0xf3, 0x3d, 0x6b, 0xff, 0xdc, 0xac, 0x22, 0x8a, 0xb0, 0x8f, 0x5e, + 0x7e, 0x4b, 0x30, 0x7c, 0xc0, 0x13, 0xb7, 0xcf, 0x2a, 0x70, 0x66, 0x9b, 0x38, 0x9e, 0xa1, 0xc7, + 0x6f, 0x86, 0x46, 0xf2, 0x5b, 0x28, 0x9e, 0x4d, 0x43, 0xc8, 0xb7, 0x49, 0x2a, 0x08, 0xa7, 0x77, + 0x01, 0x3d, 0x0b, 0xfd, 0xc4, 0xd3, 0xeb, 0x22, 0x80, 0xf3, 0xbb, 0xf3, 0x3e, 0xd9, 0xe5, 0xef, + 0x5e, 0xe8, 0x7f, 0x98, 0xe1, 0x53, 0xff, 0x4a, 0x81, 0x84, 0x6d, 0x1a, 0xfd, 0x80, 0x02, 0x63, + 0x9b, 0x44, 0xf3, 0xda, 0x0e, 0xb9, 0x2e, 0x7c, 0x42, 0xfb, 0xae, 0x8e, 0x3e, 0xfc, 0xec, 0x51, + 0x98, 0xc4, 0xe7, 0x96, 0x24, 0xc4, 0xdc, 0x59, 0x25, 0x48, 0x38, 0x21, 0x83, 0x70, 0xa4, 0x07, + 0xb3, 0x4f, 0xc1, 0x54, 0xa2, 0xe1, 0xa1, 0xae, 0x29, 0x7f, 0x55, 0x81, 0xb4, 0xdc, 0xe6, 0xe8, + 0x25, 0x18, 0xd0, 0xea, 0xf5, 0x20, 0xc5, 0xe5, 0x63, 0xf9, 0xfc, 0xa6, 0xea, 0x72, 0xa4, 0x20, + 0xf6, 0x13, 0x73, 0xb4, 0x68, 0x09, 0x90, 0x16, 0xb9, 0x97, 0x5e, 0x09, 0x43, 0x54, 0xb0, 0xeb, + 0xb4, 0xf9, 0x04, 0x14, 0xa7, 0xb4, 0x50, 0x3f, 0xae, 0x00, 0x4a, 0xa6, 0x28, 0x41, 0x0e, 0x0c, + 0x8b, 0xfd, 0xeb, 0xaf, 0x52, 0x39, 0xe7, 0x6b, 0xba, 0xc8, 0xd3, 0xd0, 0xd0, 0x09, 0x4f, 0x14, + 0xb8, 0x38, 0xa0, 0xa3, 0xfe, 0x9d, 0x02, 0x61, 0x1a, 0x3c, 0xf4, 0x4e, 0x18, 0xad, 0x13, 0x57, + 0x77, 0x8c, 0x96, 0x17, 0x3e, 0x24, 0x0d, 0x1e, 0xa4, 0x95, 0x43, 0x10, 0x96, 0xeb, 0x21, 0x15, + 0x06, 0x3d, 0xcd, 0xbd, 0x5d, 0x29, 0x0b, 0x65, 0x8f, 0x1d, 0xcd, 0x6b, 0xac, 0x04, 0x0b, 0x48, + 0x18, 0x3a, 0xb6, 0xaf, 0x8b, 0xd0, 0xb1, 0x68, 0xf3, 0x08, 0xe2, 0xe4, 0xa2, 0x83, 0x63, 0xe4, + 0xaa, 0x3f, 0x55, 0x80, 0x53, 0xb4, 0xca, 0x8a, 0x66, 0x58, 0x1e, 0xb1, 0xd8, 0xb3, 0xa9, 0x9c, + 0x93, 0xd0, 0x80, 0x71, 0x2f, 0xf2, 0x68, 0xf9, 0xf0, 0x8f, 0x6a, 0x03, 0x4f, 0xaf, 0xe8, 0x53, + 0xe5, 0x28, 0x5e, 0xf4, 0x98, 0xff, 0x6e, 0x8d, 0xab, 0xc5, 0xf7, 0xf9, 0x5b, 0x95, 0x3d, 0x46, + 0xbb, 0x27, 0x5e, 0x80, 0x07, 0xb9, 0x13, 0x23, 0x4f, 0xd4, 0x1e, 0x85, 0x71, 0xf1, 0x42, 0x82, + 0xc7, 0x00, 0x16, 0x6a, 0x31, 0x3b, 0x56, 0x96, 0x64, 0x00, 0x8e, 0xd6, 0x53, 0xff, 0xb0, 0x00, + 0xd1, 0x0c, 0x8d, 0x79, 0x67, 0x29, 0x19, 0x00, 0xb9, 0x70, 0x6c, 0x01, 0x90, 0x79, 0x8e, 0x72, + 0xe6, 0xce, 0x2d, 0xee, 0xd9, 0xe5, 0xfc, 0xc4, 0xac, 0x1c, 0x07, 0x35, 0xc2, 0x69, 0xed, 0x3f, + 0xf4, 0xb4, 0xbe, 0x53, 0x78, 0xfe, 0x0e, 0x44, 0xc2, 0x50, 0xfb, 0x9e, 0xbf, 0x53, 0x91, 0x86, + 0xd2, 0x2b, 0xbb, 0x79, 0x10, 0x39, 0x4e, 0xe8, 0xba, 0x88, 0x10, 0xd9, 0xee, 0x9a, 0xed, 0x69, + 0x26, 0x9b, 0x4e, 0x61, 0x74, 0x5d, 0x91, 0x01, 0x38, 0x5a, 0x4f, 0x5d, 0x85, 0x37, 0x2f, 0xdb, + 0x5a, 0x7d, 0x41, 0x33, 0xe9, 0xd6, 0x75, 0x84, 0x5b, 0x9e, 0xcb, 0x4e, 0xe6, 0xaa, 0x63, 0x7b, + 0xb6, 0x6e, 0x9b, 0xf4, 0xdc, 0xd4, 0x82, 0x58, 0xc1, 0xec, 0x22, 0xd2, 0x3f, 0x37, 0x45, 0xac, + 0x5d, 0xec, 0xc3, 0xd5, 0xdf, 0x56, 0x60, 0x48, 0xe4, 0x3d, 0xe9, 0xe2, 0x61, 0xe9, 0x26, 0x0c, + 0x30, 0xed, 0xa8, 0x17, 0xa9, 0xb4, 0xb6, 0x65, 0xdb, 0x5e, 0x24, 0x61, 0x12, 0x4f, 0x39, 0xc6, + 0x92, 0x44, 0x72, 0xf4, 0xcc, 0x1f, 0xd5, 0xd1, 0xb7, 0x0c, 0x8f, 0x30, 0xb7, 0x1b, 0xb1, 0xf1, + 0xb9, 0x3f, 0xaa, 0x54, 0x8e, 0x23, 0xb5, 0xd4, 0xff, 0x36, 0x00, 0x57, 0xfc, 0x1c, 0x2e, 0x71, + 0x51, 0x2d, 0xe0, 0xb9, 0xbb, 0x41, 0x52, 0x19, 0x16, 0x14, 0xdf, 0xb7, 0xa9, 0xe6, 0xd3, 0x92, + 0xcf, 0x49, 0x69, 0x63, 0x64, 0x74, 0x38, 0x8d, 0x06, 0x8f, 0xbe, 0xce, 0x8a, 0x6f, 0x10, 0xcd, + 0xf4, 0xb6, 0x7c, 0xda, 0x85, 0x5e, 0xa2, 0xaf, 0x27, 0xf1, 0xe1, 0x54, 0x2a, 0xcc, 0xa5, 0xc3, + 0x4f, 0x66, 0xe3, 0x10, 0x4d, 0xf6, 0x27, 0xe9, 0xe1, 0xb9, 0xd1, 0x4a, 0x2a, 0x46, 0x9c, 0x41, + 0x89, 0x99, 0x1b, 0xb5, 0x1d, 0x66, 0xbd, 0xc0, 0xc4, 0x73, 0x0c, 0x96, 0xf8, 0x2a, 0xb8, 0xab, + 0x58, 0x89, 0x82, 0x70, 0xbc, 0x2e, 0x7a, 0x1c, 0x26, 0x98, 0x8b, 0x4c, 0x18, 0xce, 0x73, 0x20, + 0x0c, 0xea, 0xb4, 0x1a, 0x81, 0xe0, 0x58, 0x4d, 0xf4, 0x5d, 0x0a, 0x4c, 0x1b, 0xf2, 0x63, 0x1a, + 0x7f, 0xf4, 0xf9, 0x32, 0xa3, 0xb0, 0xeb, 0x25, 0x7f, 0x1b, 0xa7, 0xa0, 0xc5, 0xa9, 0xc4, 0xd0, + 0x32, 0x4c, 0x8b, 0x17, 0xd1, 0xd1, 0x3d, 0xc0, 0x63, 0xbd, 0xb1, 0x35, 0x2d, 0xa7, 0xc0, 0x71, + 0x6a, 0x2b, 0xf5, 0x23, 0x05, 0x18, 0x3b, 0x64, 0x12, 0xd3, 0xb6, 0x24, 0x73, 0xf4, 0xf0, 0x6e, + 0x51, 0xa6, 0xda, 0x85, 0xd8, 0x81, 0x9e, 0x87, 0x89, 0x36, 0x9b, 0x08, 0x3f, 0x12, 0x9a, 0xf8, + 0xa6, 0xbf, 0x85, 0xae, 0xdc, 0x7a, 0x04, 0x72, 0x6f, 0xaf, 0x38, 0x2b, 0xa3, 0x8f, 0x42, 0x71, + 0x0c, 0x8f, 0xfa, 0xeb, 0x05, 0x40, 0x72, 0xf5, 0x25, 0x53, 0xdb, 0xb6, 0x1d, 0xf4, 0x79, 0x05, + 0xc6, 0x74, 0x39, 0x6d, 0x14, 0x17, 0xb0, 0x9e, 0xef, 0x75, 0xb0, 0x1c, 0xfd, 0x9c, 0x9c, 0x0f, + 0x8a, 0x0b, 0xc2, 0xfe, 0x73, 0xeb, 0x31, 0x19, 0x74, 0x2f, 0xf6, 0x1b, 0x47, 0xfa, 0x34, 0xfb, + 0x31, 0x05, 0xa6, 0x12, 0x98, 0x52, 0x24, 0xe3, 0x17, 0xa2, 0x31, 0xd0, 0x8f, 0x24, 0x41, 0xa4, + 0x2c, 0x5f, 0xff, 0xf8, 0x00, 0x9c, 0x4e, 0x59, 0x51, 0xe6, 0x62, 0x43, 0x62, 0xd2, 0x65, 0x2f, + 0x2e, 0x36, 0x09, 0x49, 0x35, 0x70, 0xb1, 0x89, 0x43, 0x70, 0x82, 0x2e, 0x7a, 0x16, 0xfa, 0x74, + 0xc7, 0x10, 0x9b, 0xf6, 0xd1, 0x5c, 0x53, 0x80, 0x2b, 0x0b, 0xa3, 0x82, 0x62, 0x5f, 0x09, 0x57, + 0x30, 0x45, 0x48, 0xcf, 0x62, 0xf9, 0x18, 0xf1, 0x05, 0x56, 0x76, 0x16, 0xcb, 0xa7, 0x8d, 0x8b, + 0xa3, 0xf5, 0xd0, 0xf3, 0x30, 0x23, 0x34, 0x55, 0x3f, 0x92, 0x89, 0x6d, 0xb9, 0x1e, 0xe5, 0xf8, + 0x9e, 0x90, 0x29, 0x98, 0xf7, 0xeb, 0xcd, 0x8c, 0x3a, 0x38, 0xb3, 0x35, 0xfa, 0x4e, 0x98, 0x88, + 0x70, 0x0f, 0x3f, 0x2a, 0x59, 0xce, 0x27, 0x33, 0x32, 0x26, 0xce, 0x2b, 0xa3, 0x65, 0x38, 0x46, + 0x0d, 0xfd, 0xbf, 0x0a, 0x4c, 0x05, 0x1b, 0x75, 0x97, 0x6f, 0x79, 0x3f, 0x83, 0xdf, 0xd2, 0xd1, + 0x7c, 0x41, 0x52, 0x7a, 0xd5, 0x38, 0x21, 0x9c, 0xa4, 0xad, 0xfe, 0xf0, 0x20, 0xc8, 0x19, 0x9d, + 0xd1, 0x4a, 0x2f, 0x76, 0xc9, 0x70, 0x0f, 0xf8, 0xb6, 0xc9, 0x15, 0xe8, 0x6b, 0xb4, 0xda, 0x39, + 0x0d, 0x93, 0x01, 0xba, 0xeb, 0x14, 0x5d, 0xa3, 0xd5, 0x46, 0xcf, 0x06, 0xa6, 0xce, 0x7c, 0xc6, + 0xc8, 0xe0, 0xa9, 0x64, 0xcc, 0xdc, 0xe9, 0xb3, 0xf7, 0xfe, 0x4c, 0xf6, 0xde, 0x84, 0x21, 0x57, + 0xd8, 0x41, 0x07, 0xf2, 0x87, 0x91, 0x94, 0x66, 0x5a, 0xd8, 0x3d, 0xb9, 0x85, 0xc6, 0x37, 0x8b, + 0xfa, 0x34, 0xa8, 0x22, 0xd8, 0x66, 0xe7, 0x92, 0x08, 0x92, 0xc4, 0x14, 0xc1, 0x75, 0x56, 0x82, + 0x05, 0x24, 0x21, 0xcc, 0x0d, 0x75, 0x23, 0xcc, 0xa1, 0xcf, 0xc5, 0xf9, 0xf7, 0x30, 0xdb, 0x7d, + 0xcf, 0xf4, 0x38, 0x9c, 0xff, 0xcb, 0x18, 0xf7, 0xff, 0x13, 0x1e, 0x7e, 0xd2, 0x62, 0xa1, 0xfb, + 0x60, 0x80, 0x45, 0x51, 0x12, 0x72, 0x40, 0x60, 0xdc, 0x60, 0x71, 0x74, 0x30, 0x87, 0xa1, 0x9a, + 0x08, 0x1d, 0x98, 0x6f, 0xd3, 0x33, 0xdf, 0x46, 0x41, 0x4f, 0x8a, 0x33, 0x78, 0x25, 0xf2, 0x26, + 0x32, 0x4d, 0x87, 0x58, 0x87, 0xa1, 0xa6, 0x61, 0x31, 0x77, 0x8f, 0x7c, 0x46, 0x74, 0xee, 0x82, + 0xc5, 0x51, 0x60, 0x1f, 0x97, 0xfa, 0xe7, 0x05, 0xca, 0x20, 0x42, 0xa5, 0x7e, 0x97, 0x07, 0xc2, + 0x15, 0xaf, 0xaf, 0x95, 0xfc, 0x46, 0x40, 0x09, 0xe9, 0x7c, 0x80, 0x90, 0x3b, 0x2a, 0x84, 0xbf, + 0xb1, 0x44, 0x8c, 0x92, 0xf6, 0x8c, 0x26, 0x79, 0xce, 0xb0, 0xea, 0xf6, 0x1d, 0x31, 0xbd, 0xbd, + 0x92, 0x5e, 0x0b, 0x10, 0x72, 0xd2, 0xe1, 0x6f, 0x2c, 0x11, 0xa3, 0x47, 0x12, 0x4b, 0xc6, 0x61, + 0xb1, 0x4c, 0xbc, 0xa2, 0x6f, 0x3c, 0xcf, 0xb6, 0xf0, 0x3b, 0x66, 0x47, 0x52, 0x29, 0xa3, 0x0e, + 0xce, 0x6c, 0xad, 0xfe, 0x8c, 0x02, 0x67, 0x52, 0xa7, 0x02, 0x5d, 0x87, 0xa9, 0xd0, 0x1d, 0x56, + 0x16, 0x12, 0x86, 0x43, 0x1e, 0x7f, 0x33, 0x5e, 0x01, 0x27, 0xdb, 0xf0, 0xfc, 0x66, 0x09, 0x21, + 0x44, 0xf8, 0xd2, 0xca, 0xaa, 0x96, 0x0c, 0xc6, 0x69, 0x6d, 0xd4, 0x17, 0x23, 0x9d, 0x0d, 0x27, + 0x8b, 0x7e, 0x19, 0x1b, 0xa4, 0x11, 0xbc, 0x49, 0x0f, 0xbe, 0x8c, 0x05, 0x5a, 0x88, 0x39, 0x0c, + 0x5d, 0x92, 0x43, 0x65, 0x04, 0xdc, 0xdd, 0x0f, 0x97, 0xa1, 0xfe, 0xaa, 0x02, 0xe7, 0x56, 0x34, + 0xab, 0xad, 0x99, 0xfc, 0xf1, 0x7b, 0xd5, 0xb6, 0x4d, 0x31, 0x4d, 0x9d, 0xf2, 0xad, 0x2b, 0xaf, + 0x97, 0x7c, 0xeb, 0xef, 0x87, 0x73, 0x19, 0xfe, 0x37, 0xa8, 0x0c, 0x63, 0xee, 0x1d, 0xad, 0xb5, + 0x40, 0xb6, 0xb4, 0x6d, 0x43, 0x04, 0xd6, 0xe2, 0x6e, 0xda, 0x63, 0x35, 0xa9, 0xfc, 0x5e, 0xec, + 0x37, 0x8e, 0xb4, 0x52, 0x3d, 0x00, 0xe1, 0xce, 0x6f, 0x58, 0x0d, 0xb4, 0x09, 0xc3, 0x9a, 0x49, + 0x1c, 0x2f, 0x0c, 0x1e, 0xf9, 0xad, 0xb9, 0xec, 0xb4, 0x02, 0x07, 0x7f, 0xbe, 0xe6, 0xff, 0xc2, + 0x01, 0x6e, 0xf5, 0x9f, 0x28, 0x70, 0x36, 0x3d, 0x94, 0x52, 0x17, 0x6a, 0x51, 0x13, 0x46, 0x9d, + 0xb0, 0x99, 0xf8, 0x68, 0xdf, 0x25, 0x67, 0xa8, 0x90, 0x1e, 0xde, 0x50, 0x45, 0xb0, 0xe4, 0xd8, + 0xae, 0xbf, 0x73, 0xe3, 0x49, 0x2b, 0x02, 0xab, 0x98, 0xd4, 0x13, 0x2c, 0xe3, 0x67, 0x09, 0x64, + 0x28, 0x75, 0xb7, 0xa5, 0xe9, 0xa4, 0x7e, 0xc2, 0x39, 0xd5, 0x8f, 0x20, 0x6b, 0x43, 0x7a, 0xdf, + 0x8f, 0x37, 0x81, 0x4c, 0x06, 0xcd, 0x83, 0x13, 0xc8, 0xa4, 0x37, 0x7c, 0x83, 0x64, 0x36, 0x48, + 0xef, 0x7c, 0xc6, 0xc3, 0xf7, 0xff, 0x32, 0x98, 0x35, 0xda, 0x43, 0x26, 0x66, 0xdf, 0x3e, 0xc6, + 0xc4, 0xec, 0x13, 0xdf, 0x48, 0xca, 0x9e, 0x92, 0x94, 0x3d, 0x96, 0x28, 0x7c, 0xf0, 0x84, 0x12, + 0x85, 0xbf, 0x02, 0x83, 0x2d, 0xcd, 0x21, 0x96, 0x7f, 0x97, 0x5b, 0xc9, 0xe7, 0x68, 0x10, 0x6e, + 0xb4, 0x90, 0x0b, 0x06, 0x9f, 0x64, 0x95, 0x11, 0xc0, 0x82, 0x50, 0x4a, 0xf0, 0x94, 0xe1, 0xe3, + 0x8a, 0x4e, 0x18, 0xa6, 0x28, 0x1f, 0x39, 0x8e, 0x14, 0xe5, 0xea, 0xdf, 0x2a, 0x70, 0xb1, 0x13, + 0x5b, 0x62, 0x06, 0x14, 0x3d, 0xf6, 0x19, 0xf6, 0x62, 0x40, 0x49, 0x70, 0xdb, 0xc0, 0x80, 0x12, + 0x87, 0xe0, 0x04, 0x5d, 0xf4, 0x5e, 0x40, 0xf6, 0x06, 0xf7, 0xeb, 0xb9, 0x4e, 0x69, 0xf0, 0xd7, + 0xb4, 0x05, 0xf6, 0x56, 0x21, 0xc8, 0x6b, 0x79, 0x2b, 0x51, 0x03, 0xa7, 0xb4, 0x52, 0x7f, 0xb9, + 0x00, 0xb0, 0x4a, 0xbc, 0x3b, 0xb6, 0x73, 0x9b, 0x9e, 0xf1, 0x17, 0x23, 0x57, 0x07, 0xc3, 0x5f, + 0xbf, 0x78, 0x94, 0x17, 0xa1, 0xbf, 0x65, 0xd7, 0x5d, 0xa1, 0x7f, 0xb0, 0x8e, 0xb0, 0xa7, 0x1a, + 0xac, 0x14, 0x15, 0x61, 0x80, 0x39, 0x3d, 0x09, 0x05, 0x9a, 0x5d, 0x3c, 0xb0, 0xcb, 0x15, 0xcc, + 0xcb, 0x29, 0x87, 0x14, 0x31, 0x0d, 0x5c, 0x71, 0xb9, 0x33, 0xc6, 0x63, 0x88, 0xf3, 0x32, 0x1c, + 0x40, 0xd1, 0xe3, 0x00, 0x46, 0x6b, 0x49, 0x6b, 0x1a, 0xa6, 0x21, 0x3e, 0xd7, 0x11, 0x66, 0x11, + 0x87, 0x4a, 0xd5, 0x2f, 0xbd, 0xb7, 0x57, 0x1c, 0x16, 0xbf, 0x76, 0xb1, 0x54, 0x5b, 0xfd, 0xa2, + 0x02, 0x93, 0xe1, 0xe4, 0x89, 0xad, 0xe2, 0xf7, 0x9c, 0x07, 0x03, 0xce, 0xec, 0x39, 0x0f, 0xdb, + 0xdf, 0xb9, 0xe7, 0xdc, 0x80, 0x95, 0xd5, 0xf3, 0x87, 0x60, 0x94, 0xf0, 0x90, 0x47, 0x95, 0x32, + 0xe6, 0x3c, 0x6e, 0x84, 0xab, 0x73, 0x8b, 0x61, 0x31, 0x96, 0xeb, 0xa8, 0x7f, 0xdf, 0x07, 0x63, + 0xab, 0x0d, 0xc3, 0xda, 0xf1, 0x63, 0x3b, 0x05, 0x17, 0xef, 0xca, 0xf1, 0x5c, 0xbc, 0x3f, 0x0f, + 0x33, 0xa6, 0x7c, 0xcd, 0xc5, 0x05, 0x27, 0xcd, 0x6a, 0x04, 0x33, 0xc0, 0xf4, 0x98, 0xe5, 0x8c, + 0x3a, 0x38, 0xb3, 0x35, 0xf2, 0x60, 0x50, 0xf7, 0xf3, 0x2b, 0xe6, 0x8e, 0x57, 0x24, 0xcf, 0xc5, + 0x9c, 0x1c, 0xba, 0x23, 0xe0, 0x79, 0x62, 0x7b, 0x0a, 0x5a, 0xe8, 0xa3, 0x0a, 0x9c, 0x21, 0x3b, + 0x3c, 0x74, 0xcd, 0x9a, 0xa3, 0x6d, 0x6e, 0x1a, 0xba, 0x78, 0xf1, 0xc7, 0x77, 0xe2, 0xf2, 0xfe, + 0x5e, 0xf1, 0xcc, 0x62, 0x5a, 0x85, 0x7b, 0x7b, 0xc5, 0x6b, 0xa9, 0x91, 0x84, 0xd8, 0x6a, 0xa6, + 0x36, 0xc1, 0xe9, 0xa4, 0x66, 0x1f, 0x83, 0xd1, 0x43, 0xbc, 0x13, 0x8f, 0xc4, 0x0b, 0xfa, 0x95, + 0x02, 0x8c, 0xd1, 0xed, 0xb6, 0x6c, 0xeb, 0x9a, 0x59, 0x5e, 0xad, 0xa1, 0x07, 0xe2, 0x61, 0x12, + 0x83, 0x2b, 0xc6, 0x44, 0xa8, 0xc4, 0x65, 0x98, 0xde, 0xb4, 0x1d, 0x9d, 0xac, 0x95, 0xaa, 0x6b, + 0xb6, 0x70, 0x3e, 0x2b, 0xaf, 0xd6, 0x84, 0x5e, 0xc7, 0xae, 0x3c, 0x96, 0x52, 0xe0, 0x38, 0xb5, + 0x15, 0xba, 0x05, 0x67, 0xc2, 0xf2, 0xf5, 0x16, 0x7f, 0xb0, 0x40, 0xd1, 0xf5, 0x85, 0x0f, 0x2e, + 0x96, 0xd2, 0x2a, 0xe0, 0xf4, 0x76, 0x48, 0x83, 0x0b, 0xe2, 0x6e, 0x65, 0xc9, 0x76, 0xee, 0x68, + 0x4e, 0x3d, 0x8a, 0xb6, 0x3f, 0xcc, 0x55, 0x5e, 0xce, 0xae, 0x86, 0x3b, 0xe1, 0x50, 0xff, 0x4e, + 0x81, 0x68, 0x10, 0x4a, 0x74, 0x1e, 0xfa, 0x1c, 0x91, 0xbe, 0x4f, 0x04, 0x63, 0xa4, 0x2a, 0x02, + 0x2d, 0x43, 0x73, 0x00, 0x4e, 0x18, 0x09, 0xb3, 0x10, 0xa6, 0xb5, 0x90, 0x62, 0x58, 0x4a, 0x35, + 0x28, 0x2a, 0x4f, 0x6b, 0x08, 0x86, 0xc7, 0x50, 0xad, 0x69, 0x0d, 0x4c, 0xcb, 0x58, 0xfe, 0x12, + 0xa3, 0x41, 0x5c, 0xdf, 0x1c, 0xcd, 0xf3, 0x97, 0xb0, 0x12, 0x2c, 0x20, 0x48, 0x83, 0xf1, 0x56, + 0xdb, 0x14, 0xf1, 0x94, 0xa8, 0xea, 0xc3, 0xcd, 0x86, 0x57, 0xd3, 0x92, 0xf3, 0xb1, 0xd5, 0x4f, + 0xcd, 0xd0, 0x57, 0x95, 0x51, 0xe0, 0x28, 0x46, 0xf5, 0xc7, 0x06, 0x41, 0x0a, 0xaf, 0x73, 0x08, + 0x29, 0xf4, 0x27, 0x15, 0x98, 0xd6, 0x4d, 0x83, 0x58, 0x5e, 0x2c, 0x52, 0x05, 0x3f, 0x3e, 0xd6, + 0x73, 0xc5, 0xfd, 0x69, 0x11, 0xab, 0x52, 0x16, 0xcf, 0x5b, 0x4a, 0x29, 0xc8, 0xc5, 0x13, 0xa0, + 0x14, 0x08, 0x4e, 0xed, 0x0c, 0x1b, 0x0f, 0x2b, 0xaf, 0x94, 0xe5, 0xe8, 0x99, 0x25, 0x51, 0x86, + 0x03, 0x28, 0xe5, 0xbc, 0x0d, 0xc7, 0x6e, 0xb7, 0xdc, 0x12, 0x7b, 0xc5, 0xca, 0x17, 0x85, 0x71, + 0xde, 0xeb, 0x61, 0x31, 0x96, 0xeb, 0xa0, 0x47, 0x60, 0x8c, 0xff, 0xac, 0x3a, 0x64, 0xd3, 0xd8, + 0x11, 0x87, 0x12, 0x33, 0x9e, 0x5e, 0x97, 0xca, 0x71, 0xa4, 0x16, 0x8b, 0xdf, 0xe6, 0xba, 0x6d, + 0xe2, 0xac, 0xe3, 0x65, 0x91, 0xe3, 0x99, 0xc7, 0x6f, 0xf3, 0x0b, 0x71, 0x08, 0x47, 0x3f, 0xa4, + 0xc0, 0x84, 0x43, 0x5e, 0x69, 0x1b, 0x0e, 0x15, 0x61, 0x34, 0xa3, 0xe9, 0x8a, 0x18, 0x47, 0xb8, + 0xb7, 0xb8, 0x4a, 0x73, 0x38, 0x82, 0x94, 0x33, 0xc8, 0xc0, 0x95, 0x23, 0x0a, 0xc4, 0xb1, 0x1e, + 0xd0, 0xa9, 0x72, 0x8d, 0x86, 0x65, 0x58, 0x8d, 0x79, 0xb3, 0xe1, 0x8a, 0x44, 0x56, 0xdc, 0xe6, + 0x18, 0x16, 0x63, 0xb9, 0x0e, 0x7a, 0x14, 0xc6, 0xdb, 0x2e, 0x65, 0x7b, 0x4d, 0xc2, 0xe7, 0x77, + 0x24, 0xf4, 0x75, 0x59, 0x97, 0x01, 0x38, 0x5a, 0x0f, 0x3d, 0x0e, 0x13, 0x7e, 0x81, 0x98, 0x65, + 0xe0, 0xd9, 0x54, 0xd8, 0xdd, 0x64, 0x04, 0x82, 0x63, 0x35, 0x67, 0xe7, 0xe1, 0x74, 0xca, 0x30, + 0x0f, 0xc5, 0x5b, 0xff, 0xb7, 0x02, 0x67, 0xb8, 0xd4, 0xe5, 0x67, 0x87, 0xf6, 0x73, 0x5b, 0xa4, + 0xa7, 0x89, 0x50, 0x8e, 0x35, 0x4d, 0xc4, 0xd7, 0x21, 0x1d, 0x86, 0xfa, 0xd3, 0x05, 0x78, 0xf3, + 0x81, 0xdf, 0x25, 0xfa, 0x47, 0x0a, 0x8c, 0x92, 0x1d, 0xcf, 0xd1, 0x82, 0xa7, 0xfe, 0x74, 0x93, + 0x6e, 0x1e, 0x0b, 0x13, 0x98, 0x5b, 0x0c, 0x09, 0xf1, 0x8d, 0x1b, 0xa8, 0x52, 0x12, 0x04, 0xcb, + 0xfd, 0xa1, 0xdc, 0x96, 0x67, 0x1b, 0x92, 0x9d, 0xe2, 0x04, 0x17, 0x14, 0x90, 0xd9, 0x27, 0x61, + 0x32, 0x8e, 0xf9, 0x50, 0x7b, 0xe5, 0x0b, 0x0a, 0xa4, 0x86, 0xe3, 0x44, 0x25, 0x98, 0xd2, 0xda, + 0x9e, 0x1d, 0xb9, 0xd7, 0x13, 0xa2, 0x24, 0x73, 0xfa, 0x9e, 0x8f, 0x03, 0x71, 0xb2, 0x3e, 0x37, + 0xc0, 0x5a, 0x6d, 0xcd, 0x8c, 0xa2, 0xe1, 0x02, 0x97, 0x30, 0xc0, 0x26, 0xc0, 0x38, 0xad, 0x8d, + 0xfa, 0xcf, 0x14, 0x38, 0x93, 0x6a, 0x96, 0xec, 0xc2, 0x16, 0x97, 0xbe, 0xed, 0x0b, 0xc7, 0xb9, + 0xed, 0xd5, 0x5f, 0x2a, 0xc0, 0x50, 0xd5, 0xb1, 0xe9, 0xd1, 0x76, 0x02, 0x16, 0x38, 0x2d, 0x62, + 0x81, 0xcb, 0x65, 0x5f, 0x10, 0x9d, 0xcd, 0x34, 0xb9, 0x19, 0x31, 0x93, 0xdb, 0x7c, 0x2f, 0x44, + 0x3a, 0xdb, 0xd8, 0x7e, 0x4f, 0x81, 0x51, 0x51, 0xf3, 0x04, 0x8c, 0x6a, 0xdf, 0x11, 0x35, 0xaa, + 0xbd, 0xa7, 0x87, 0x71, 0x65, 0x58, 0xd1, 0x3e, 0xab, 0xc0, 0xb8, 0xa8, 0xb1, 0x42, 0x9a, 0x1b, + 0xc4, 0x41, 0x4b, 0x30, 0xe4, 0xb6, 0xd9, 0x42, 0x8a, 0x01, 0x5d, 0x90, 0xc5, 0x23, 0x67, 0x43, + 0xd3, 0x69, 0xf7, 0x6b, 0xbc, 0x8a, 0x94, 0x09, 0x99, 0x17, 0x60, 0xbf, 0x31, 0xdd, 0xfb, 0x8e, + 0x6d, 0x26, 0xc2, 0xd6, 0x63, 0xdb, 0x24, 0x98, 0x41, 0xa8, 0x9e, 0x47, 0xff, 0xfa, 0x3a, 0x1c, + 0xd3, 0xf3, 0x28, 0xd8, 0xc5, 0xbc, 0x5c, 0xfd, 0xe2, 0x40, 0x30, 0xd9, 0x4c, 0xa9, 0xbf, 0x01, + 0x23, 0xba, 0x43, 0x34, 0x8f, 0xd4, 0x17, 0x76, 0xbb, 0xe9, 0x1c, 0x93, 0x03, 0x4a, 0x7e, 0x0b, + 0x1c, 0x36, 0xa6, 0x47, 0xae, 0xec, 0xe0, 0x59, 0x08, 0xa5, 0x93, 0x4c, 0xe7, 0xce, 0x6f, 0x85, + 0x01, 0xfb, 0x8e, 0x15, 0x3c, 0x0e, 0xe9, 0x48, 0x98, 0x0d, 0xe5, 0x16, 0xad, 0x8d, 0x79, 0x23, + 0x39, 0x6d, 0x43, 0x7f, 0x87, 0xb4, 0x0d, 0x26, 0x0c, 0x35, 0xd9, 0x32, 0xf4, 0x94, 0x18, 0x37, + 0xb2, 0xa0, 0xe1, 0x12, 0xf1, 0xdf, 0x2e, 0xf6, 0x49, 0x50, 0xd1, 0xc9, 0xf2, 0x2d, 0x3a, 0xb2, + 0xe8, 0x14, 0x98, 0x79, 0x70, 0x08, 0x47, 0xbb, 0xd1, 0x7c, 0x20, 0x43, 0xf9, 0xed, 0xa4, 0xa2, + 0x7b, 0x52, 0x0a, 0x10, 0x3e, 0xf5, 0x59, 0x39, 0x41, 0xd0, 0x17, 0x14, 0x38, 0x57, 0x4f, 0x4f, + 0xb8, 0x26, 0xae, 0xca, 0x73, 0x3d, 0xcc, 0xce, 0xc8, 0xe1, 0xb6, 0x50, 0x14, 0x13, 0x96, 0x95, + 0xe4, 0x0d, 0x67, 0x75, 0x46, 0xfd, 0xde, 0xfe, 0xe0, 0x6b, 0x12, 0x96, 0x8e, 0x74, 0x3b, 0x94, + 0x92, 0xc7, 0x0e, 0x85, 0xde, 0xe1, 0xa7, 0xff, 0xe2, 0xdb, 0xf5, 0x52, 0x3c, 0xfd, 0xd7, 0x98, + 0x20, 0x1d, 0x49, 0xf9, 0xd5, 0x86, 0xd3, 0xae, 0xa7, 0x99, 0xa4, 0x66, 0x88, 0xcb, 0x41, 0xd7, + 0xd3, 0x9a, 0xad, 0x1c, 0xf9, 0xb7, 0x78, 0xa0, 0x86, 0x24, 0x2a, 0x9c, 0x86, 0x1f, 0x7d, 0x37, + 0x0b, 0xbe, 0xa7, 0x99, 0xec, 0xf2, 0x94, 0xe7, 0x20, 0x0d, 0x89, 0x1f, 0xde, 0xdd, 0x5d, 0x84, + 0xd6, 0x4b, 0xc7, 0x87, 0x33, 0x29, 0xa1, 0x57, 0xe1, 0x0c, 0x3d, 0x00, 0xe7, 0x75, 0xcf, 0xd8, + 0x36, 0xbc, 0xdd, 0xb0, 0x0b, 0x87, 0x4f, 0xba, 0xc5, 0xb4, 0xed, 0xe5, 0x34, 0x64, 0x38, 0x9d, + 0x86, 0xfa, 0x37, 0x0a, 0xa0, 0xe4, 0x5e, 0x47, 0x26, 0x0c, 0xd7, 0xfd, 0xc8, 0x09, 0xca, 0x91, + 0x64, 0xd5, 0x09, 0x8e, 0x90, 0x20, 0xe0, 0x42, 0x40, 0x01, 0xd9, 0x30, 0x72, 0x67, 0xcb, 0xf0, + 0x88, 0x69, 0xb8, 0xde, 0x11, 0x25, 0xf1, 0x09, 0x62, 0xc2, 0x3f, 0xe7, 0x23, 0xc6, 0x21, 0x0d, + 0xf5, 0x53, 0xfd, 0x30, 0x1c, 0x24, 0xd3, 0x3c, 0xd8, 0xcd, 0xba, 0x0d, 0x48, 0x97, 0x02, 0x02, + 0xf6, 0x62, 0x33, 0x65, 0x32, 0x50, 0x29, 0x81, 0x0c, 0xa7, 0x10, 0x40, 0xaf, 0xc2, 0xb4, 0x61, + 0x6d, 0x3a, 0x5a, 0x10, 0xee, 0xb0, 0xe4, 0x1b, 0xca, 0x72, 0x10, 0x66, 0x5a, 0x74, 0x25, 0x05, + 0x1d, 0x4e, 0x25, 0x82, 0x08, 0x0c, 0xf1, 0x7c, 0xc6, 0xfe, 0xad, 0x4b, 0x2e, 0x5b, 0x3d, 0x97, + 0x35, 0x43, 0xf6, 0xee, 0xcb, 0x9e, 0x3e, 0x6e, 0x1e, 0x9c, 0x96, 0xff, 0xef, 0x5f, 0x48, 0x89, + 0x7d, 0x5f, 0xca, 0x4f, 0x2f, 0xbc, 0xdb, 0xe2, 0xc1, 0x69, 0xa3, 0x85, 0x38, 0x4e, 0x50, 0xfd, + 0x1d, 0x05, 0x06, 0x78, 0x0c, 0xb0, 0xe3, 0x17, 0x35, 0xdf, 0x1f, 0x11, 0x35, 0x9f, 0xc8, 0x33, + 0x48, 0xd6, 0xd5, 0xcc, 0x8c, 0xfc, 0xbf, 0xad, 0xc0, 0x08, 0xab, 0x71, 0x02, 0xb2, 0xdf, 0x4b, + 0x51, 0xd9, 0xef, 0xb1, 0xdc, 0xa3, 0xc9, 0x90, 0xfc, 0x7e, 0xa7, 0x4f, 0x8c, 0x85, 0x89, 0x56, + 0x15, 0x38, 0x2d, 0x1e, 0xc6, 0x2e, 0x1b, 0x9b, 0x84, 0x6e, 0xf1, 0xb2, 0xb6, 0xeb, 0x8a, 0xa7, + 0x1a, 0x3c, 0xe8, 0x4c, 0x12, 0x8c, 0xd3, 0xda, 0xa0, 0x5f, 0x51, 0xa8, 0x10, 0xe3, 0x39, 0x86, + 0xde, 0xd3, 0x65, 0x70, 0xd0, 0xb7, 0xb9, 0x15, 0x8e, 0x8c, 0xeb, 0xa6, 0xeb, 0xa1, 0x34, 0xc3, + 0x4a, 0xef, 0xed, 0x15, 0x8b, 0x29, 0x36, 0xe3, 0x30, 0x2b, 0xb5, 0xeb, 0x7d, 0xf4, 0x2f, 0x3a, + 0x56, 0x61, 0xda, 0x98, 0xdf, 0x63, 0x74, 0x03, 0x06, 0x5c, 0xdd, 0x6e, 0xf9, 0x4f, 0xab, 0xef, + 0x4b, 0xb3, 0x0d, 0xc6, 0xcd, 0x82, 0xc1, 0x04, 0xd7, 0x68, 0x4b, 0xcc, 0x11, 0xcc, 0xbe, 0x0c, + 0x63, 0x72, 0xcf, 0x53, 0x74, 0xdf, 0x72, 0xd4, 0x63, 0xee, 0x90, 0xce, 0x61, 0xb2, 0xae, 0xfc, + 0x47, 0x7d, 0x30, 0x88, 0x49, 0x43, 0xa4, 0x4c, 0x3b, 0x40, 0xe7, 0x34, 0xfc, 0xf4, 0xbf, 0x85, + 0xfc, 0xef, 0xf0, 0xe4, 0xa4, 0x38, 0x2f, 0xd8, 0x96, 0x34, 0x07, 0x72, 0x06, 0x60, 0x64, 0x05, + 0x89, 0xa4, 0xfa, 0xf2, 0x3b, 0xd4, 0xf2, 0x81, 0x75, 0x93, 0x3a, 0x0a, 0xfd, 0xa0, 0x02, 0x48, + 0xd3, 0x75, 0xe2, 0xba, 0x98, 0xb8, 0x74, 0xee, 0xb9, 0xb0, 0xca, 0xb9, 0x6c, 0xbe, 0xa8, 0xd8, + 0x71, 0x6c, 0xa1, 0xd8, 0x96, 0x00, 0xb9, 0x38, 0x85, 0x78, 0x2f, 0xe9, 0xac, 0x7e, 0x5f, 0x81, + 0xb1, 0x48, 0xb6, 0xb0, 0x66, 0x68, 0x4b, 0xcf, 0xef, 0xb2, 0xe3, 0xbf, 0xfe, 0xba, 0xd0, 0xa1, + 0x12, 0xb7, 0xcf, 0xdf, 0x0a, 0xd2, 0x5d, 0x1c, 0x4d, 0x62, 0x31, 0xf5, 0xd3, 0x0a, 0x9c, 0xf5, + 0x07, 0x14, 0x8d, 0x6b, 0x8e, 0xae, 0xc2, 0xb0, 0xd6, 0x32, 0x98, 0xa1, 0x57, 0x36, 0x95, 0xcf, + 0x57, 0x2b, 0xac, 0x0c, 0x07, 0xd0, 0x48, 0x8e, 0xe5, 0xc2, 0x81, 0x39, 0x96, 0xdf, 0x22, 0x65, + 0x8d, 0x1e, 0x08, 0x65, 0x97, 0x80, 0x30, 0x77, 0xe6, 0x54, 0xdf, 0x05, 0x23, 0xb5, 0xda, 0x0d, + 0xbe, 0xa4, 0x87, 0xb8, 0xf1, 0x51, 0x3f, 0xd1, 0x07, 0xe3, 0x22, 0x41, 0x83, 0xc1, 0x4c, 0x40, + 0x27, 0x70, 0xce, 0xad, 0xc1, 0x88, 0x1b, 0xdc, 0x61, 0x14, 0xb2, 0xf9, 0x54, 0x70, 0x2d, 0x11, + 0xcf, 0xb2, 0x17, 0x5e, 0x5f, 0x84, 0x88, 0xd0, 0x4d, 0x18, 0x7c, 0x85, 0xf2, 0x5c, 0xff, 0x5b, + 0xed, 0x8a, 0xf5, 0x05, 0x1f, 0x22, 0x63, 0xd7, 0x2e, 0x16, 0x28, 0x90, 0xcb, 0x9e, 0x27, 0x32, + 0x21, 0xb0, 0x97, 0x50, 0x9d, 0x91, 0x99, 0x0d, 0x72, 0xc6, 0x8f, 0x89, 0x57, 0x8e, 0xec, 0x17, + 0x0e, 0x08, 0xb1, 0x14, 0xa1, 0x91, 0x16, 0x6f, 0x90, 0x14, 0xa1, 0x91, 0x3e, 0x67, 0x1c, 0xd7, + 0x8f, 0xc1, 0x99, 0xd4, 0xc9, 0x38, 0x58, 0xc4, 0x56, 0xff, 0x79, 0x01, 0xfa, 0x6b, 0x84, 0xd4, + 0x4f, 0x60, 0x67, 0xbe, 0x14, 0x91, 0xc0, 0xbe, 0x35, 0x77, 0x92, 0xd2, 0x2c, 0x4b, 0xdf, 0x66, + 0xcc, 0xd2, 0xf7, 0x64, 0x6e, 0x0a, 0x9d, 0xcd, 0x7c, 0x7f, 0x55, 0x80, 0x21, 0x5a, 0xad, 0xbc, + 0x5a, 0x43, 0x4d, 0x69, 0x2b, 0x17, 0xf2, 0x8b, 0xcf, 0x02, 0xdd, 0x41, 0x9b, 0x98, 0x7e, 0x39, + 0x86, 0x48, 0xfc, 0x22, 0x06, 0x59, 0x39, 0x02, 0x72, 0x22, 0xf6, 0x04, 0x23, 0xea, 0xe7, 0x95, + 0xc1, 0x01, 0x21, 0x74, 0x47, 0xd2, 0x49, 0xfb, 0xf3, 0xfb, 0xee, 0xa6, 0x13, 0xed, 0xa0, 0x9e, + 0xaa, 0x9f, 0x50, 0xe0, 0x54, 0xac, 0x55, 0x17, 0x4a, 0xe3, 0xb1, 0x30, 0x40, 0xf5, 0x6b, 0x0a, + 0xfd, 0xc6, 0x52, 0x46, 0xd0, 0x45, 0x8f, 0xc2, 0x04, 0x60, 0x85, 0x8e, 0x09, 0xc0, 0x2e, 0x42, + 0x3f, 0x95, 0x8c, 0x64, 0x9f, 0x1d, 0x2a, 0x30, 0x61, 0x56, 0x8a, 0x74, 0x98, 0xd0, 0xa5, 0x14, + 0xee, 0x64, 0x53, 0xf0, 0xce, 0xae, 0x58, 0x71, 0x70, 0xdd, 0x58, 0x8a, 0xa0, 0xc0, 0x31, 0x94, + 0xea, 0x6f, 0x29, 0x30, 0x4c, 0x87, 0x79, 0x02, 0xcc, 0xf1, 0xdb, 0xa3, 0xcc, 0xf1, 0xdd, 0x79, + 0xf7, 0x54, 0x06, 0x4f, 0xfc, 0xeb, 0x02, 0xb0, 0x0c, 0xc6, 0xc2, 0xbf, 0x48, 0xf2, 0x1c, 0x52, + 0x32, 0x7c, 0x9e, 0xae, 0x08, 0xc7, 0xa3, 0x98, 0x51, 0x5a, 0x72, 0x3e, 0x7a, 0x5b, 0xc4, 0xb7, + 0x28, 0x22, 0x5c, 0xa4, 0xf8, 0x17, 0xdd, 0x85, 0x71, 0x77, 0xcb, 0xb6, 0xbd, 0x72, 0xf8, 0xf1, + 0xe4, 0xbe, 0x80, 0x60, 0x8f, 0xc5, 0xfd, 0xa1, 0xf0, 0xab, 0xdc, 0x9a, 0x8c, 0x1b, 0x47, 0x49, + 0xa1, 0x39, 0x80, 0x0d, 0xd3, 0xd6, 0x6f, 0x73, 0xd7, 0x26, 0xfe, 0x38, 0x98, 0x39, 0x4f, 0x2c, + 0x04, 0xa5, 0x58, 0xaa, 0xd1, 0x93, 0x17, 0xd7, 0xd7, 0x14, 0x3e, 0xd3, 0x87, 0xf8, 0x46, 0x4f, + 0xd0, 0x11, 0xee, 0xad, 0x30, 0xe8, 0x30, 0x1d, 0x41, 0xac, 0x58, 0xf0, 0xf1, 0x71, 0xcd, 0x01, + 0x0b, 0x28, 0xdd, 0x1e, 0x5c, 0xf1, 0xe9, 0x0f, 0x2f, 0x1c, 0x64, 0x75, 0x45, 0xfd, 0x25, 0x31, + 0xcc, 0x20, 0x09, 0x76, 0x0b, 0xc6, 0x99, 0x66, 0x11, 0xcb, 0xbe, 0xfd, 0x8e, 0x2e, 0xbf, 0x11, + 0xb9, 0x69, 0xe8, 0x96, 0x1b, 0x29, 0xc6, 0x51, 0x02, 0xe8, 0x51, 0x18, 0xf7, 0x47, 0xc7, 0xbd, + 0x63, 0x0b, 0xe1, 0x0b, 0xcd, 0xaa, 0x0c, 0xc0, 0xd1, 0x7a, 0xea, 0x67, 0x0a, 0x70, 0x89, 0xf7, + 0x9d, 0x59, 0x5e, 0xca, 0xa4, 0x45, 0xac, 0x3a, 0xb1, 0xf4, 0x5d, 0x26, 0x67, 0xd7, 0xed, 0x06, + 0x7a, 0x15, 0x06, 0xef, 0x10, 0x52, 0x0f, 0xae, 0x30, 0x9e, 0xcb, 0x9f, 0x43, 0x3c, 0x83, 0xc4, + 0x73, 0x0c, 0x3d, 0xbf, 0x0d, 0xe6, 0xff, 0x63, 0x41, 0x92, 0x12, 0x6f, 0x39, 0xf6, 0x46, 0x20, + 0x0e, 0x1e, 0x3d, 0xf1, 0x2a, 0x43, 0xcf, 0x89, 0xf3, 0xff, 0xb1, 0x20, 0xa9, 0x56, 0xe1, 0xbe, + 0x2e, 0x9a, 0x1e, 0x46, 0xec, 0x3f, 0x08, 0x23, 0x1f, 0xfd, 0x61, 0x30, 0xfe, 0x89, 0x02, 0xf7, + 0x4b, 0x28, 0x17, 0x77, 0xa8, 0x26, 0x52, 0xd2, 0x5a, 0x9a, 0x4e, 0x75, 0x7d, 0x16, 0xa3, 0xee, + 0x50, 0x59, 0x7b, 0x3f, 0xa1, 0xc0, 0x10, 0xf7, 0xc8, 0xf3, 0xd9, 0xef, 0x4b, 0x3d, 0x4e, 0x79, + 0x66, 0x97, 0xfc, 0x6c, 0x66, 0xfe, 0xd8, 0xf8, 0x6f, 0x17, 0xfb, 0xf4, 0xd5, 0x7f, 0x3d, 0x00, + 0xdf, 0xdc, 0x3d, 0x22, 0xf4, 0x35, 0x45, 0xce, 0x36, 0xce, 0x6d, 0xe4, 0xcd, 0xe3, 0xed, 0x7c, + 0x60, 0x0d, 0x12, 0x06, 0x86, 0xe7, 0x12, 0x09, 0xc9, 0x8f, 0xc8, 0xd0, 0x14, 0x0e, 0x0c, 0xfd, + 0x53, 0x05, 0xc6, 0xe8, 0xb1, 0x14, 0x30, 0x17, 0xbe, 0x4c, 0xad, 0x63, 0x1e, 0xe9, 0xaa, 0x44, + 0x32, 0x16, 0xd7, 0x4a, 0x06, 0xe1, 0x48, 0xdf, 0xd0, 0x7a, 0xf4, 0xfa, 0x8f, 0xab, 0x88, 0x97, + 0xd3, 0xe4, 0x92, 0xc3, 0xa4, 0xfb, 0x9f, 0x35, 0x61, 0x22, 0x3a, 0xf3, 0xc7, 0x69, 0x26, 0x9b, + 0x7d, 0x0a, 0xa6, 0x12, 0xa3, 0x3f, 0x94, 0x41, 0xe6, 0x67, 0x07, 0xa1, 0x28, 0x4d, 0x75, 0x5a, + 0x78, 0x1a, 0xf4, 0x39, 0x05, 0x46, 0x35, 0xcb, 0x12, 0x8e, 0x4d, 0xfe, 0xfe, 0xad, 0xf7, 0xb8, + 0xaa, 0x69, 0xa4, 0xe6, 0xe6, 0x43, 0x32, 0x31, 0xcf, 0x1d, 0x09, 0x82, 0xe5, 0xde, 0x74, 0xf0, + 0xce, 0x2d, 0x9c, 0x98, 0x77, 0x2e, 0xfa, 0xa0, 0x7f, 0x10, 0xf7, 0xe5, 0x0f, 0x54, 0x71, 0xc0, + 0xdc, 0xb0, 0x73, 0x3d, 0xc3, 0x2a, 0xf9, 0xfd, 0x0a, 0x3b, 0x64, 0xc3, 0x28, 0x42, 0xe2, 0x4c, + 0xca, 0xe5, 0x64, 0x79, 0x60, 0x88, 0xa2, 0xe0, 0xec, 0x0e, 0x8b, 0x70, 0x94, 0x3c, 0xfa, 0x11, + 0x05, 0xc6, 0xee, 0xda, 0x96, 0x66, 0x0a, 0x3f, 0x6b, 0x71, 0x4d, 0xf3, 0xfe, 0xe3, 0x99, 0x97, + 0x80, 0x0c, 0xf7, 0xad, 0x94, 0x4b, 0x70, 0xa4, 0x1b, 0xb3, 0x4f, 0xc2, 0x64, 0x7c, 0x8b, 0x1d, + 0xea, 0x73, 0x59, 0x83, 0x07, 0x0f, 0xd1, 0x9d, 0x2e, 0x4f, 0x36, 0xf5, 0xd7, 0xfb, 0x23, 0x27, + 0x65, 0xe6, 0xea, 0x77, 0x61, 0x0a, 0x7f, 0x2d, 0xf6, 0xad, 0x72, 0x0e, 0x6c, 0x1c, 0xd7, 0x7e, + 0x3c, 0xda, 0x0f, 0xb6, 0xef, 0xe4, 0x3e, 0xd8, 0xd7, 0xdb, 0x17, 0xd3, 0xf3, 0xce, 0x5c, 0xe0, + 0xaa, 0xbe, 0x58, 0xb0, 0x30, 0x17, 0x13, 0x8b, 0xc3, 0x69, 0xb8, 0x86, 0x1f, 0x4d, 0x5a, 0x92, + 0xd8, 0x9e, 0xe5, 0xc5, 0xd8, 0x87, 0xab, 0xcb, 0x91, 0xb3, 0x60, 0xcd, 0x6e, 0xd9, 0xa6, 0xdd, + 0xd8, 0x9d, 0xbf, 0xa3, 0x39, 0x04, 0xdb, 0x6d, 0x4f, 0x60, 0xeb, 0x56, 0xfe, 0xfb, 0xc8, 0x00, + 0x5c, 0x91, 0xd0, 0xa5, 0x06, 0xdd, 0x3c, 0xcc, 0x53, 0x84, 0xd7, 0xe2, 0xb1, 0x31, 0x0b, 0xf9, + 0x7d, 0x48, 0x0f, 0xea, 0x57, 0xbe, 0x58, 0x99, 0x68, 0x4f, 0x01, 0x68, 0x6a, 0x3b, 0x22, 0x50, + 0x9b, 0x38, 0x0c, 0xea, 0xc7, 0xd2, 0xc1, 0x95, 0x80, 0x0c, 0xef, 0xde, 0xf3, 0xbe, 0x19, 0x33, + 0x04, 0x1c, 0x91, 0x80, 0x27, 0x8d, 0xa8, 0xe7, 0x60, 0xa0, 0xb3, 0x4d, 0x38, 0x15, 0xeb, 0xf9, + 0xb1, 0x5e, 0x23, 0xfe, 0xde, 0x90, 0xaf, 0xff, 0x8a, 0x08, 0x72, 0xbf, 0xa8, 0xc0, 0x79, 0x92, + 0x25, 0x4f, 0x0a, 0x65, 0xf8, 0xf9, 0xe3, 0x92, 0x57, 0x45, 0x3e, 0xad, 0x2c, 0x30, 0xce, 0xee, + 0x19, 0xda, 0x05, 0x70, 0x83, 0x6f, 0xba, 0x97, 0xb0, 0x14, 0xa9, 0x4c, 0x82, 0x9b, 0x59, 0xc2, + 0xdf, 0x58, 0x22, 0x86, 0x7e, 0x42, 0x81, 0x69, 0x33, 0x85, 0xc3, 0x09, 0x8e, 0x59, 0x3b, 0x86, + 0xb3, 0x85, 0x3b, 0xa0, 0xa4, 0x41, 0x70, 0x6a, 0x57, 0xd0, 0x4f, 0x65, 0x46, 0x10, 0xe6, 0x82, + 0xc7, 0xda, 0x71, 0x7c, 0x83, 0x39, 0x82, 0x09, 0x7f, 0x46, 0x01, 0x54, 0x4f, 0xe8, 0xd6, 0xc2, + 0xf7, 0xf0, 0x99, 0x23, 0xb7, 0x20, 0x70, 0x0f, 0xa2, 0x64, 0x39, 0x4e, 0xe9, 0x04, 0x5b, 0x67, + 0x2f, 0x85, 0xe7, 0x8b, 0x17, 0xbe, 0xbd, 0xae, 0x73, 0xda, 0x71, 0xc2, 0xd7, 0x39, 0x0d, 0x82, + 0x53, 0xbb, 0xa2, 0xfe, 0xe6, 0x08, 0x37, 0xf5, 0x32, 0x17, 0x8f, 0x97, 0x60, 0x70, 0x43, 0xd3, + 0x6f, 0x8b, 0xeb, 0xd5, 0x9c, 0x4e, 0x47, 0x0b, 0x0c, 0x03, 0x37, 0xb2, 0xf0, 0xff, 0xb1, 0xc0, + 0x8a, 0x5e, 0x80, 0xbe, 0xba, 0xe5, 0x3f, 0xa0, 0x7f, 0x4f, 0x0f, 0xd7, 0x07, 0x61, 0x18, 0x92, + 0xf2, 0x6a, 0x0d, 0x53, 0xa4, 0xc8, 0x82, 0x61, 0x4b, 0x58, 0x46, 0x85, 0xf1, 0xea, 0xe9, 0xbc, + 0x04, 0x02, 0x0b, 0x6b, 0x60, 0xd7, 0xf5, 0x4b, 0x70, 0x40, 0x83, 0xd2, 0x8b, 0x5d, 0x5f, 0xe6, + 0xa6, 0x17, 0x5c, 0xf8, 0xc8, 0xf1, 0x59, 0xe3, 0x97, 0x3e, 0x04, 0x06, 0x3d, 0xcd, 0xb0, 0x3c, + 0xff, 0x31, 0xfc, 0x13, 0x79, 0xa9, 0xad, 0x51, 0x2c, 0xa1, 0x01, 0x94, 0xfd, 0x74, 0xb1, 0x40, + 0x8e, 0x36, 0x60, 0x90, 0x3f, 0x88, 0x17, 0x9f, 0x50, 0xee, 0xeb, 0x33, 0xfe, 0xc6, 0x9e, 0x6f, + 0x03, 0xfe, 0x3f, 0x16, 0x98, 0xd1, 0xcb, 0x30, 0xec, 0xfa, 0xde, 0x66, 0xc3, 0xbd, 0x4d, 0x5d, + 0xe0, 0x6a, 0x26, 0x9e, 0xf7, 0x0a, 0x1f, 0xb3, 0x00, 0x3f, 0xda, 0x80, 0x21, 0x43, 0x68, 0x4c, + 0x23, 0xf9, 0xb7, 0x9d, 0xaf, 0x0d, 0x31, 0x6d, 0xc3, 0x57, 0x84, 0x7c, 0xc4, 0x59, 0x2e, 0x25, + 0xf0, 0x75, 0x74, 0x29, 0x41, 0xaf, 0x00, 0x10, 0xdf, 0x42, 0xee, 0xce, 0x8c, 0xe6, 0xdf, 0x32, + 0x92, 0x9d, 0xdd, 0x17, 0x88, 0x82, 0x22, 0x17, 0x4b, 0x44, 0xd0, 0xab, 0xb2, 0x49, 0x6e, 0xac, + 0xb7, 0x18, 0x1f, 0xc9, 0xa8, 0x35, 0xe1, 0xcd, 0x5c, 0x60, 0x0d, 0x92, 0x2c, 0x65, 0xea, 0xef, + 0x01, 0x40, 0x78, 0x6b, 0x8b, 0x36, 0x61, 0xd8, 0xa7, 0xd2, 0x4b, 0x94, 0x9d, 0xeb, 0x02, 0xcc, + 0xb7, 0x97, 0xff, 0x0b, 0x07, 0xb8, 0x51, 0x29, 0x2d, 0xda, 0x53, 0x98, 0x00, 0xb7, 0xbb, 0x48, + 0x4f, 0xaf, 0x00, 0xe8, 0x61, 0x0c, 0xd7, 0xbe, 0xfc, 0x6b, 0x15, 0xc4, 0x77, 0x0d, 0xd7, 0x4a, + 0x0a, 0x01, 0x2b, 0x11, 0xc9, 0x70, 0x3a, 0xef, 0xcf, 0xe5, 0x74, 0xfe, 0x04, 0x9c, 0x12, 0x4e, + 0x7e, 0x15, 0x76, 0x89, 0xe8, 0xed, 0x8a, 0x77, 0x99, 0xcc, 0xfd, 0xb3, 0x14, 0x05, 0xe1, 0x78, + 0x5d, 0xf4, 0xaf, 0x14, 0x18, 0xd6, 0x85, 0x80, 0x26, 0x78, 0xdb, 0x72, 0x6f, 0x77, 0xf6, 0x73, + 0xbe, 0xbc, 0xc7, 0x05, 0xf9, 0x67, 0x7d, 0xae, 0xea, 0x17, 0x1f, 0x91, 0x18, 0x1f, 0xf4, 0x1a, + 0xfd, 0xae, 0x02, 0xa3, 0x9a, 0x69, 0xda, 0xba, 0xe6, 0xb1, 0xe8, 0x7f, 0xfc, 0xc1, 0xe8, 0xad, + 0x1e, 0x47, 0x31, 0x1f, 0x62, 0xe4, 0x03, 0x79, 0x5f, 0x60, 0x09, 0x08, 0x21, 0x47, 0x34, 0x16, + 0xb9, 0xfb, 0xe8, 0x1f, 0x2b, 0x70, 0x3f, 0x7f, 0xa5, 0x5b, 0xa2, 0x32, 0xd7, 0xa6, 0xa1, 0x6b, + 0x1e, 0xe1, 0x21, 0x49, 0xfd, 0x17, 0x62, 0xdc, 0x45, 0x7e, 0xf8, 0xd0, 0x2e, 0xf2, 0x57, 0xf7, + 0xf7, 0x8a, 0xf7, 0x97, 0xba, 0xc0, 0x8d, 0xbb, 0xea, 0x01, 0xba, 0x0b, 0xe3, 0xa6, 0x1c, 0x5e, + 0x5c, 0x30, 0xf9, 0x5c, 0xb7, 0xab, 0x91, 0x38, 0xe5, 0xdc, 0xc0, 0x10, 0x29, 0xc2, 0x51, 0x52, + 0xb3, 0xb7, 0x61, 0x3c, 0xb2, 0xd1, 0x8e, 0xd5, 0x2e, 0x6d, 0xc1, 0x64, 0x7c, 0x3f, 0x1c, 0xab, + 0x9e, 0x77, 0x13, 0x46, 0x02, 0x61, 0x01, 0x5d, 0x92, 0x08, 0x85, 0xa2, 0xd7, 0x4d, 0xb2, 0xcb, + 0xa9, 0x16, 0x23, 0x36, 0x14, 0x7e, 0x69, 0xca, 0x82, 0x33, 0x0a, 0x84, 0xea, 0x1f, 0x88, 0x4b, + 0xd3, 0x35, 0xd2, 0x6c, 0x99, 0x9a, 0x47, 0xde, 0xf8, 0x6e, 0x46, 0xea, 0x7f, 0x54, 0xf8, 0x79, + 0xc3, 0x45, 0x1b, 0xa4, 0xc1, 0x68, 0x93, 0xe7, 0xb6, 0x63, 0xa1, 0x1c, 0x95, 0xfc, 0x41, 0x24, + 0x57, 0x42, 0x34, 0x58, 0xc6, 0x89, 0xee, 0xc0, 0x88, 0x2f, 0x0c, 0xfa, 0x96, 0x9a, 0xa5, 0xde, + 0x84, 0xb3, 0x40, 0xee, 0x0c, 0x8e, 0x56, 0xbf, 0xc4, 0xc5, 0x21, 0x2d, 0x55, 0x03, 0x94, 0x6c, + 0x83, 0x1e, 0x08, 0x9f, 0xab, 0x29, 0xd1, 0x6c, 0x34, 0x89, 0x27, 0x6b, 0xbe, 0x91, 0xb5, 0x90, + 0x65, 0x64, 0x55, 0x7f, 0xa3, 0x00, 0xd3, 0x42, 0xf5, 0x9c, 0xd7, 0x75, 0xbb, 0x6d, 0x79, 0xe1, + 0x2b, 0x62, 0xfe, 0x34, 0x5f, 0x10, 0x61, 0xe2, 0x24, 0x7f, 0xb7, 0x8f, 0x05, 0x04, 0xdd, 0xe2, + 0xd6, 0x4f, 0xab, 0xbe, 0x66, 0xdf, 0x26, 0x56, 0xc8, 0x25, 0xe4, 0x18, 0x18, 0x8b, 0x69, 0x15, + 0x70, 0x7a, 0x3b, 0xb4, 0x0d, 0xa8, 0xa9, 0xed, 0xc4, 0xb1, 0xe5, 0xcb, 0x71, 0xc6, 0xf4, 0xc5, + 0x95, 0x04, 0x36, 0x9c, 0x42, 0x81, 0x1e, 0xa4, 0x54, 0x92, 0x6b, 0x79, 0xa4, 0xce, 0x87, 0xe8, + 0xfb, 0x6c, 0xb0, 0x83, 0x74, 0x3e, 0x0a, 0xc2, 0xf1, 0xba, 0xea, 0xc7, 0x06, 0xe1, 0x7c, 0x74, + 0x12, 0xe9, 0x17, 0xea, 0xbf, 0x9e, 0x7f, 0xca, 0x7f, 0x1a, 0xc6, 0x27, 0xf2, 0x81, 0xf8, 0xd3, + 0xb0, 0x19, 0xd9, 0x27, 0x48, 0x34, 0x8a, 0x3c, 0x13, 0xfb, 0x3a, 0x3c, 0x85, 0xcf, 0x78, 0xfb, + 0xdc, 0x77, 0xac, 0x4f, 0xfe, 0x3f, 0xa9, 0xc0, 0x6c, 0xb4, 0x78, 0xc9, 0xb0, 0x0c, 0x77, 0x4b, + 0xa4, 0x35, 0x39, 0xfc, 0xcb, 0x34, 0x96, 0x18, 0x79, 0x39, 0x13, 0x23, 0xee, 0x40, 0x0d, 0x7d, + 0x9f, 0x02, 0x17, 0x62, 0xf3, 0x12, 0x49, 0xb2, 0x72, 0xf8, 0x47, 0x6a, 0x2c, 0x76, 0xcb, 0x72, + 0x36, 0x4a, 0xdc, 0x89, 0x5e, 0xa7, 0x80, 0x9e, 0x83, 0xaf, 0x97, 0x80, 0x9e, 0xbf, 0x50, 0x80, + 0x01, 0xe6, 0x16, 0xf5, 0xc6, 0x78, 0x4f, 0xc4, 0xba, 0x9a, 0xe9, 0xce, 0xda, 0x88, 0xb9, 0xb3, + 0x3e, 0x95, 0x9f, 0x44, 0x67, 0x7f, 0xd6, 0xf7, 0xc1, 0x59, 0x56, 0x6d, 0xbe, 0xce, 0x0c, 0x6d, + 0x2e, 0xa9, 0xcf, 0xd7, 0xeb, 0x4c, 0xbd, 0x3d, 0xf8, 0x8e, 0xec, 0x12, 0xf4, 0xb5, 0x1d, 0x33, + 0x1e, 0x21, 0x76, 0x1d, 0x2f, 0x63, 0x5a, 0xae, 0x7e, 0x52, 0x81, 0x49, 0x86, 0x5b, 0x62, 0x31, + 0x68, 0x1b, 0x86, 0x1d, 0xc1, 0x66, 0xc4, 0xda, 0x2c, 0xe7, 0x1e, 0x5a, 0x0a, 0xeb, 0xe2, 0x1a, + 0x9b, 0xff, 0x0b, 0x07, 0xb4, 0xd4, 0x3f, 0x1d, 0x80, 0x99, 0xac, 0x46, 0xe8, 0x87, 0x14, 0x38, + 0xab, 0x87, 0x12, 0xe7, 0x7c, 0xdb, 0xdb, 0xb2, 0x1d, 0x3f, 0x61, 0x42, 0x6e, 0x73, 0x48, 0x69, + 0x3e, 0xe8, 0x15, 0xcb, 0x12, 0x52, 0x4a, 0xa5, 0x80, 0x33, 0x28, 0xa3, 0x57, 0x01, 0x5c, 0x77, + 0xeb, 0x26, 0xd9, 0x6d, 0x69, 0x86, 0xef, 0x98, 0x75, 0x33, 0xff, 0x36, 0xa8, 0xdd, 0x10, 0xa8, + 0x82, 0x4e, 0x71, 0x5b, 0x75, 0x58, 0x2e, 0x91, 0x43, 0x1f, 0x55, 0x60, 0xdc, 0x96, 0xc3, 0xb1, + 0xf4, 0xe2, 0xab, 0x9f, 0x1a, 0xd7, 0x85, 0x4b, 0xda, 0x51, 0x50, 0x94, 0x24, 0x5d, 0x96, 0x29, + 0x37, 0x7e, 0xb2, 0x09, 0xde, 0xb7, 0x92, 0x4f, 0x06, 0xca, 0x38, 0x26, 0xb9, 0xd6, 0x9e, 0x04, + 0x27, 0xc9, 0xb3, 0x4e, 0x11, 0x4f, 0xaf, 0x2f, 0x5a, 0xba, 0xb3, 0xcb, 0x02, 0x00, 0xd0, 0x4e, + 0x0d, 0xe6, 0xef, 0xd4, 0xe2, 0x5a, 0xa9, 0x1c, 0x41, 0x16, 0xed, 0x54, 0x12, 0x9c, 0x24, 0xaf, + 0x7e, 0xa4, 0x00, 0xe7, 0xd8, 0x32, 0xdf, 0x6c, 0x6f, 0x10, 0xee, 0x5b, 0xf5, 0x0f, 0x2e, 0x7e, + 0xce, 0x6f, 0x2b, 0x30, 0xc2, 0xe6, 0xe0, 0x0d, 0xf2, 0x04, 0x93, 0xf5, 0x35, 0xc3, 0x7f, 0xf9, + 0xb7, 0x14, 0x98, 0x4a, 0xa4, 0x88, 0xea, 0xea, 0x01, 0xdf, 0x89, 0xb9, 0xd6, 0xbe, 0x25, 0x4c, + 0x4b, 0xd9, 0x17, 0xc6, 0xad, 0x88, 0xa7, 0xa4, 0x54, 0x9f, 0x83, 0xf1, 0x88, 0xfb, 0xb2, 0x14, + 0xe1, 0x31, 0x2d, 0x36, 0xa5, 0x1c, 0xc0, 0xb1, 0xd0, 0x29, 0xf4, 0x64, 0xb8, 0xe5, 0x93, 0x9c, + 0xed, 0x1f, 0xce, 0x96, 0x47, 0x62, 0xcb, 0xfb, 0xd7, 0x38, 0x2c, 0x6e, 0xa4, 0xdb, 0xcb, 0x35, + 0x0e, 0x0b, 0x40, 0x29, 0xe2, 0xbc, 0xf2, 0xff, 0xb1, 0xc0, 0x8a, 0x9e, 0x8e, 0x46, 0x71, 0x5d, + 0x0d, 0x75, 0xbb, 0xe9, 0x78, 0xec, 0x55, 0xb6, 0x25, 0x13, 0xb5, 0x11, 0xe6, 0x17, 0x41, 0xfc, + 0x2c, 0xcb, 0x95, 0xbc, 0xa6, 0xbc, 0x5a, 0xe3, 0xe1, 0xfd, 0x82, 0x0b, 0xa0, 0xa8, 0xc5, 0xbb, + 0xff, 0x24, 0x2c, 0xde, 0x0e, 0x8c, 0x6e, 0x19, 0x1b, 0xc4, 0xb1, 0xb8, 0x18, 0x33, 0x90, 0x5f, + 0x42, 0xbb, 0x11, 0xa2, 0xe1, 0x66, 0x00, 0xa9, 0x00, 0xcb, 0x44, 0x90, 0x13, 0x89, 0x45, 0x3d, + 0x98, 0x5f, 0x2a, 0x09, 0x4d, 0xd3, 0xe1, 0x38, 0x33, 0xe2, 0x50, 0x5b, 0x00, 0x56, 0x10, 0xa0, + 0xb5, 0x97, 0x8b, 0xa1, 0x30, 0xcc, 0x2b, 0x17, 0x3a, 0xc2, 0xdf, 0x58, 0xa2, 0x40, 0xe7, 0xb5, + 0x19, 0xe6, 0x2b, 0x10, 0x66, 0xc6, 0xa7, 0x7a, 0xcc, 0x19, 0x21, 0xcc, 0x2b, 0x61, 0x01, 0x96, + 0x89, 0xd0, 0x31, 0x36, 0x83, 0x28, 0xfd, 0xc2, 0x8c, 0x98, 0x6b, 0x8c, 0x61, 0xac, 0x7f, 0x3e, + 0xc6, 0xf0, 0x37, 0x96, 0x28, 0xa0, 0x97, 0xa5, 0xfb, 0x43, 0xc8, 0x6f, 0xa4, 0xea, 0xea, 0xee, + 0xf0, 0x9d, 0xa1, 0xad, 0x66, 0x94, 0x7d, 0xa7, 0x17, 0x24, 0x3b, 0x0d, 0xcb, 0x5e, 0x40, 0x79, + 0x47, 0xc2, 0x6e, 0x13, 0x3e, 0x9a, 0x18, 0xeb, 0xf8, 0x68, 0xa2, 0x44, 0xa5, 0x33, 0xe9, 0xe1, + 0x21, 0x63, 0x08, 0xe3, 0xe1, 0x25, 0x48, 0x2d, 0x0e, 0xc4, 0xc9, 0xfa, 0x9c, 0xe1, 0x93, 0x3a, + 0x6b, 0x3b, 0x21, 0x33, 0x7c, 0x5e, 0x86, 0x03, 0x28, 0xda, 0x86, 0x31, 0x57, 0x7a, 0x81, 0x31, + 0x73, 0xaa, 0xd7, 0x2b, 0x44, 0xf1, 0xfa, 0x82, 0xb9, 0x3a, 0xca, 0x25, 0x38, 0x42, 0x27, 0x7a, + 0xbf, 0x35, 0x79, 0xb2, 0xf7, 0x5b, 0xa8, 0x1d, 0x75, 0xae, 0x9e, 0x3a, 0x92, 0x30, 0x2d, 0x07, + 0x3a, 0x5f, 0xd3, 0xa5, 0x25, 0x3b, 0x2d, 0xdb, 0x6d, 0x3b, 0x84, 0x65, 0xcb, 0x61, 0xcb, 0x83, + 0xc2, 0xa5, 0x5d, 0x8c, 0x03, 0x71, 0xb2, 0x3e, 0xfa, 0x1e, 0x05, 0x26, 0xdd, 0x5d, 0xd7, 0x23, + 0x4d, 0x7a, 0x6c, 0xd9, 0x16, 0xb1, 0x3c, 0x77, 0xe6, 0x74, 0xfe, 0xb0, 0xdf, 0xb5, 0x18, 0x2e, + 0x7e, 0xec, 0xc4, 0x4b, 0x71, 0x82, 0x26, 0xdd, 0x39, 0x72, 0xa0, 0x97, 0x99, 0xe9, 0xfc, 0x3b, + 0x47, 0x0e, 0x22, 0xc3, 0x77, 0x8e, 0x5c, 0x82, 0x23, 0x74, 0xd0, 0xa3, 0x30, 0xee, 0xfa, 0x29, + 0xca, 0xd9, 0x0c, 0x9e, 0x09, 0x63, 0x71, 0xd6, 0x64, 0x00, 0x8e, 0xd6, 0x43, 0x1f, 0x82, 0x31, + 0xf9, 0xec, 0x9c, 0x39, 0x7b, 0xd4, 0x51, 0xe9, 0x79, 0xcf, 0x65, 0x50, 0x84, 0x20, 0xc2, 0x70, + 0x56, 0x7a, 0x1b, 0x28, 0x7f, 0xdf, 0xe7, 0xd8, 0x10, 0xb8, 0x3e, 0x9b, 0x5a, 0x03, 0x67, 0xb4, + 0x44, 0x3f, 0x96, 0x7e, 0x5d, 0x3e, 0xc3, 0xb6, 0x74, 0xf5, 0x48, 0xae, 0xcb, 0x9f, 0x33, 0xbc, + 0xad, 0x5b, 0x2d, 0x1e, 0x38, 0xec, 0x90, 0x37, 0xe7, 0xea, 0x1f, 0x29, 0x00, 0x81, 0xc1, 0xe4, + 0x24, 0xae, 0x2a, 0xea, 0x11, 0x1b, 0xd2, 0x42, 0x4f, 0x06, 0x9e, 0xcc, 0xa4, 0x23, 0xea, 0x97, + 0x15, 0x98, 0x08, 0xab, 0x9d, 0x80, 0x6a, 0xa4, 0x47, 0x55, 0xa3, 0x27, 0x7b, 0x1b, 0x57, 0x86, + 0x7e, 0xf4, 0xbf, 0x0a, 0xf2, 0xa8, 0x98, 0xf4, 0xbb, 0x1d, 0xb9, 0xfa, 0xa7, 0xa4, 0x6f, 0xf4, + 0x72, 0xf5, 0x2f, 0x07, 0xd7, 0x08, 0xc7, 0x9b, 0xe2, 0x0a, 0xf0, 0x9d, 0x11, 0xf9, 0xb3, 0x87, + 0xb0, 0x36, 0x81, 0xb0, 0xe9, 0x93, 0xe6, 0x13, 0x70, 0x90, 0x30, 0xfa, 0x8a, 0x7c, 0x3c, 0xf5, + 0x90, 0x28, 0x24, 0x32, 0xe0, 0xce, 0x4e, 0x17, 0xbf, 0x30, 0x05, 0xa3, 0x92, 0x6d, 0x31, 0xe6, + 0xc8, 0xa0, 0x9c, 0x84, 0x23, 0x83, 0x07, 0xa3, 0x7a, 0x90, 0x29, 0xd2, 0x9f, 0xf6, 0x1e, 0x69, + 0x06, 0xc7, 0x62, 0x98, 0x83, 0xd2, 0xc5, 0x32, 0x19, 0x2a, 0xbc, 0x05, 0x7b, 0xac, 0xef, 0x08, + 0xdc, 0x4b, 0x3a, 0xed, 0xab, 0x47, 0x00, 0x7c, 0xf9, 0x9f, 0xd4, 0x45, 0x00, 0xf6, 0xc0, 0x75, + 0xba, 0xe2, 0xde, 0x08, 0x60, 0x58, 0xaa, 0x97, 0xbc, 0x18, 0x1f, 0x38, 0xb1, 0x8b, 0x71, 0xba, + 0x0d, 0x4c, 0x3f, 0xa7, 0x7e, 0x4f, 0xee, 0x6a, 0x41, 0x66, 0xfe, 0x70, 0x1b, 0x04, 0x45, 0x2e, + 0x96, 0x88, 0x64, 0xf8, 0xb3, 0x0c, 0xe5, 0xf2, 0x67, 0x69, 0xc3, 0x69, 0x87, 0x78, 0xce, 0x6e, + 0x69, 0x57, 0x67, 0x99, 0x4b, 0x1c, 0x8f, 0x69, 0xf0, 0xc3, 0xf9, 0xe2, 0x21, 0xe2, 0x24, 0x2a, + 0x9c, 0x86, 0x3f, 0x22, 0x00, 0x8f, 0x74, 0x14, 0x80, 0xdf, 0x09, 0xa3, 0x1e, 0xd1, 0xb7, 0x2c, + 0x43, 0xd7, 0xcc, 0x4a, 0x59, 0x84, 0xe7, 0x0e, 0x65, 0xb9, 0x10, 0x84, 0xe5, 0x7a, 0x68, 0x01, + 0xfa, 0xda, 0x46, 0x5d, 0x68, 0x00, 0xdf, 0x12, 0x58, 0xe9, 0x2b, 0xe5, 0x7b, 0x7b, 0xc5, 0x37, + 0x87, 0x0e, 0x22, 0xc1, 0xa8, 0xae, 0xb5, 0x6e, 0x37, 0xae, 0x79, 0xbb, 0x2d, 0xe2, 0xce, 0xad, + 0x57, 0xca, 0x98, 0x36, 0x4e, 0xf3, 0xf5, 0x19, 0x3b, 0x84, 0xaf, 0xcf, 0x67, 0x14, 0x38, 0xad, + 0xc5, 0x2f, 0x18, 0x88, 0x3b, 0x33, 0x9e, 0x9f, 0x5b, 0xa6, 0x5f, 0x5a, 0x2c, 0x5c, 0x10, 0xe3, + 0x3b, 0x3d, 0x9f, 0x24, 0x87, 0xd3, 0xfa, 0x80, 0x1c, 0x40, 0x4d, 0xa3, 0x11, 0xa4, 0xb7, 0x17, + 0xab, 0x3e, 0x91, 0xcf, 0x6e, 0xb3, 0x92, 0xc0, 0x84, 0x53, 0xb0, 0xa3, 0x3b, 0x30, 0x2a, 0x09, + 0x49, 0x42, 0x93, 0x29, 0x1f, 0xc5, 0x3d, 0x08, 0xd7, 0x76, 0xe5, 0x3b, 0x0e, 0x99, 0x52, 0x70, + 0xc9, 0x29, 0x99, 0x19, 0xc4, 0x45, 0x1f, 0x1b, 0xf5, 0x64, 0xfe, 0x4b, 0xce, 0x74, 0x8c, 0xb8, + 0x03, 0x35, 0x16, 0x85, 0x90, 0x82, 0x25, 0xdd, 0x7c, 0x66, 0x2a, 0x7f, 0x18, 0x95, 0xe5, 0x28, + 0x2a, 0xbe, 0x35, 0x63, 0x85, 0x38, 0x4e, 0x10, 0x2d, 0x01, 0x22, 0xdc, 0x94, 0x1e, 0x2a, 0x67, + 0xee, 0x0c, 0x62, 0xf7, 0xef, 0x6c, 0x49, 0x17, 0x13, 0x50, 0x9c, 0xd2, 0x02, 0x79, 0x11, 0x5b, + 0x49, 0x0f, 0x5a, 0x4e, 0x3c, 0x25, 0x4e, 0x47, 0x8b, 0xc9, 0x77, 0x2b, 0x89, 0x1c, 0xc9, 0x5c, + 0xb9, 0xb9, 0xd1, 0x7b, 0x8e, 0x64, 0x41, 0xbe, 0x9b, 0x4c, 0xc9, 0x9f, 0x57, 0xe0, 0x5c, 0x33, + 0x3d, 0x17, 0x24, 0xd3, 0x79, 0x72, 0x5e, 0x5c, 0x65, 0xa4, 0x97, 0x64, 0x66, 0x8c, 0xac, 0xdc, + 0x93, 0x38, 0xab, 0x23, 0xea, 0x1f, 0x2a, 0xc2, 0x14, 0x7d, 0x82, 0xee, 0x48, 0xc7, 0x7d, 0x4f, + 0xac, 0x3e, 0x07, 0x33, 0x35, 0x3f, 0x86, 0x68, 0x3d, 0x96, 0x2a, 0xe0, 0x3d, 0x30, 0xce, 0xaf, + 0x82, 0x56, 0xb4, 0xd6, 0x6a, 0x78, 0x6f, 0x10, 0x44, 0x9b, 0x28, 0xc9, 0x40, 0x1c, 0xad, 0xab, + 0x7e, 0x55, 0x81, 0x73, 0x51, 0xcc, 0xb6, 0x63, 0xdc, 0xed, 0x1d, 0x31, 0xfa, 0xb8, 0x02, 0xa3, + 0xb7, 0x83, 0x5b, 0x2a, 0x5f, 0x74, 0xcb, 0xf5, 0x64, 0xc3, 0xef, 0x15, 0x71, 0xa4, 0x6b, 0xaf, + 0x64, 0xfe, 0xc9, 0x10, 0xe8, 0x62, 0x99, 0xb4, 0xfa, 0x5f, 0x15, 0x48, 0x98, 0x0f, 0xd0, 0x06, + 0x0c, 0x51, 0x22, 0xe5, 0xd5, 0x9a, 0xd8, 0x13, 0xef, 0xc9, 0x27, 0x55, 0x32, 0x14, 0xfc, 0x52, + 0x44, 0xfc, 0xc0, 0x3e, 0x62, 0xb4, 0xcd, 0x9f, 0xf7, 0xfb, 0x79, 0x84, 0xc4, 0xf6, 0xc8, 0x25, + 0xb6, 0xcb, 0xf9, 0x88, 0xb8, 0x5a, 0x2f, 0x97, 0xe0, 0x08, 0x1d, 0x75, 0x19, 0x20, 0x34, 0xf9, + 0xf4, 0xec, 0xde, 0xf7, 0xef, 0x4e, 0xc3, 0x99, 0x9e, 0x1f, 0x23, 0x7e, 0x54, 0x81, 0xb3, 0x64, + 0xdb, 0xd0, 0xbd, 0xf9, 0x4d, 0x8f, 0x38, 0xb7, 0x6e, 0xad, 0xac, 0x6d, 0x39, 0xc4, 0xdd, 0xb2, + 0xcd, 0x7a, 0x37, 0xce, 0x8c, 0x29, 0x9e, 0x57, 0xcc, 0x34, 0xb1, 0x98, 0x8a, 0x11, 0x67, 0x50, + 0x62, 0xe6, 0xae, 0x6d, 0x6e, 0x08, 0xc0, 0x54, 0xe7, 0x6a, 0x3b, 0xae, 0x27, 0xc2, 0x02, 0x72, + 0x73, 0x57, 0x1c, 0x88, 0x93, 0xf5, 0xe3, 0x48, 0x58, 0x92, 0x3c, 0x26, 0xb7, 0x2b, 0x49, 0x24, + 0x0c, 0x88, 0x93, 0xf5, 0x65, 0x24, 0x7c, 0xa5, 0xe8, 0xa1, 0x38, 0x90, 0x44, 0x12, 0x00, 0x71, + 0xb2, 0x3e, 0xaa, 0xc3, 0x45, 0x87, 0xe8, 0x76, 0xb3, 0x49, 0xac, 0x3a, 0x9b, 0x94, 0x15, 0xcd, + 0x69, 0x18, 0xd6, 0x92, 0xa3, 0xb1, 0x8a, 0xec, 0xf6, 0x40, 0x61, 0xf9, 0x6c, 0x2f, 0xe2, 0x0e, + 0xf5, 0x70, 0x47, 0x2c, 0xa8, 0x09, 0xa7, 0xda, 0x8c, 0xff, 0x3b, 0x2c, 0x6c, 0xd8, 0xb6, 0x66, + 0x8a, 0x2b, 0x82, 0xc3, 0xae, 0x18, 0x3b, 0xa8, 0xd7, 0xa3, 0xa8, 0x70, 0x1c, 0x37, 0xda, 0xa5, + 0xe2, 0xb9, 0xe8, 0x8e, 0x44, 0x72, 0x38, 0x17, 0x49, 0x21, 0xa2, 0x27, 0xd0, 0xe1, 0x34, 0x1a, + 0xa8, 0x02, 0xa7, 0x3d, 0xcd, 0x69, 0x10, 0xaf, 0x54, 0x5d, 0xaf, 0x12, 0x47, 0xa7, 0x3c, 0xd6, + 0xe4, 0xd2, 0xba, 0xc2, 0x51, 0xad, 0x25, 0xc1, 0x38, 0xad, 0x0d, 0xfa, 0x10, 0xbc, 0x25, 0x3a, + 0xa9, 0xcb, 0xf6, 0x1d, 0xe2, 0x2c, 0xd8, 0x6d, 0xab, 0x1e, 0x45, 0x0e, 0x0c, 0xf9, 0x03, 0xfb, + 0x7b, 0xc5, 0xb7, 0xe0, 0x6e, 0x1a, 0xe0, 0xee, 0xf0, 0x26, 0x3b, 0xb0, 0xde, 0x6a, 0xa5, 0x76, + 0x60, 0x34, 0xab, 0x03, 0x19, 0x0d, 0x70, 0x77, 0x78, 0x11, 0x86, 0xb3, 0x7c, 0x62, 0x78, 0xf6, + 0x65, 0x89, 0xe2, 0x18, 0xa3, 0xc8, 0xbe, 0xdf, 0xb5, 0xd4, 0x1a, 0x38, 0xa3, 0x25, 0x3d, 0x53, + 0xae, 0x66, 0x0d, 0x3f, 0x41, 0x66, 0x9c, 0x91, 0x79, 0xdb, 0xfe, 0x5e, 0xf1, 0x2a, 0xee, 0xb2, + 0x0d, 0xee, 0x1a, 0x7b, 0x4a, 0x57, 0xc2, 0x89, 0x48, 0x74, 0x65, 0x22, 0xab, 0x2b, 0xd9, 0x6d, + 0x70, 0xd7, 0xd8, 0xd1, 0xf7, 0x2a, 0x70, 0x5e, 0x6f, 0xb5, 0x6f, 0x18, 0xae, 0x67, 0x37, 0x1c, + 0xad, 0x59, 0x26, 0xba, 0xb6, 0x7b, 0x43, 0x33, 0x37, 0x97, 0x8d, 0x4d, 0x22, 0x94, 0x8e, 0xc3, + 0x7e, 0x38, 0xec, 0xe1, 0x6d, 0xa9, 0xba, 0x9e, 0x8e, 0x14, 0x67, 0xd3, 0x43, 0x3f, 0xac, 0xc0, + 0xc5, 0x26, 0xeb, 0x62, 0x46, 0x87, 0x26, 0x73, 0x75, 0x88, 0x71, 0xb1, 0x95, 0x0e, 0x78, 0x71, + 0x47, 0xaa, 0x6c, 0x92, 0x78, 0x85, 0xf9, 0x46, 0xc3, 0x21, 0x0d, 0x86, 0x35, 0xe0, 0x2e, 0x53, + 0xf9, 0x27, 0x69, 0x25, 0x0b, 0x29, 0xce, 0xa6, 0x87, 0x5e, 0x86, 0xcb, 0x99, 0xc0, 0x92, 0xdd, + 0xb6, 0x3c, 0x76, 0x09, 0xd3, 0xb7, 0xa0, 0xee, 0xef, 0x15, 0x2f, 0xaf, 0x74, 0xac, 0x89, 0x0f, + 0xc0, 0xc4, 0x42, 0x8b, 0x44, 0xc2, 0x00, 0x9c, 0x66, 0x92, 0xd8, 0x8b, 0xb9, 0xb2, 0xed, 0x1e, + 0xe1, 0xdb, 0xff, 0x2f, 0x47, 0xdf, 0xfe, 0x4f, 0xb3, 0x5e, 0xbd, 0xef, 0xe8, 0x7a, 0xf5, 0x8d, + 0x07, 0xff, 0xdd, 0x3c, 0x04, 0xf9, 0x8c, 0x02, 0xe2, 0xfd, 0x26, 0xba, 0x18, 0x71, 0x3b, 0x1a, + 0x8e, 0xb9, 0x1c, 0xf9, 0x49, 0x6f, 0x0b, 0xa9, 0x49, 0x6f, 0xdf, 0x2a, 0xc5, 0x58, 0x1e, 0x09, + 0x95, 0x25, 0x8e, 0x39, 0x0c, 0xb2, 0x8c, 0x1e, 0x84, 0x91, 0x40, 0xa9, 0x16, 0xc6, 0x4e, 0x96, + 0x70, 0x26, 0xd4, 0xbe, 0x43, 0xb8, 0xfa, 0xfb, 0x0a, 0x40, 0x98, 0xcb, 0x19, 0xdd, 0x07, 0x03, + 0xba, 0xa9, 0xb9, 0xbe, 0x4b, 0x51, 0x70, 0x63, 0xc0, 0x6e, 0x22, 0x31, 0x87, 0x1d, 0xfc, 0x18, + 0x01, 0xa9, 0x30, 0xd8, 0x66, 0xd9, 0x2b, 0xc5, 0x03, 0x02, 0xe6, 0x02, 0xb3, 0xce, 0x4a, 0xb0, + 0x80, 0xa0, 0x75, 0x18, 0x6a, 0x1a, 0x16, 0x7b, 0xeb, 0xd1, 0x9f, 0xeb, 0xad, 0x07, 0xd3, 0x07, + 0x56, 0x38, 0x0a, 0xec, 0xe3, 0x52, 0x7f, 0x51, 0x81, 0x53, 0xd1, 0xa0, 0xd7, 0x2c, 0xe4, 0x8d, + 0x88, 0x85, 0x2a, 0x62, 0xed, 0xb3, 0xa6, 0x22, 0xc6, 0x23, 0xf6, 0x61, 0xd1, 0xdb, 0xe9, 0x1e, + 0x6e, 0x1f, 0xd2, 0x63, 0x6f, 0x1f, 0x70, 0x11, 0xf0, 0xa3, 0xa7, 0x61, 0x90, 0x2b, 0xda, 0x54, + 0x8e, 0x4f, 0x89, 0x6d, 0x75, 0x33, 0x7f, 0x3a, 0x89, 0x3c, 0x11, 0x71, 0xe4, 0xa4, 0x9a, 0x85, + 0x8e, 0x49, 0x35, 0x31, 0xf4, 0xe9, 0x8e, 0xd1, 0x8b, 0x27, 0x52, 0x09, 0x57, 0xb8, 0x27, 0x52, + 0x09, 0x57, 0x30, 0x45, 0x86, 0xbc, 0x88, 0x8b, 0x4e, 0x7f, 0x7e, 0x13, 0x10, 0x9f, 0x00, 0xc9, + 0x51, 0x67, 0xa2, 0xa3, 0x93, 0x8e, 0x1f, 0x48, 0x7f, 0x20, 0xff, 0xe3, 0x20, 0x31, 0xe5, 0xdd, + 0x04, 0xd2, 0xf7, 0x3f, 0xa4, 0xc1, 0xcc, 0x0f, 0x69, 0x13, 0x86, 0xc4, 0xa7, 0x20, 0x14, 0x82, + 0xf7, 0xf4, 0x90, 0x3a, 0x5e, 0x4a, 0x52, 0xc5, 0x0b, 0xb0, 0x8f, 0x9c, 0x6a, 0x99, 0x4d, 0x6d, + 0xc7, 0x68, 0xb6, 0x9b, 0x4c, 0x0b, 0x18, 0x90, 0xab, 0xb2, 0x62, 0xec, 0xc3, 0x59, 0x55, 0xfe, + 0xa6, 0x8a, 0x49, 0xed, 0x72, 0x55, 0x5e, 0x8c, 0x7d, 0x38, 0x7a, 0x01, 0x86, 0x9b, 0xda, 0x4e, + 0xad, 0xed, 0x34, 0x88, 0x70, 0xd0, 0xc9, 0x36, 0x0a, 0xb5, 0x3d, 0xc3, 0x9c, 0x33, 0x2c, 0xcf, + 0xf5, 0x9c, 0xb9, 0x8a, 0xe5, 0xdd, 0x72, 0x6a, 0x1e, 0x73, 0x00, 0x62, 0xbb, 0x6e, 0x45, 0x60, + 0xc1, 0x01, 0x3e, 0x64, 0xc2, 0x44, 0x53, 0xdb, 0x59, 0xb7, 0x34, 0x9e, 0x23, 0x41, 0x48, 0xd9, + 0x79, 0x28, 0x30, 0xab, 0xdc, 0x4a, 0x04, 0x17, 0x8e, 0xe1, 0x4e, 0x71, 0x06, 0x1d, 0x3b, 0x2e, + 0x67, 0xd0, 0xf9, 0x20, 0x4a, 0x01, 0x37, 0xe9, 0x9f, 0x4f, 0x0d, 0xff, 0xd7, 0x31, 0x02, 0xc1, + 0x4b, 0x41, 0x04, 0x82, 0x89, 0xfc, 0xde, 0x8b, 0x1d, 0xa2, 0x0f, 0xb4, 0x61, 0xb4, 0xae, 0x79, + 0x1a, 0x2f, 0x75, 0x67, 0x4e, 0xe5, 0xbf, 0x9d, 0x2e, 0x07, 0x68, 0x42, 0x96, 0x14, 0x96, 0xb9, + 0x58, 0xa6, 0x83, 0x6e, 0xc1, 0x19, 0xfa, 0xb1, 0x9a, 0xc4, 0x0b, 0xab, 0x30, 0x9b, 0xd9, 0x24, + 0xfb, 0x7e, 0xd8, 0x2b, 0xb5, 0x9b, 0x69, 0x15, 0x70, 0x7a, 0xbb, 0x30, 0x54, 0xed, 0x54, 0x7a, + 0xa8, 0x5a, 0xf4, 0xa9, 0x34, 0xb7, 0x1b, 0xc4, 0xe6, 0xf4, 0xbd, 0xf9, 0x79, 0x43, 0x6e, 0xe7, + 0x9b, 0x7f, 0xa1, 0xc0, 0x8c, 0xd8, 0x65, 0xc2, 0x55, 0xc6, 0x24, 0xce, 0x8a, 0x66, 0x69, 0x0d, + 0xe2, 0x08, 0x3b, 0xf9, 0x5a, 0x0f, 0xfc, 0x21, 0x81, 0x33, 0x08, 0x0d, 0x71, 0xff, 0xfe, 0x5e, + 0xf1, 0xca, 0x41, 0xb5, 0x70, 0x66, 0xdf, 0x90, 0x03, 0x43, 0xee, 0xae, 0xab, 0x7b, 0xa6, 0x2b, + 0x64, 0xd0, 0xeb, 0x3d, 0x70, 0xd6, 0x1a, 0xc7, 0xc4, 0x59, 0x6b, 0x98, 0x1a, 0x91, 0x97, 0x62, + 0x9f, 0x10, 0xfa, 0x41, 0x05, 0xa6, 0xc4, 0xe5, 0x99, 0x14, 0x7a, 0xe7, 0x4c, 0xfe, 0x77, 0x32, + 0xa5, 0x38, 0x32, 0xdf, 0x3d, 0x86, 0x59, 0x93, 0x12, 0x50, 0x9c, 0xa4, 0x4e, 0x0f, 0xd5, 0x96, + 0x63, 0xd8, 0x8e, 0xe1, 0xed, 0x32, 0x47, 0xa4, 0x01, 0x3f, 0x40, 0x3d, 0x2f, 0xc3, 0x01, 0x14, + 0xd5, 0x60, 0x82, 0x5b, 0x6d, 0x6a, 0x9e, 0xa3, 0x79, 0xa4, 0xb1, 0x2b, 0xbc, 0x85, 0x1e, 0x64, + 0x29, 0x84, 0x23, 0x90, 0x7b, 0x7b, 0xc5, 0x33, 0x62, 0x6d, 0xa2, 0x00, 0x1c, 0x43, 0x81, 0x3e, + 0x10, 0x73, 0xde, 0x9a, 0xc9, 0x9f, 0x5e, 0x90, 0xaf, 0xc5, 0x61, 0x5c, 0xb8, 0x7a, 0x8d, 0x26, + 0xd7, 0x43, 0x8a, 0x97, 0xd9, 0xc7, 0x61, 0x4c, 0xde, 0x35, 0x87, 0x0a, 0x62, 0xe7, 0x01, 0x4a, + 0x0e, 0xf6, 0xb8, 0xe3, 0xfc, 0xa8, 0x3f, 0xa9, 0xc0, 0x64, 0x5c, 0x76, 0x41, 0x5b, 0x30, 0x24, + 0x18, 0x99, 0xa0, 0x3a, 0x9f, 0xd7, 0x6b, 0xd9, 0x24, 0xe2, 0x79, 0x30, 0x17, 0x85, 0x45, 0x11, + 0xf6, 0xd1, 0xcb, 0x2f, 0x12, 0x0a, 0x1d, 0x5e, 0x24, 0x3c, 0x01, 0x67, 0xd3, 0x59, 0x1a, 0x55, + 0x24, 0x34, 0xaa, 0x5f, 0x09, 0xa3, 0x75, 0xa0, 0x48, 0x30, 0xa5, 0x0b, 0x73, 0x98, 0xfa, 0x41, + 0x88, 0xa7, 0x36, 0x43, 0x2f, 0xc3, 0x88, 0xeb, 0x6e, 0x71, 0x3f, 0x33, 0x31, 0xc8, 0x7c, 0x57, + 0x3d, 0x7e, 0x9a, 0x19, 0xae, 0xfb, 0x04, 0x3f, 0x71, 0x88, 0x7e, 0xe1, 0xf9, 0x2f, 0x7d, 0xf5, + 0xf2, 0x9b, 0xfe, 0xe0, 0xab, 0x97, 0xdf, 0xf4, 0x95, 0xaf, 0x5e, 0x7e, 0xd3, 0x87, 0xf7, 0x2f, + 0x2b, 0x5f, 0xda, 0xbf, 0xac, 0xfc, 0xc1, 0xfe, 0x65, 0xe5, 0x2b, 0xfb, 0x97, 0x95, 0xbf, 0xdc, + 0xbf, 0xac, 0xfc, 0xc0, 0xbf, 0xbf, 0xfc, 0xa6, 0x17, 0x1e, 0x0e, 0xa9, 0x5f, 0xf3, 0x89, 0x86, + 0xff, 0xb4, 0x6e, 0x37, 0xa8, 0xd2, 0xea, 0xfa, 0x5a, 0x2b, 0xa3, 0xfe, 0x7f, 0x02, 0x00, 0x00, + 0xff, 0xff, 0x03, 0x88, 0x36, 0xbe, 0x42, 0x25, 0x01, 0x00, } func (m *APIServerLogging) Marshal() (dAtA []byte, err error) { @@ -13265,6 +13296,43 @@ func (m *MaintenanceTimeWindow) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ManualWorkerPoolRollout) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ManualWorkerPoolRollout) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ManualWorkerPoolRollout) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.PendingWorkersRollouts) > 0 { + for iNdEx := len(m.PendingWorkersRollouts) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.PendingWorkersRollouts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *MemorySwapConfiguration) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -17805,6 +17873,20 @@ func (m *ShootStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ManualWorkerPoolRollout != nil { + { + size, err := m.ManualWorkerPoolRollout.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xaa + } if m.InPlaceUpdates != nil { { size, err := m.InPlaceUpdates.MarshalToSizedBuffer(dAtA[:i]) @@ -21474,6 +21556,21 @@ func (m *MaintenanceTimeWindow) Size() (n int) { return n } +func (m *ManualWorkerPoolRollout) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.PendingWorkersRollouts) > 0 { + for _, e := range m.PendingWorkersRollouts { + l = e.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + } + return n +} + func (m *MemorySwapConfiguration) Size() (n int) { if m == nil { return 0 @@ -23254,6 +23351,10 @@ func (m *ShootStatus) Size() (n int) { l = m.InPlaceUpdates.Size() n += 2 + l + sovGenerated(uint64(l)) } + if m.ManualWorkerPoolRollout != nil { + l = m.ManualWorkerPoolRollout.Size() + n += 2 + l + sovGenerated(uint64(l)) + } return n } @@ -25254,6 +25355,21 @@ func (this *MaintenanceTimeWindow) String() string { }, "") return s } +func (this *ManualWorkerPoolRollout) String() string { + if this == nil { + return "nil" + } + repeatedStringForPendingWorkersRollouts := "[]PendingWorkersRollout{" + for _, f := range this.PendingWorkersRollouts { + repeatedStringForPendingWorkersRollouts += strings.Replace(strings.Replace(f.String(), "PendingWorkersRollout", "PendingWorkersRollout", 1), `&`, ``, 1) + "," + } + repeatedStringForPendingWorkersRollouts += "}" + s := strings.Join([]string{`&ManualWorkerPoolRollout{`, + `PendingWorkersRollouts:` + repeatedStringForPendingWorkersRollouts + `,`, + `}`, + }, "") + return s +} func (this *MemorySwapConfiguration) String() string { if this == nil { return "nil" @@ -26543,6 +26659,7 @@ func (this *ShootStatus) String() string { `EncryptedResources:` + fmt.Sprintf("%v", this.EncryptedResources) + `,`, `Networking:` + strings.Replace(this.Networking.String(), "NetworkingStatus", "NetworkingStatus", 1) + `,`, `InPlaceUpdates:` + strings.Replace(this.InPlaceUpdates.String(), "InPlaceUpdatesStatus", "InPlaceUpdatesStatus", 1) + `,`, + `ManualWorkerPoolRollout:` + strings.Replace(this.ManualWorkerPoolRollout.String(), "ManualWorkerPoolRollout", "ManualWorkerPoolRollout", 1) + `,`, `}`, }, "") return s @@ -45572,6 +45689,90 @@ func (m *MaintenanceTimeWindow) Unmarshal(dAtA []byte) error { } return nil } +func (m *ManualWorkerPoolRollout) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ManualWorkerPoolRollout: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ManualWorkerPoolRollout: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PendingWorkersRollouts", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PendingWorkersRollouts = append(m.PendingWorkersRollouts, PendingWorkersRollout{}) + if err := m.PendingWorkersRollouts[len(m.PendingWorkersRollouts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *MemorySwapConfiguration) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -59748,6 +59949,42 @@ func (m *ShootStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 21: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ManualWorkerPoolRollout", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ManualWorkerPoolRollout == nil { + m.ManualWorkerPoolRollout = &ManualWorkerPoolRollout{} + } + if err := m.ManualWorkerPoolRollout.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/core/v1beta1/generated.proto b/pkg/apis/core/v1beta1/generated.proto index 404abf251ab..1e397da85de 100644 --- a/pkg/apis/core/v1beta1/generated.proto +++ b/pkg/apis/core/v1beta1/generated.proto @@ -2050,6 +2050,13 @@ message MaintenanceTimeWindow { optional string end = 2; } +// ManualWorkerPoolRollout contains information about the worker pool rollout progress that has been initiated via the gardener.cloud/operation=rollout-workers annotation. +message ManualWorkerPoolRollout { + // PendingWorkersRollouts contains the names of the worker pools that are still pending rollout. + // +optional + repeated PendingWorkersRollout pendingWorkersRollouts = 1; +} + // MemorySwapConfiguration contains kubelet swap configuration // For more information, please see KEP: 2400-node-swap message MemorySwapConfiguration { @@ -2346,13 +2353,12 @@ message PendingWorkerUpdates { repeated string manualInPlaceUpdate = 2; } -// PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout due to -// credentials rotation. +// PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout. message PendingWorkersRollout { // Name is the name of a worker pool. optional string name = 1; - // LastInitiationTime is the most recent time when the credential rotation was initiated. + // LastInitiationTime is the most recent time when the worker rollout was initiated. // +optional optional .k8s.io.apimachinery.pkg.apis.meta.v1.Time lastInitiationTime = 2; } @@ -3490,6 +3496,10 @@ message ShootStatus { // InPlaceUpdates contains information about in-place updates for the Shoot workers. // +optional optional InPlaceUpdatesStatus inPlaceUpdates = 20; + + // ManualWorkerPoolRollout contains information about the worker pool rollout progress. + // +optional + optional ManualWorkerPoolRollout manualWorkerPoolRollout = 21; } // ShootTemplate is a template for creating a Shoot object. diff --git a/pkg/apis/core/v1beta1/helper/shoot.go b/pkg/apis/core/v1beta1/helper/shoot.go index 86e7f047ba8..a18b5a2fc53 100644 --- a/pkg/apis/core/v1beta1/helper/shoot.go +++ b/pkg/apis/core/v1beta1/helper/shoot.go @@ -264,6 +264,20 @@ func GetShootETCDEncryptionKeyRotationPhase(credentials *gardencorev1beta1.Shoot return "" } +// MutateShootWorkerPoolRollout mutates the .status.manualWorkerPoolRollout field based on the provided mutation function. +// If any field is nil, it gets initialized. +func MutateShootWorkerPoolRollout(shoot *gardencorev1beta1.Shoot, f func(*gardencorev1beta1.ManualWorkerPoolRollout)) { + if f == nil { + return + } + + if shoot.Status.ManualWorkerPoolRollout == nil { + shoot.Status.ManualWorkerPoolRollout = &gardencorev1beta1.ManualWorkerPoolRollout{} + } + + f(shoot.Status.ManualWorkerPoolRollout) +} + // ShouldETCDEncryptionKeyRotationBeAutoCompleteAfterPrepared returns whether the current ETCD encryption key rotation should // be auto completed after the preparation phase has finished. // diff --git a/pkg/apis/core/v1beta1/helper/shoot_test.go b/pkg/apis/core/v1beta1/helper/shoot_test.go index 515fd0eeaeb..04886592f3a 100644 --- a/pkg/apis/core/v1beta1/helper/shoot_test.go +++ b/pkg/apis/core/v1beta1/helper/shoot_test.go @@ -540,6 +540,34 @@ var _ = Describe("Helper", func() { ) }) + Describe("#MutateShootWorkerPoolRollout", func() { + It("should do nothing when mutate function is nil", func() { + shoot := &gardencorev1beta1.Shoot{} + MutateShootWorkerPoolRollout(shoot, nil) + Expect(shoot.Status.ManualWorkerPoolRollout).To(BeNil()) + }) + + DescribeTable("mutate function not nil", + func(shoot *gardencorev1beta1.Shoot, pendingRollouts []gardencorev1beta1.PendingWorkersRollout) { + MutateShootWorkerPoolRollout(shoot, func(rollout *gardencorev1beta1.ManualWorkerPoolRollout) { + rollout.PendingWorkersRollouts = pendingRollouts + }) + Expect(shoot.Status.ManualWorkerPoolRollout.PendingWorkersRollouts).To(Equal(pendingRollouts)) + }, + + Entry("status nil", &gardencorev1beta1.Shoot{}, []gardencorev1beta1.PendingWorkersRollout{ + {Name: "worker1", LastInitiationTime: &metav1.Time{Time: time.Now()}}, + }), + Entry("manualWorkerPoolRollout nil", &gardencorev1beta1.Shoot{Status: gardencorev1beta1.ShootStatus{}}, []gardencorev1beta1.PendingWorkersRollout{ + {Name: "worker2", LastInitiationTime: &metav1.Time{Time: time.Now()}}, + }), + Entry("manualWorkerPoolRollout non-nil", &gardencorev1beta1.Shoot{Status: gardencorev1beta1.ShootStatus{ManualWorkerPoolRollout: &gardencorev1beta1.ManualWorkerPoolRollout{}}}, []gardencorev1beta1.PendingWorkersRollout{ + {Name: "worker3", LastInitiationTime: &metav1.Time{Time: time.Now()}}, + }), + Entry("empty pending rollouts", &gardencorev1beta1.Shoot{}, []gardencorev1beta1.PendingWorkersRollout{}), + ) + }) + Describe("#GetAllZonesFromShoot", func() { It("should return an empty list because there are no zones", func() { Expect(sets.List(GetAllZonesFromShoot(&gardencorev1beta1.Shoot{}))).To(BeEmpty()) diff --git a/pkg/apis/core/v1beta1/types_shoot.go b/pkg/apis/core/v1beta1/types_shoot.go index ae546316215..a58e7c1cd91 100644 --- a/pkg/apis/core/v1beta1/types_shoot.go +++ b/pkg/apis/core/v1beta1/types_shoot.go @@ -222,6 +222,9 @@ type ShootStatus struct { // InPlaceUpdates contains information about in-place updates for the Shoot workers. // +optional InPlaceUpdates *InPlaceUpdatesStatus `json:"inPlaceUpdates,omitempty" protobuf:"bytes,20,opt,name=inPlaceUpdates"` + // ManualWorkerPoolRollout contains information about the worker pool rollout progress. + // +optional + ManualWorkerPoolRollout *ManualWorkerPoolRollout `json:"manualWorkerPoolRollout,omitempty" protobuf:"bytes,21,opt,name=manualWorkerPoolRollout"` } // LastMaintenance holds information about a maintenance operation on the Shoot. @@ -327,6 +330,13 @@ type CARotation struct { PendingWorkersRollouts []PendingWorkersRollout `json:"pendingWorkersRollouts,omitempty" protobuf:"bytes,6,rep,name=pendingWorkersRollouts"` } +// ManualWorkerPoolRollout contains information about the worker pool rollout progress that has been initiated via the gardener.cloud/operation=rollout-workers annotation. +type ManualWorkerPoolRollout struct { + // PendingWorkersRollouts contains the names of the worker pools that are still pending rollout. + // +optional + PendingWorkersRollouts []PendingWorkersRollout `json:"pendingWorkersRollouts,omitempty" protobuf:"bytes,1,rep,name=pendingWorkersRollouts"` +} + // ShootKubeconfigRotation contains information about the kubeconfig credential rotation. type ShootKubeconfigRotation struct { // LastInitiationTime is the most recent time when the kubeconfig credential rotation was initiated. @@ -433,12 +443,11 @@ const ( RotationCompleted CredentialsRotationPhase = "Completed" ) -// PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout due to -// credentials rotation. +// PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout. type PendingWorkersRollout struct { // Name is the name of a worker pool. Name string `json:"name" protobuf:"bytes,1,opt,name=name"` - // LastInitiationTime is the most recent time when the credential rotation was initiated. + // LastInitiationTime is the most recent time when the worker rollout was initiated. // +optional LastInitiationTime *metav1.Time `json:"lastInitiationTime,omitempty" protobuf:"bytes,2,opt,name=lastInitiationTime"` } diff --git a/pkg/apis/core/v1beta1/zz_generated.conversion.go b/pkg/apis/core/v1beta1/zz_generated.conversion.go index 720d866939a..e7aa9f0f336 100644 --- a/pkg/apis/core/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/core/v1beta1/zz_generated.conversion.go @@ -1116,6 +1116,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ManualWorkerPoolRollout)(nil), (*core.ManualWorkerPoolRollout)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ManualWorkerPoolRollout_To_core_ManualWorkerPoolRollout(a.(*ManualWorkerPoolRollout), b.(*core.ManualWorkerPoolRollout), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*core.ManualWorkerPoolRollout)(nil), (*ManualWorkerPoolRollout)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_core_ManualWorkerPoolRollout_To_v1beta1_ManualWorkerPoolRollout(a.(*core.ManualWorkerPoolRollout), b.(*ManualWorkerPoolRollout), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*MemorySwapConfiguration)(nil), (*core.MemorySwapConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_MemorySwapConfiguration_To_core_MemorySwapConfiguration(a.(*MemorySwapConfiguration), b.(*core.MemorySwapConfiguration), scope) }); err != nil { @@ -4995,6 +5005,26 @@ func Convert_core_MaintenanceTimeWindow_To_v1beta1_MaintenanceTimeWindow(in *cor return autoConvert_core_MaintenanceTimeWindow_To_v1beta1_MaintenanceTimeWindow(in, out, s) } +func autoConvert_v1beta1_ManualWorkerPoolRollout_To_core_ManualWorkerPoolRollout(in *ManualWorkerPoolRollout, out *core.ManualWorkerPoolRollout, s conversion.Scope) error { + out.PendingWorkersRollouts = *(*[]core.PendingWorkersRollout)(unsafe.Pointer(&in.PendingWorkersRollouts)) + return nil +} + +// Convert_v1beta1_ManualWorkerPoolRollout_To_core_ManualWorkerPoolRollout is an autogenerated conversion function. +func Convert_v1beta1_ManualWorkerPoolRollout_To_core_ManualWorkerPoolRollout(in *ManualWorkerPoolRollout, out *core.ManualWorkerPoolRollout, s conversion.Scope) error { + return autoConvert_v1beta1_ManualWorkerPoolRollout_To_core_ManualWorkerPoolRollout(in, out, s) +} + +func autoConvert_core_ManualWorkerPoolRollout_To_v1beta1_ManualWorkerPoolRollout(in *core.ManualWorkerPoolRollout, out *ManualWorkerPoolRollout, s conversion.Scope) error { + out.PendingWorkersRollouts = *(*[]PendingWorkersRollout)(unsafe.Pointer(&in.PendingWorkersRollouts)) + return nil +} + +// Convert_core_ManualWorkerPoolRollout_To_v1beta1_ManualWorkerPoolRollout is an autogenerated conversion function. +func Convert_core_ManualWorkerPoolRollout_To_v1beta1_ManualWorkerPoolRollout(in *core.ManualWorkerPoolRollout, out *ManualWorkerPoolRollout, s conversion.Scope) error { + return autoConvert_core_ManualWorkerPoolRollout_To_v1beta1_ManualWorkerPoolRollout(in, out, s) +} + func autoConvert_v1beta1_MemorySwapConfiguration_To_core_MemorySwapConfiguration(in *MemorySwapConfiguration, out *core.MemorySwapConfiguration, s conversion.Scope) error { out.SwapBehavior = (*core.SwapBehavior)(unsafe.Pointer(in.SwapBehavior)) return nil @@ -7045,6 +7075,7 @@ func autoConvert_v1beta1_ShootStatus_To_core_ShootStatus(in *ShootStatus, out *c out.EncryptedResources = *(*[]string)(unsafe.Pointer(&in.EncryptedResources)) out.Networking = (*core.NetworkingStatus)(unsafe.Pointer(in.Networking)) out.InPlaceUpdates = (*core.InPlaceUpdatesStatus)(unsafe.Pointer(in.InPlaceUpdates)) + out.ManualWorkerPoolRollout = (*core.ManualWorkerPoolRollout)(unsafe.Pointer(in.ManualWorkerPoolRollout)) return nil } @@ -7076,6 +7107,7 @@ func autoConvert_core_ShootStatus_To_v1beta1_ShootStatus(in *core.ShootStatus, o out.EncryptedResources = *(*[]string)(unsafe.Pointer(&in.EncryptedResources)) out.Networking = (*NetworkingStatus)(unsafe.Pointer(in.Networking)) out.InPlaceUpdates = (*InPlaceUpdatesStatus)(unsafe.Pointer(in.InPlaceUpdates)) + out.ManualWorkerPoolRollout = (*ManualWorkerPoolRollout)(unsafe.Pointer(in.ManualWorkerPoolRollout)) return nil } diff --git a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go index 82df039f8a7..bb6815fd427 100644 --- a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go @@ -3477,6 +3477,29 @@ func (in *MaintenanceTimeWindow) DeepCopy() *MaintenanceTimeWindow { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManualWorkerPoolRollout) DeepCopyInto(out *ManualWorkerPoolRollout) { + *out = *in + if in.PendingWorkersRollouts != nil { + in, out := &in.PendingWorkersRollouts, &out.PendingWorkersRollouts + *out = make([]PendingWorkersRollout, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManualWorkerPoolRollout. +func (in *ManualWorkerPoolRollout) DeepCopy() *ManualWorkerPoolRollout { + if in == nil { + return nil + } + out := new(ManualWorkerPoolRollout) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemorySwapConfiguration) DeepCopyInto(out *MemorySwapConfiguration) { *out = *in @@ -5904,6 +5927,11 @@ func (in *ShootStatus) DeepCopyInto(out *ShootStatus) { *out = new(InPlaceUpdatesStatus) (*in).DeepCopyInto(*out) } + if in.ManualWorkerPoolRollout != nil { + in, out := &in.ManualWorkerPoolRollout, &out.ManualWorkerPoolRollout + *out = new(ManualWorkerPoolRollout) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index 66938606743..f3c475ad171 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -75,6 +75,7 @@ var ( v1beta1constants.OperationRotateObservabilityCredentials, v1beta1constants.ShootOperationRotateSSHKeypair, v1beta1constants.OperationRotateRolloutWorkers, + v1beta1constants.OperationRolloutWorkers, ).Union(forbiddenShootOperationsWhenHibernated) forbiddenShootOperationsWhenHibernated = sets.New( v1beta1constants.OperationRotateCredentialsStart, @@ -2920,11 +2921,11 @@ func validateShootOperation(operation, maintenanceOperation string, shoot *core. if forbiddenETCDEncryptionKeyShootOperationsWithK8s134.Has(operation) && !k8sLess134 { allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("for Kubernetes versions >= 1.34, operation '%s' is no longer supported, please use 'rotate-etcd-encryption-key' instead, which performs a complete etcd encryption key rotation", operation))) } - if !availableShootOperations.Has(operation) && !strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers) { + if !availableShootOperations.Has(operation) && !strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers) && !strings.HasPrefix(operation, v1beta1constants.OperationRolloutWorkers) { allErrs = append(allErrs, field.NotSupported(fldPathOp, operation, sets.List(availableShootOperations))) } if helper.IsShootInHibernation(shoot) && - (forbiddenShootOperationsWhenHibernated.Has(operation) || strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers)) { + (forbiddenShootOperationsWhenHibernated.Has(operation) || strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers) || strings.HasPrefix(operation, v1beta1constants.OperationRolloutWorkers)) { allErrs = append(allErrs, field.Forbidden(fldPathOp, "operation is not permitted when shoot is hibernated or is waking up")) } if !encryptedResources.Equal(sets.New(getResourcesForEncryption(shoot.Spec.Kubernetes.KubeAPIServer)...)) && @@ -2937,11 +2938,11 @@ func validateShootOperation(operation, maintenanceOperation string, shoot *core. if forbiddenETCDEncryptionKeyShootOperationsWithK8s134.Has(maintenanceOperation) && !k8sLess134 { allErrs = append(allErrs, field.Forbidden(fldPathMaintOp, fmt.Sprintf("for Kubernetes versions >= 1.34, operation '%s' is no longer supported, please use 'rotate-etcd-encryption-key' instead, which performs a complete etcd encryption key rotation", maintenanceOperation))) } - if !availableShootMaintenanceOperations.Has(maintenanceOperation) && !strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRotateRolloutWorkers) { + if !availableShootMaintenanceOperations.Has(maintenanceOperation) && !strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRotateRolloutWorkers) && !strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRolloutWorkers) { allErrs = append(allErrs, field.NotSupported(fldPathMaintOp, maintenanceOperation, sets.List(availableShootMaintenanceOperations))) } if helper.IsShootInHibernation(shoot) && - (forbiddenShootOperationsWhenHibernated.Has(maintenanceOperation) || strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRotateRolloutWorkers)) { + (forbiddenShootOperationsWhenHibernated.Has(maintenanceOperation) || strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRotateRolloutWorkers) || strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRolloutWorkers)) { allErrs = append(allErrs, field.Forbidden(fldPathMaintOp, "operation is not permitted when shoot is hibernated or is waking up")) } if !encryptedResources.Equal(sets.New(getResourcesForEncryption(shoot.Spec.Kubernetes.KubeAPIServer)...)) && forbiddenShootOperationsWhenEncryptionChangeIsRollingOut.Has(maintenanceOperation) { @@ -3079,6 +3080,49 @@ func validateShootOperationContext(operation string, shoot *core.Shoot, fldPath } } } + if strings.HasPrefix(operation, v1beta1constants.OperationRolloutWorkers) { + allErrs = append(allErrs, validateOperationRolloutWorkers(operation, shoot, fldPath)...) + } + + return allErrs +} + +func validateOperationRolloutWorkers(operation string, shoot *core.Shoot, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + // Normalise representation of pool names + poolNames := strings.Split(strings.TrimPrefix(operation, v1beta1constants.OperationRolloutWorkers+"="), ",") + if len(poolNames) == 0 || sets.New(poolNames...).Has("") { + allErrs = append(allErrs, field.Required(fldPath, "must provide either '*' or at least one pool name via "+v1beta1constants.OperationRolloutWorkers+"=[,,...]")) + } + if len(poolNames) == 1 && poolNames[0] == "*" { + poolNames = make([]string, 0, len(shoot.Spec.Provider.Workers)) + for _, worker := range shoot.Spec.Provider.Workers { + poolNames = append(poolNames, worker.Name) + } + } + if sets.New(poolNames...).Has("*") { + allErrs = append(allErrs, field.Forbidden(fldPath, "if '*' is provided, no other pool names are allowed")) + } + + // Validate list of pool names. When a '*' was provided, there won't be any duplicates + // but we still want to check if any of them are using in-place update strategy. + names := sets.New[string]() + for _, poolName := range poolNames { + if names.Has(poolName) { + allErrs = append(allErrs, field.Duplicate(fldPath, "pool name "+poolName+" was specified multiple times")) + } + names.Insert(poolName) + + workerIndex := slices.IndexFunc(shoot.Spec.Provider.Workers, func(worker core.Worker) bool { + return worker.Name == poolName + }) + if workerIndex == -1 { + allErrs = append(allErrs, field.Invalid(fldPath, poolName, "worker pool name "+poolName+" does not exist in .spec.provider.workers[]")) + } else if worker := shoot.Spec.Provider.Workers[workerIndex]; helper.IsUpdateStrategyInPlace(worker.UpdateStrategy) { + allErrs = append(allErrs, field.Invalid(fldPath, poolName, "worker pool "+poolName+" is using in-place update strategy; manual worker pool rollout is not supported for such pools")) + } + } return allErrs } diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index 8e29a27cd5d..cad37c2c1b8 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -4817,6 +4817,147 @@ var _ = Describe("Shoot Validation Tests", func() { Entry("maintenance.gardener.cloud/operation=rotate-serviceaccount-key-start-without-workers-rollout", "maintenance.gardener.cloud/operation", "rotate-serviceaccount-key-start-without-workers-rollout"), ) + DescribeTable("validate specifying valid rollout worker operation annotations", + func(key, value string) { + shoot.Spec.Provider.Workers = []core.Worker{ + { + Name: "worker-1", + Machine: core.Machine{ + Type: "xlarge", + }, + Maximum: 1, + Minimum: 0, + MaxSurge: &maxSurge, + }, + { + Name: "worker-2", + Machine: core.Machine{ + Type: "xlarge", + }, + Maximum: 1, + Minimum: 0, + MaxSurge: &maxSurge, + }, + } + + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, key, value) + Expect(ValidateShoot(shoot)).To(BeEmpty()) + }, + + Entry("gardener.cloud/operation=rollout-workers=worker-1", "gardener.cloud/operation", "rollout-workers=worker-1"), + Entry("gardener.cloud/operation=rollout-workers=worker-2", "gardener.cloud/operation", "rollout-workers=worker-2"), + Entry("gardener.cloud/operation=rollout-workers=worker-1,worker-2", "gardener.cloud/operation", "rollout-workers=worker-1,worker-2"), + Entry("gardener.cloud/operation=rollout-workers=worker-2,worker-1", "gardener.cloud/operation", "rollout-workers=worker-2,worker-1"), + Entry("gardener.cloud/operation=rollout-workers=*", "gardener.cloud/operation", "rollout-workers=*"), + ) + + DescribeTable("validate specifying bad rollout worker operation annotations", + func(key, value string) { + shoot.Spec.Provider.Workers = []core.Worker{ + { + Name: "worker-1", + Machine: core.Machine{ + Type: "xlarge", + }, + Maximum: 1, + Minimum: 0, + }, + { + Name: "worker-2", + Machine: core.Machine{ + Type: "xlarge", + }, + Maximum: 1, + Minimum: 0, + }, + } + + matcher := ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal(fmt.Sprintf("metadata.annotations[%s]", key)), + "Detail": ContainSubstring(("worker pool name unknown-worker does not exist")), + }))) + + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, key, value) + Expect(ValidateShoot(shoot)).To(matcher) + }, + + Entry("gardener.cloud/operation=rollout-workers=unknown-worker", "gardener.cloud/operation", "rollout-workers=unknown-worker"), + ) + + DescribeTable("validate specifying '*' alongside other worker names in rollout worker operation annotations", + func(key, value string) { + shoot.Spec.Provider.Workers = []core.Worker{ + { + Name: "worker-1", + Machine: core.Machine{ + Type: "xlarge", + }, + Maximum: 1, + Minimum: 0, + }, + { + Name: "worker-2", + Machine: core.Machine{ + Type: "xlarge", + }, + Maximum: 1, + Minimum: 0, + }, + } + + matcher := ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal(fmt.Sprintf("metadata.annotations[%s]", key)), + "Detail": ContainSubstring(("if '*' is provided, no other pool names are allowed")), + }))) + + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, key, value) + Expect(ValidateShoot(shoot)).To(matcher) + }, + + Entry("gardener.cloud/operation=rollout-workers=*,worker-1", "gardener.cloud/operation", "rollout-workers=*,worker-1"), + Entry("gardener.cloud/operation=rollout-workers=worker-1,*", "gardener.cloud/operation", "rollout-workers=worker-1,*"), + Entry("gardener.cloud/operation=rollout-workers=worker-1,*,worker-2", "gardener.cloud/operation", "rollout-workers=worker-1,*,worker-2"), + ) + + DescribeTable("validate specifying a rollout for a worker with an `inPlaceUpdate` strategy", + func(key, value string) { + shoot.Spec.Provider.Workers = []core.Worker{ + { + Name: "worker-1", + Machine: core.Machine{ + Type: "xlarge", + }, + Maximum: 1, + Minimum: 0, + UpdateStrategy: ptr.To(core.AutoInPlaceUpdate), + }, + { + Name: "worker-2", + Machine: core.Machine{ + Type: "xlarge", + }, + Maximum: 1, + Minimum: 0, + UpdateStrategy: ptr.To(core.ManualInPlaceUpdate), + }, + } + + matcher := ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal(fmt.Sprintf("metadata.annotations[%s]", key)), + "Detail": ContainSubstring(("using in-place update strategy; manual worker pool rollout is not supported for such pools")), + }))) + + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, key, value) + Expect(ValidateShoot(shoot)).To(matcher) + }, + + Entry("gardener.cloud/operation=rollout-workers=worker-1", "gardener.cloud/operation", "rollout-workers=worker-1"), + Entry("gardener.cloud/operation=rollout-workers=worker-2", "gardener.cloud/operation", "rollout-workers=worker-2"), + ) + DescribeTable("starting rotation of all credentials", func(allowed bool, status core.ShootStatus) { metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "gardener.cloud/operation", "rotate-credentials-start") diff --git a/pkg/apis/core/zz_generated.deepcopy.go b/pkg/apis/core/zz_generated.deepcopy.go index 0e0e422f3f0..4c0893d3504 100644 --- a/pkg/apis/core/zz_generated.deepcopy.go +++ b/pkg/apis/core/zz_generated.deepcopy.go @@ -3477,6 +3477,29 @@ func (in *MaintenanceTimeWindow) DeepCopy() *MaintenanceTimeWindow { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManualWorkerPoolRollout) DeepCopyInto(out *ManualWorkerPoolRollout) { + *out = *in + if in.PendingWorkersRollouts != nil { + in, out := &in.PendingWorkersRollouts, &out.PendingWorkersRollouts + *out = make([]PendingWorkersRollout, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManualWorkerPoolRollout. +func (in *ManualWorkerPoolRollout) DeepCopy() *ManualWorkerPoolRollout { + if in == nil { + return nil + } + out := new(ManualWorkerPoolRollout) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemorySwapConfiguration) DeepCopyInto(out *MemorySwapConfiguration) { *out = *in @@ -5899,6 +5922,11 @@ func (in *ShootStatus) DeepCopyInto(out *ShootStatus) { *out = new(InPlaceUpdatesStatus) (*in).DeepCopyInto(*out) } + if in.ManualWorkerPoolRollout != nil { + in, out := &in.ManualWorkerPoolRollout, &out.ManualWorkerPoolRollout + *out = new(ManualWorkerPoolRollout) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apiserver/openapi/api_violations.report b/pkg/apiserver/openapi/api_violations.report index b9d7b7a03d2..302d97ecf80 100644 --- a/pkg/apiserver/openapi/api_violations.report +++ b/pkg/apiserver/openapi/api_violations.report @@ -34,6 +34,7 @@ API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,MachineImageVersion,Architectures API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,MachineImageVersion,CRI API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,MachineImageVersion,CapabilityFlavors +API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,ManualWorkerPoolRollout,PendingWorkersRollouts API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,NamespacedCloudProfileSpec,MachineImages API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,NamespacedCloudProfileSpec,MachineTypes API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,NamespacedCloudProfileSpec,VolumeTypes diff --git a/pkg/apiserver/openapi/openapi_generated.go b/pkg/apiserver/openapi/openapi_generated.go index 05d2c502b8e..da3d893769c 100644 --- a/pkg/apiserver/openapi/openapi_generated.go +++ b/pkg/apiserver/openapi/openapi_generated.go @@ -142,6 +142,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/gardener/gardener/pkg/apis/core/v1beta1.Maintenance": schema_pkg_apis_core_v1beta1_Maintenance(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.MaintenanceAutoUpdate": schema_pkg_apis_core_v1beta1_MaintenanceAutoUpdate(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.MaintenanceTimeWindow": schema_pkg_apis_core_v1beta1_MaintenanceTimeWindow(ref), + "github.com/gardener/gardener/pkg/apis/core/v1beta1.ManualWorkerPoolRollout": schema_pkg_apis_core_v1beta1_ManualWorkerPoolRollout(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.MemorySwapConfiguration": schema_pkg_apis_core_v1beta1_MemorySwapConfiguration(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.Monitoring": schema_pkg_apis_core_v1beta1_Monitoring(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.NamedResourceReference": schema_pkg_apis_core_v1beta1_NamedResourceReference(ref), @@ -6070,6 +6071,35 @@ func schema_pkg_apis_core_v1beta1_MaintenanceTimeWindow(ref common.ReferenceCall } } +func schema_pkg_apis_core_v1beta1_ManualWorkerPoolRollout(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ManualWorkerPoolRollout contains information about the worker pool rollout progress that has been initiated via the gardener.cloud/operation=rollout-workers annotation.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "pendingWorkersRollouts": { + SchemaProps: spec.SchemaProps{ + Description: "PendingWorkersRollouts contains the names of the worker pools that are still pending rollout.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/gardener/gardener/pkg/apis/core/v1beta1.PendingWorkersRollout"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/gardener/gardener/pkg/apis/core/v1beta1.PendingWorkersRollout"}, + } +} + func schema_pkg_apis_core_v1beta1_MemorySwapConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -6881,7 +6911,7 @@ func schema_pkg_apis_core_v1beta1_PendingWorkersRollout(ref common.ReferenceCall return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout due to credentials rotation.", + Description: "PendingWorkersRollout contains the name of a worker pool and the initiation time of their last rollout.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "name": { @@ -6894,7 +6924,7 @@ func schema_pkg_apis_core_v1beta1_PendingWorkersRollout(ref common.ReferenceCall }, "lastInitiationTime": { SchemaProps: spec.SchemaProps{ - Description: "LastInitiationTime is the most recent time when the credential rotation was initiated.", + Description: "LastInitiationTime is the most recent time when the worker rollout was initiated.", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), }, }, @@ -9978,12 +10008,18 @@ func schema_pkg_apis_core_v1beta1_ShootStatus(ref common.ReferenceCallback) comm Ref: ref("github.com/gardener/gardener/pkg/apis/core/v1beta1.InPlaceUpdatesStatus"), }, }, + "manualWorkerPoolRollout": { + SchemaProps: spec.SchemaProps{ + Description: "ManualWorkerPoolRollout contains information about the worker pool rollout progress.", + Ref: ref("github.com/gardener/gardener/pkg/apis/core/v1beta1.ManualWorkerPoolRollout"), + }, + }, }, Required: []string{"gardener", "hibernated", "technicalID", "uid"}, }, }, Dependencies: []string{ - "github.com/gardener/gardener/pkg/apis/core/v1beta1.Condition", "github.com/gardener/gardener/pkg/apis/core/v1beta1.Gardener", "github.com/gardener/gardener/pkg/apis/core/v1beta1.InPlaceUpdatesStatus", "github.com/gardener/gardener/pkg/apis/core/v1beta1.LastError", "github.com/gardener/gardener/pkg/apis/core/v1beta1.LastMaintenance", "github.com/gardener/gardener/pkg/apis/core/v1beta1.LastOperation", "github.com/gardener/gardener/pkg/apis/core/v1beta1.NetworkingStatus", "github.com/gardener/gardener/pkg/apis/core/v1beta1.ShootAdvertisedAddress", "github.com/gardener/gardener/pkg/apis/core/v1beta1.ShootCredentials", "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, + "github.com/gardener/gardener/pkg/apis/core/v1beta1.Condition", "github.com/gardener/gardener/pkg/apis/core/v1beta1.Gardener", "github.com/gardener/gardener/pkg/apis/core/v1beta1.InPlaceUpdatesStatus", "github.com/gardener/gardener/pkg/apis/core/v1beta1.LastError", "github.com/gardener/gardener/pkg/apis/core/v1beta1.LastMaintenance", "github.com/gardener/gardener/pkg/apis/core/v1beta1.LastOperation", "github.com/gardener/gardener/pkg/apis/core/v1beta1.ManualWorkerPoolRollout", "github.com/gardener/gardener/pkg/apis/core/v1beta1.NetworkingStatus", "github.com/gardener/gardener/pkg/apis/core/v1beta1.ShootAdvertisedAddress", "github.com/gardener/gardener/pkg/apis/core/v1beta1.ShootCredentials", "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, } } diff --git a/pkg/apiserver/registry/core/shoot/strategy.go b/pkg/apiserver/registry/core/shoot/strategy.go index 3ac2594911d..9abdddfce49 100644 --- a/pkg/apiserver/registry/core/shoot/strategy.go +++ b/pkg/apiserver/registry/core/shoot/strategy.go @@ -146,9 +146,10 @@ func mustIncreaseGeneration(oldShoot, newShoot *core.Shoot) bool { mustIncrease, mustRemoveOperationAnnotation = false, false } - if strings.HasPrefix(newShoot.Annotations[v1beta1constants.GardenerOperation], v1beta1constants.OperationRotateRolloutWorkers) { + if strings.HasPrefix(newShoot.Annotations[v1beta1constants.GardenerOperation], v1beta1constants.OperationRotateRolloutWorkers) || + strings.HasPrefix(newShoot.Annotations[v1beta1constants.GardenerOperation], v1beta1constants.OperationRolloutWorkers) { // We don't want to remove the annotation so that the gardenlet can pick it up and perform - // the rotation. It has to remove the annotation after it is done. + // the rotation/rollout. It has to remove the annotation after it is done. mustIncrease, mustRemoveOperationAnnotation = true, false } } diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler.go b/pkg/gardenlet/controller/shoot/shoot/reconciler.go index 0d08239c000..7c2bfdaf6c1 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler.go @@ -12,6 +12,7 @@ import ( "strings" "time" + machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -621,13 +622,17 @@ func (r *Reconciler) updateShootStatusOperationStart( LastUpdateTime: now, } - var mustRemoveOperationAnnotation bool + var ( + operation = shoot.Annotations[v1beta1constants.GardenerOperation] + mustRemoveOperationAnnotation bool + ) + k8sLess134, err := versionutils.CompareVersions(shoot.Spec.Kubernetes.Version, "<", "1.34") if err != nil { return fmt.Errorf("failed checking if Shoot k8s version is less than 1.34: %w", err) } - switch shoot.Annotations[v1beta1constants.GardenerOperation] { + switch operation { case v1beta1constants.OperationRotateCredentialsStart: mustRemoveOperationAnnotation = true startRotationCA(shoot, &now) @@ -695,7 +700,8 @@ func (r *Reconciler) updateShootStatusOperationStart( completeRotationETCDEncryptionKey(shoot, &now) } - if operation := shoot.Annotations[v1beta1constants.GardenerOperation]; strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers) { + switch { + case strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers): mustRemoveOperationAnnotation = true poolNames := sets.NewString(strings.Split(strings.TrimPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers+"="), ",")...) @@ -714,6 +720,53 @@ func (r *Reconciler) updateShootStatusOperationStart( }) }) } + case strings.HasPrefix(operation, v1beta1constants.OperationRolloutWorkers): + mustRemoveOperationAnnotation = true + poolNames := sets.NewString(strings.Split(strings.TrimPrefix(operation, v1beta1constants.OperationRolloutWorkers+"="), ",")...) + + if poolNames.Has("*") { + poolNames = sets.NewString() + for _, pool := range shoot.Spec.Provider.Workers { + poolNames.Insert(pool.Name) + } + } + + for poolName := range poolNames { + machineDeploymentList := &machinev1alpha1.MachineDeploymentList{} + if err := r.SeedClientSet.Client().List(ctx, machineDeploymentList, client.InNamespace(shoot.Status.TechnicalID), client.MatchingLabels{v1beta1constants.LabelWorkerPool: poolName}); err != nil { + return fmt.Errorf("failed to list MachineDeployments for pool %s in namespace %s: %w", poolName, shoot.Status.TechnicalID, err) + } + + if len(machineDeploymentList.Items) == 0 { + return fmt.Errorf("no MachineDeployment found for worker pool %s in namespace %s", poolName, shoot.Status.TechnicalID) + } + + if len(machineDeploymentList.Items) > 1 { + return fmt.Errorf("multiple MachineDeployments found for worker pool %s in namespace %s", poolName, shoot.Status.TechnicalID) + } + + machineDeployment := &machineDeploymentList.Items[0] + + patch := client.MergeFrom(machineDeployment.DeepCopy()) + metav1.SetMetaDataAnnotation(&machineDeployment.Spec.Template.ObjectMeta, v1beta1constants.OperationRolloutWorkers, now.String()) + if err := r.SeedClientSet.Client().Patch(ctx, machineDeployment, patch); err != nil { + return fmt.Errorf("failed to annotate MachineDeployment %s: %w", client.ObjectKeyFromObject(machineDeployment), err) + } + } + + v1beta1helper.MutateShootWorkerPoolRollout(shoot, func(rollout *gardencorev1beta1.ManualWorkerPoolRollout) { + workerRolloutInitiationTime := &now + + var pendingWorkersRollouts []gardencorev1beta1.PendingWorkersRollout + for worker := range poolNames { + pendingWorkersRollouts = append(pendingWorkersRollouts, gardencorev1beta1.PendingWorkersRollout{ + Name: worker, + LastInitiationTime: workerRolloutInitiationTime, + }) + } + + rollout.PendingWorkersRollouts = append(rollout.PendingWorkersRollouts, pendingWorkersRollouts...) + }) } removeNonExistentPoolsFromPendingWorkersRollouts(shoot, v1beta1helper.HibernationIsEnabled(shoot)) @@ -850,6 +903,12 @@ func (r *Reconciler) patchShootStatusOperationSuccess( }) } + if shoot.Status.ManualWorkerPoolRollout != nil { + v1beta1helper.MutateShootWorkerPoolRollout(shoot, func(rollout *gardencorev1beta1.ManualWorkerPoolRollout) { + rollout.PendingWorkersRollouts = nil + }) + } + switch v1beta1helper.GetShootServiceAccountKeyRotationPhase(shoot.Status.Credentials) { case gardencorev1beta1.RotationPreparing: if !manualInPlacePendingWorkersPresent(shoot.Status.InPlaceUpdates) { @@ -999,6 +1058,12 @@ func removeNonExistentPoolsFromPendingWorkersRollouts(shoot *gardencorev1beta1.S shoot.Status.InPlaceUpdates = nil } } + + if shoot.Status.ManualWorkerPoolRollout != nil && shoot.Status.ManualWorkerPoolRollout.PendingWorkersRollouts != nil { + shoot.Status.ManualWorkerPoolRollout.PendingWorkersRollouts = slices.DeleteFunc(shoot.Status.ManualWorkerPoolRollout.PendingWorkersRollouts, func(rollout gardencorev1beta1.PendingWorkersRollout) bool { + return !poolNames.Has(rollout.Name) + }) + } } func (r *Reconciler) shootHasBastions(ctx context.Context, shoot *gardencorev1beta1.Shoot) (bool, error) { diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_test.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_test.go index 5ec6a34e498..fde165e53ec 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_test.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_test.go @@ -85,6 +85,12 @@ var _ = Describe("Reconciler", func() { ManualInPlaceUpdate: []string{"worker2", "worker5"}, }, }, + ManualWorkerPoolRollout: &gardencorev1beta1.ManualWorkerPoolRollout{ + PendingWorkersRollouts: []gardencorev1beta1.PendingWorkersRollout{ + {Name: "worker1"}, + {Name: "worker3"}, + }, + }, }, } }) @@ -108,6 +114,10 @@ var _ = Describe("Reconciler", func() { gardencorev1beta1.PendingWorkersRollout{Name: "worker4"}, )) + Expect(shoot.Status.ManualWorkerPoolRollout.PendingWorkersRollouts).To(ConsistOf( + gardencorev1beta1.PendingWorkersRollout{Name: "worker3"}, + )) + Expect(shoot.Status.InPlaceUpdates.PendingWorkerUpdates.AutoInPlaceUpdate).To(ConsistOf("worker4")) Expect(shoot.Status.InPlaceUpdates.PendingWorkerUpdates.ManualInPlaceUpdate).To(ConsistOf("worker5")) }) @@ -146,6 +156,16 @@ var _ = Describe("Reconciler", func() { }) }) + It("should clean the pending workers", func() { + shoot.Status.ManualWorkerPoolRollout.PendingWorkersRollouts = []gardencorev1beta1.PendingWorkersRollout{{Name: "worker-1"}} + Expect(gardenClient.Status().Update(ctx, shoot)).To(Succeed()) + + Expect(reconciler.patchShootStatusOperationSuccess(ctx, shoot, nil, gardencorev1beta1.LastOperationTypeReconcile)).To(Succeed()) + + Expect(gardenClient.Get(ctx, client.ObjectKeyFromObject(shoot), shoot)).To(Succeed()) + Expect(shoot.Status.ManualWorkerPoolRollout.PendingWorkersRollouts).To(BeNil()) + }) + It("should not set the rotation status to Prepared if current status is Preparing and manual in-place update pending workers are present", func() { shoot.Status.Credentials.Rotation.CertificateAuthorities.Phase = gardencorev1beta1.RotationPreparing shoot.Status.Credentials.Rotation.ServiceAccountKey.Phase = gardencorev1beta1.RotationPreparing diff --git a/test/e2e/gardener/shoot/create_rotate_delete.go b/test/e2e/gardener/shoot/create_rotate_delete.go index f33323465f4..c2da923e1f8 100644 --- a/test/e2e/gardener/shoot/create_rotate_delete.go +++ b/test/e2e/gardener/shoot/create_rotate_delete.go @@ -13,6 +13,7 @@ import ( "strings" "time" + machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -238,9 +239,70 @@ func testCredentialRotationWithoutWorkersRollout(s *ShootContext, shootVerifiers testCredentialRotationComplete(s, shootVerifiers, utilsverifiers, v1beta1constants.OperationRotateCredentialsComplete) } +// Testing the annotation requires that we assert that a rollout has been triggered and finishes successfully. +// Current check verifies that a new MachineSet gets created after the annotation is set by +// checking the creation timestamp of the current MachineSet, annotating the Shoot, then checking that +// the creation timestamp of the new MachineSet is newer than the old one. +func testManualWorkersRollout(s *ShootContext) { + var oldMachineSetCreationTimestamp time.Time + + It("Should fetch old machine set creation timestamp", func(ctx SpecContext) { + Eventually(ctx, func(g Gomega) { + poolName := s.Shoot.Spec.Provider.Workers[0].Name + + machineDeployments := &machinev1alpha1.MachineDeploymentList{} + g.Expect(s.SeedClient.List(ctx, machineDeployments, client.InNamespace(s.Shoot.Status.TechnicalID), client.MatchingLabels{"worker.gardener.cloud/pool": poolName})).To(Succeed()) + g.Expect(machineDeployments.Items).To(HaveLen(1), "expected exactly one MachineDeployment for worker pool %s", poolName) + + machineDeployment := &machineDeployments.Items[0] + + machineSetList := &machinev1alpha1.MachineSetList{} + g.Expect(s.SeedClient.List(ctx, machineSetList, client.InNamespace(s.Shoot.Status.TechnicalID))).To(Succeed()) + + ownerToMachineSets := gardenerutils.BuildOwnerToMachineSetsMap(machineSetList.Items) + machineSetListForDeployment := ownerToMachineSets[machineDeployment.Name] + g.Expect(machineSetListForDeployment).NotTo(BeEmpty(), "no MachineSets found for MachineDeployment %s", machineDeployment.Name) + g.Expect(machineSetListForDeployment).To(HaveLen(1), "expected exactly one MachineSet for MachineDeployment %s", machineDeployment.Name) + + oldMachineSetCreationTimestamp = machineSetListForDeployment[0].CreationTimestamp.Time + }).Should(Succeed()) + }, SpecTimeout(10*time.Second)) + + ItShouldAnnotateShoot(s, map[string]string{ + v1beta1constants.GardenerOperation: "rollout-workers=" + s.Shoot.Spec.Provider.Workers[0].Name, + }) + ItShouldEventuallyNotHaveOperationAnnotation(s.GardenKomega, s.Shoot) + + It("Should fetch new MachineSet creation timestamp and ensure it's newer", func(ctx SpecContext) { + Eventually(ctx, func(g Gomega) { + poolName := s.Shoot.Spec.Provider.Workers[0].Name + + machineDeployments := &machinev1alpha1.MachineDeploymentList{} + g.Expect(s.SeedClient.List(ctx, machineDeployments, client.InNamespace(s.Shoot.Status.TechnicalID), client.MatchingLabels{"worker.gardener.cloud/pool": poolName})).To(Succeed()) + g.Expect(machineDeployments.Items).To(HaveLen(1), "expected exactly one MachineDeployment for worker pool %s", poolName) + + machineDeployment := &machineDeployments.Items[0] + + machineSetList := &machinev1alpha1.MachineSetList{} + g.Expect(s.SeedClient.List(ctx, machineSetList, client.InNamespace(s.Shoot.Status.TechnicalID))).To(Succeed()) + + ownerToMachineSets := gardenerutils.BuildOwnerToMachineSetsMap(machineSetList.Items) + machineSetListForDeployment := ownerToMachineSets[machineDeployment.Name] + g.Expect(machineSetListForDeployment).NotTo(BeEmpty(), "no MachineSets found for MachineDeployment %s", machineDeployment.Name) + g.Expect(machineSetListForDeployment).To(HaveLen(1), "expected exactly one MachineSet for MachineDeployment %s", machineDeployment.Name) + + newMachineSetCreationTimestamp := machineSetListForDeployment[0].CreationTimestamp.Time + + g.Expect(oldMachineSetCreationTimestamp.Before(newMachineSetCreationTimestamp)).To(BeTrue(), "new MachineSet creation timestamp should be newer than the old one") + }).Should(Succeed()) + }, SpecTimeout(5*time.Minute)) + + ItShouldWaitForShootToBeReconciledAndHealthy(s) +} + var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() { Describe("Create Shoot, Rotate Credentials and Delete Shoot", Label("credentials-rotation"), func() { - test := func(s *ShootContext, withoutWorkersRollout, inPlaceUpdate bool) { + test := func(s *ShootContext, withoutWorkersRollout, inPlaceUpdate, workersRollout bool) { ItShouldCreateShoot(s) ItShouldWaitForShootToBeReconciledAndHealthy(s) ItShouldInitializeShootClient(s) @@ -360,6 +422,10 @@ var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() { } } + if workersRollout { + testManualWorkersRollout(s) + } + ItShouldDeleteShoot(s) ItShouldWaitForShootToBeDeleted(s) } @@ -411,7 +477,7 @@ var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() { Context("Shoot with workers", Label("basic"), func() { Context("with workers rollout", Label("with-workers-rollout"), Ordered, func() { - test(NewTestContext().ForShoot(DefaultShoot("e2e-rotate")), false, false) + test(NewTestContext().ForShoot(DefaultShoot("e2e-rotate")), false, false, false) }) Context("with workers rollout, in-place update strategy", Label("with-workers-rollout", "in-place"), Ordered, func() { @@ -440,7 +506,7 @@ var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() { return } - test(s, false, true) + test(s, false, true, false) }) Context("without workers rollout", Label("without-workers-rollout"), Ordered, func() { @@ -455,7 +521,7 @@ var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() { s = NewTestContext().ForShoot(shoot) }) - test(s, true, false) + test(s, true, false, true) }) Context("without workers rollout, in-place update strategy", Label("without-workers-rollout", "in-place"), Ordered, func() { @@ -488,12 +554,12 @@ var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() { return } - test(s, true, true) + test(s, true, true, false) }) }) Context("Workerless Shoot", Label("workerless"), Ordered, func() { - test(NewTestContext().ForShoot(DefaultWorkerlessShoot("e2e-rotate")), false, false) + test(NewTestContext().ForShoot(DefaultWorkerlessShoot("e2e-rotate")), false, false, false) }) // TODO(AleksandarSavchev): Remove this e2e test when the k8s version for the default shoots is >= 1.34. From 9d8d9927af76aacb4ff87c09607c79a1aa5819a5 Mon Sep 17 00:00:00 2001 From: Axel Siebenborn Date: Wed, 5 Nov 2025 19:00:28 +0100 Subject: [PATCH 046/176] Allow migration from dual-stack [IPv4, IPv6] to single-stack [IPv4]. (#12967) * Allow migration from dual-stack IPv4,IPv6 to single-stack IPv4. * Address review comments * Add documentation * Fix golangci-lint finding. * Fix format. --- .../dual-stack-networking-migration.md | 50 ++++++++ pkg/apis/core/validation/shoot.go | 9 ++ pkg/apis/core/validation/shoot_test.go | 20 ++- pkg/apis/extensions/validation/network.go | 8 ++ .../extensions/validation/network_test.go | 24 +++- .../operation/botanist/dualstackmigration.go | 114 ++++++++++++------ .../botanist/dualstackmigration_test.go | 44 ++++++- pkg/gardenlet/operation/shoot/shoot.go | 52 +++++--- pkg/gardenlet/operation/shoot/shoot_test.go | 44 +++++++ plugin/pkg/shoot/mutator/admission.go | 2 +- 10 files changed, 302 insertions(+), 65 deletions(-) diff --git a/docs/usage/networking/dual-stack-networking-migration.md b/docs/usage/networking/dual-stack-networking-migration.md index e209b509933..e65e02b0081 100644 --- a/docs/usage/networking/dual-stack-networking-migration.md +++ b/docs/usage/networking/dual-stack-networking-migration.md @@ -94,3 +94,53 @@ After completing the migration: - Existing pods will only receive a second IP address upon recreation. - If full dual-stack networking is required, all pods need to be rolled. - Existing services remain IPv4-only until recreated with dual-stack configuration. + + +# Migrate from Dual-Stack to Single-Stack IPv4 + +In general, it should not be necessary to migrate from dual-stack to single-stack IPv4. However, if unforeseen problems arise, clusters can be migrated back to single-stack networking. This migration is possible even if the original dual-stack migration is not fully completed. + +## Important Considerations + +**⚠️ Partial Reversibility**: The migration back to single-stack is not a complete reversal of the dual-stack migration: + +- **Infrastructure remnants**: Not all infrastructure changes can be reverted. IPv6 addresses will still be present in the shoot status, representing IPv6 addresses of infrastructure resources that were created during the dual-stack migration. +- **Pod range assignment**: Depending on the infrastructure provider, the method for pod range assignment may differ compared to IPv4 shoots that were never migrated to dual-stack. + +## Migration Process + +The migration back to single-stack follows a similar pattern to the dual-stack migration but in reverse: + +### Step 1: Update Networking Configuration + +Remove IPv6 from the `spec.networking.ipFamilies` field. Change the configuration from `[IPv4, IPv6]` back to `[IPv4]`. + +```yaml +spec: + ... + networking: + ipFamilies: + - IPv4 # Remove IPv6 entry + ... +``` + +The shoot will reconcile automatically after the configuration change. Control plane components will be reconfigured to IPv4 single-stack mode. During this step, the `DualStackNodesMigrationReady` constraint is set to `true`, indicating that nodes and pods still have IPv6 addresses but the control plane is ready for the migration. + +### Step 2: Node Rollout + +After the control plane reconfiguration, all nodes must be rolled to remove IPv6 configuration and addresses. + +**Note**: Nodes and pods will continue to have IPv6 addresses until they are rolled/recreated. + +After rolling all nodes, the `DualStackNodesMigrationReady` constraint will be removed automatically during the next reconciliation. At this point: +- All nodes will have only IPv4 addresses +- All pods will have only IPv4 addresses +- The cluster operates in single-stack IPv4 mode + +## Post-Migration State + +After completing the migration back to single-stack: +- The cluster operates with IPv4-only networking +- New pods receive only IPv4 addresses +- Services operate with IPv4 cluster IPs only +- Some infrastructure resources may retain IPv6 addresses in their metadata (as noted in the considerations above) \ No newline at end of file diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index f3c475ad171..93f6b55d465 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -834,6 +834,7 @@ func validateNetworkingUpdate(newNetworking, oldNetworking *core.Networking, fld func validateIPFamiliesUpdate(newIPFamilies, oldIPFamilies []core.IPFamily, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} + // Allow migration from IPv4 to dual-stack if len(oldIPFamilies) == 1 && oldIPFamilies[0] == core.IPFamilyIPv4 && len(newIPFamilies) == 2 && @@ -842,6 +843,14 @@ func validateIPFamiliesUpdate(newIPFamilies, oldIPFamilies []core.IPFamily, fldP return allErrs } + // Allow migration from dual-stack to IPv4 + if len(oldIPFamilies) == 2 && + oldIPFamilies[0] == core.IPFamilyIPv4 && + oldIPFamilies[1] == core.IPFamilyIPv6 && + len(newIPFamilies) == 1 && newIPFamilies[0] == core.IPFamilyIPv4 { + return allErrs + } + if !apiequality.Semantic.DeepEqual(newIPFamilies, oldIPFamilies) { allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("unsupported IP family update: oldIPFamilies=%v, newIPFamilies=%v", oldIPFamilies, newIPFamilies))) diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index cad37c2c1b8..c4cec75e980 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -4591,7 +4591,7 @@ var _ = Describe("Shoot Validation Tests", func() { }) }) - It("should allow updating ipfamilies from IPv4 to dual-stack [IPv4 IPv6]", func() { + It("should allow updating ipfamilies from IPv4 to dual-stack [IPv4, IPv6]", func() { shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv4} newShoot := prepareShootForUpdate(shoot) @@ -4600,7 +4600,7 @@ var _ = Describe("Shoot Validation Tests", func() { errorList := ValidateShootUpdate(newShoot, shoot) Expect(errorList).To(BeEmpty()) }) - It("should forbid changing ipfamilies from IPv6 to dual-stack [IPv6 IPv4]", func() { + It("should forbid changing ipfamilies from IPv6 to dual-stack [IPv6, IPv4]", func() { shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6} newShoot := prepareShootForUpdate(shoot) newShoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6, core.IPFamilyIPv4} @@ -4611,7 +4611,7 @@ var _ = Describe("Shoot Validation Tests", func() { "Detail": Equal("unsupported IP family update: oldIPFamilies=[IPv6], newIPFamilies=[IPv6 IPv4]"), })))) }) - It("should forbid changing ipfamilies from single-stack IPv4 to dual-stack [IPv6 IPv4]", func() { + It("should forbid changing ipfamilies from single-stack IPv4 to dual-stack [IPv6, IPv4]", func() { shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv4} newShoot := prepareShootForUpdate(shoot) newShoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6, core.IPFamilyIPv4} @@ -4622,18 +4622,26 @@ var _ = Describe("Shoot Validation Tests", func() { "Detail": Equal("unsupported IP family update: oldIPFamilies=[IPv4], newIPFamilies=[IPv6 IPv4]"), })))) }) - It("should forbid changing ipfamilies from dual-stack to single-stack", func() { + It("should forbid changing ipfamilies from dual-stack [IPv4, IPv6] to single-stack [IPv6]", func() { shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv4, core.IPFamilyIPv6} newShoot := prepareShootForUpdate(shoot) - newShoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv4} + newShoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv6} errorList := ValidateShootUpdate(newShoot, shoot) Expect(errorList).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.networking.ipFamilies"), - "Detail": Equal("unsupported IP family update: oldIPFamilies=[IPv4 IPv6], newIPFamilies=[IPv4]"), + "Detail": Equal("unsupported IP family update: oldIPFamilies=[IPv4 IPv6], newIPFamilies=[IPv6]"), })))) }) + It("should allow changing ipfamilies from dual-stack [IPv4, IPv6] to single-stack [IPv4]", func() { + shoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv4, core.IPFamilyIPv6} + + newShoot := prepareShootForUpdate(shoot) + newShoot.Spec.Networking.IPFamilies = []core.IPFamily{core.IPFamilyIPv4} + errorList := ValidateShootUpdate(newShoot, shoot) + Expect(errorList).To(BeEmpty()) + }) }) Context("dual-stack", func() { diff --git a/pkg/apis/extensions/validation/network.go b/pkg/apis/extensions/validation/network.go index 6e411b37362..8c37aec788e 100644 --- a/pkg/apis/extensions/validation/network.go +++ b/pkg/apis/extensions/validation/network.go @@ -141,6 +141,14 @@ func ValidateIPFamiliesUpdate(newIPFamilies, oldIPFamilies []extensionsv1alpha1. return allErrs } + if len(oldIPFamilies) == 2 && + oldIPFamilies[0] == extensionsv1alpha1.IPFamilyIPv4 && + oldIPFamilies[1] == extensionsv1alpha1.IPFamilyIPv6 && + len(newIPFamilies) == 1 && + newIPFamilies[0] == extensionsv1alpha1.IPFamilyIPv4 { + return allErrs + } + if !apiequality.Semantic.DeepEqual(newIPFamilies, oldIPFamilies) { allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("unsupported IP family update: oldIPFamilies=%v, newIPFamilies=%v", oldIPFamilies, newIPFamilies))) diff --git a/pkg/apis/extensions/validation/network_test.go b/pkg/apis/extensions/validation/network_test.go index 86e1eefdae6..64b1c545070 100644 --- a/pkg/apis/extensions/validation/network_test.go +++ b/pkg/apis/extensions/validation/network_test.go @@ -314,10 +314,10 @@ var _ = Describe("Network validation tests", func() { )) }) - It("should not allow removing an address family", func() { + It("should not allow to update from [IPv4, IPv6] to [IPv6]", func() { network.Spec.IPFamilies = []extensionsv1alpha1.IPFamily{extensionsv1alpha1.IPFamilyIPv4, extensionsv1alpha1.IPFamilyIPv6} newNetwork := prepareNetworkForUpdate(network) - newNetwork.Spec.IPFamilies = []extensionsv1alpha1.IPFamily{extensionsv1alpha1.IPFamilyIPv4} + newNetwork.Spec.IPFamilies = []extensionsv1alpha1.IPFamily{extensionsv1alpha1.IPFamilyIPv6} errorList := ValidateNetworkUpdate(newNetwork, network) @@ -326,8 +326,28 @@ var _ = Describe("Network validation tests", func() { "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.ipFamilies"), })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.podCIDR"), + "Detail": Equal("must be a valid IPv6 address"), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.serviceCIDR"), + "Detail": Equal("must be a valid IPv6 address"), + })), )) }) + + It("should allow to update from [IPv4, IPv6] to [IPv4]", func() { + network.Spec.IPFamilies = []extensionsv1alpha1.IPFamily{extensionsv1alpha1.IPFamilyIPv4, extensionsv1alpha1.IPFamilyIPv6} + newNetwork := prepareNetworkForUpdate(network) + newNetwork.Spec.IPFamilies = []extensionsv1alpha1.IPFamily{extensionsv1alpha1.IPFamilyIPv4} + + errorList := ValidateNetworkUpdate(newNetwork, network) + + Expect(errorList).To(BeEmpty()) + }) }) Describe("#ValidateIPFamilies", func() { diff --git a/pkg/gardenlet/operation/botanist/dualstackmigration.go b/pkg/gardenlet/operation/botanist/dualstackmigration.go index 6863e11ee99..ee3cae5d4c4 100644 --- a/pkg/gardenlet/operation/botanist/dualstackmigration.go +++ b/pkg/gardenlet/operation/botanist/dualstackmigration.go @@ -17,42 +17,78 @@ import ( corednsconstants "github.com/gardener/gardener/pkg/component/networking/coredns/constants" ) +func nodesMigrated(nodeList *corev1.NodeList, ipFamiliesLen int) bool { + if nodeList == nil || len(nodeList.Items) == 0 { + return false + } + for _, node := range nodeList.Items { + if len(node.Spec.PodCIDRs) != ipFamiliesLen { + return false + } + } + return true +} + // DetermineUpdateFunction determines the update function for the shoot's status based on dual-stack migration readiness. func (b *Botanist) DetermineUpdateFunction(networkReadyForDualStackMigration bool, nodeList *corev1.NodeList) func(*gardencorev1beta1.Shoot) error { + shootIPFamilies := b.Shoot.GetInfo().Spec.Networking.IPFamilies + isSingleStack := len(shootIPFamilies) == 1 + allNodesMigrated := nodesMigrated(nodeList, len(shootIPFamilies)) + if networkReadyForDualStackMigration { - return func(shoot *gardencorev1beta1.Shoot) error { - shoot.Status.Constraints = v1beta1helper.RemoveConditions(shoot.Status.Constraints, gardencorev1beta1.ShootDualStackNodesMigrationReady) - constraint := v1beta1helper.GetCondition(shoot.Status.Constraints, gardencorev1beta1.ShootDNSServiceMigrationReady) - if constraint == nil { - constraint := v1beta1helper.GetOrInitConditionWithClock(b.Clock, shoot.Status.Constraints, gardencorev1beta1.ShootDNSServiceMigrationReady) - constraint = v1beta1helper.UpdatedConditionWithClock(b.Clock, constraint, gardencorev1beta1.ConditionProgressing, "DNSServiceMigration", "The shoot is migrating the kube-dns service.") - shoot.Status.Constraints = v1beta1helper.MergeConditions(shoot.Status.Constraints, constraint) - } - return nil - } + return b.createConstraintRemovalFunction(isSingleStack, allNodesMigrated) } + return b.createNodeMigrationFunction(isSingleStack, allNodesMigrated) +} - allNodesDualStack := true - conditionStatus := gardencorev1beta1.ConditionProgressing - conditionReason := "NodesNotMigrated" - conditionMessage := "The shoot is migrating to dual-stack networking." - for _, node := range nodeList.Items { - allNodesDualStack = allNodesDualStack && len(node.Spec.PodCIDRs) == 2 +func (b *Botanist) createConstraintRemovalFunction(isSingleStack, allNodesMigrated bool) func(*gardencorev1beta1.Shoot) error { + return func(shoot *gardencorev1beta1.Shoot) error { + if isSingleStack && !allNodesMigrated { + return nil // Don't remove constraint yet + } + + shoot.Status.Constraints = v1beta1helper.RemoveConditions(shoot.Status.Constraints, gardencorev1beta1.ShootDualStackNodesMigrationReady) + + if !isSingleStack && !b.hasDNSMigrationConstraint(shoot) { + b.addDNSMigrationConstraint(shoot) + } + return nil } - if allNodesDualStack { - conditionStatus = gardencorev1beta1.ConditionTrue - conditionReason = "NodesMigrated" - conditionMessage = "All nodes were migrated to dual-stack networking." +} + +func (b *Botanist) createNodeMigrationFunction(isSingleStack, allNodesMigrated bool) func(*gardencorev1beta1.Shoot) error { + status := gardencorev1beta1.ConditionProgressing + reason := "NodesNotMigrated" + message := "Migrating node pod CIDRs to match target network stack." + + if allNodesMigrated { + status = gardencorev1beta1.ConditionTrue + reason = "NodesMigrated" + message = "All node pod CIDRs migrated to target network stack." + } else if isSingleStack { + status = gardencorev1beta1.ConditionTrue + reason = "NodesMigrating" + message = "Node pod CIDRs are currently migrated to target network stack." } return func(shoot *gardencorev1beta1.Shoot) error { constraint := v1beta1helper.GetOrInitConditionWithClock(b.Clock, shoot.Status.Constraints, gardencorev1beta1.ShootDualStackNodesMigrationReady) - constraint = v1beta1helper.UpdatedConditionWithClock(b.Clock, constraint, conditionStatus, conditionReason, conditionMessage) + constraint = v1beta1helper.UpdatedConditionWithClock(b.Clock, constraint, status, reason, message) shoot.Status.Constraints = v1beta1helper.MergeConditions(shoot.Status.Constraints, constraint) return nil } } +func (b *Botanist) hasDNSMigrationConstraint(shoot *gardencorev1beta1.Shoot) bool { + return v1beta1helper.GetCondition(shoot.Status.Constraints, gardencorev1beta1.ShootDNSServiceMigrationReady) != nil +} + +func (b *Botanist) addDNSMigrationConstraint(shoot *gardencorev1beta1.Shoot) { + constraint := v1beta1helper.GetOrInitConditionWithClock(b.Clock, shoot.Status.Constraints, gardencorev1beta1.ShootDNSServiceMigrationReady) + constraint = v1beta1helper.UpdatedConditionWithClock(b.Clock, constraint, gardencorev1beta1.ConditionProgressing, "DNSServiceMigration", "The shoot is migrating the kube-dns service.") + shoot.Status.Constraints = v1beta1helper.MergeConditions(shoot.Status.Constraints, constraint) +} + // DetermineUpdateFunctionDNS determines the update function for the shoot's status based on DNS service and pod readiness. func (b *Botanist) DetermineUpdateFunctionDNS(svcReady, podsReady bool) func(*gardencorev1beta1.Shoot) error { if svcReady { @@ -92,7 +128,7 @@ func (b *Botanist) CheckDNSServiceMigration(ctx context.Context) error { return err } - svcReady := len(service.Spec.ClusterIPs) == 2 + svcReady := len(service.Spec.ClusterIPs) == len(b.Shoot.GetInfo().Spec.Networking.IPFamilies) podList := &corev1.PodList{} if err := b.ShootClientSet.Client().List(ctx, podList, client.InNamespace(metav1.NamespaceSystem), client.MatchingLabels{corednsconstants.LabelKey: corednsconstants.LabelValue}); err != nil { @@ -105,12 +141,12 @@ func (b *Botanist) CheckDNSServiceMigration(ctx context.Context) error { podsReady := len(podList.Items) != 0 for _, pod := range podList.Items { - podsReady = podsReady && len(pod.Status.PodIPs) == 2 + podsReady = podsReady && len(pod.Status.PodIPs) == len(b.Shoot.GetInfo().Spec.Networking.IPFamilies) } updateFunction := b.DetermineUpdateFunctionDNS(svcReady, podsReady) if err := b.Shoot.UpdateInfoStatus(ctx, b.GardenClient, true, false, updateFunction); err != nil { - return fmt.Errorf("failed to update shoot info status during dual-stack migration: %w", err) + return fmt.Errorf("failed to update shoot info status during network stack migration: %w", err) } return nil } @@ -138,13 +174,13 @@ func (b *Botanist) CheckPodCIDRsInNodes(ctx context.Context) error { nodeList := &corev1.NodeList{} if err := b.ShootClientSet.Client().List(ctx, nodeList); err != nil { - return fmt.Errorf("failed to list nodes during dual-stack migration: %w", err) + return fmt.Errorf("failed to list nodes during network stack migration: %w", err) } - networkReadyForDualStackMigration := len(network.Status.IPFamilies) == 2 + networkReadyForDualStackMigration := len(network.Status.IPFamilies) == len(b.Shoot.GetInfo().Spec.Networking.IPFamilies) updateFunction := b.DetermineUpdateFunction(networkReadyForDualStackMigration, nodeList) if err := b.Shoot.UpdateInfoStatus(ctx, b.GardenClient, true, false, updateFunction); err != nil { - return fmt.Errorf("failed to update shoot info status during dual-stack migration: %w", err) + return fmt.Errorf("failed to update shoot info status during network stack migration: %w", err) } return nil @@ -154,17 +190,23 @@ func (b *Botanist) CheckPodCIDRsInNodes(ctx context.Context) error { func (b *Botanist) UpdateDualStackMigrationConditionIfNeeded(ctx context.Context) error { shoot := b.Shoot.GetInfo() - constraint := v1beta1helper.GetCondition(shoot.Status.Constraints, gardencorev1beta1.ShootDualStackNodesMigrationReady) - if constraint == nil && len(shoot.Spec.Networking.IPFamilies) == 2 && shoot.Status.Networking != nil && len(shoot.Status.Networking.Nodes) == 1 { - if err := b.Shoot.UpdateInfoStatus(ctx, b.GardenClient, true, false, func(shoot *gardencorev1beta1.Shoot) error { - constraint := v1beta1helper.GetOrInitConditionWithClock(b.Clock, shoot.Status.Constraints, gardencorev1beta1.ShootDualStackNodesMigrationReady) - constraint = v1beta1helper.UpdatedConditionWithClock(b.Clock, constraint, gardencorev1beta1.ConditionProgressing, "DualStackMigration", "The shoot is migrating to dual-stack networking.") - shoot.Status.Constraints = v1beta1helper.MergeConditions(shoot.Status.Constraints, constraint) - return nil - }); err != nil { - return fmt.Errorf("failed updating %s constraint in shoot status: %w", gardencorev1beta1.ShootDualStackNodesMigrationReady, err) + nodeList := &corev1.NodeList{} + if b.ShootClientSet != nil { + if err := b.ShootClientSet.Client().List(ctx, nodeList); err != nil { + return fmt.Errorf("failed to list nodes during network stack migration: %w", err) } } + if constraint := v1beta1helper.GetCondition(shoot.Status.Constraints, gardencorev1beta1.ShootDualStackNodesMigrationReady); constraint == nil { + network, _ := b.Shoot.Components.Extensions.Network.Get(ctx) + if network == nil { + return nil // Network not yet created, nothing to do + } + networkReadyForDualStackMigration := len(network.Status.IPFamilies) == len(b.Shoot.GetInfo().Spec.Networking.IPFamilies) + updateFunction := b.DetermineUpdateFunction(networkReadyForDualStackMigration, nodeList) + if err := b.Shoot.UpdateInfoStatus(ctx, b.GardenClient, true, false, updateFunction); err != nil { + return fmt.Errorf("failed updating %s constraint in shoot status: %w", gardencorev1beta1.ShootDualStackNodesMigrationReady, err) + } + } return nil } diff --git a/pkg/gardenlet/operation/botanist/dualstackmigration_test.go b/pkg/gardenlet/operation/botanist/dualstackmigration_test.go index e05971f3bd5..b71412e2089 100644 --- a/pkg/gardenlet/operation/botanist/dualstackmigration_test.go +++ b/pkg/gardenlet/operation/botanist/dualstackmigration_test.go @@ -38,6 +38,22 @@ var _ = Describe("DualStackMigration", func() { }, Clock: mockClock, }} + + // Initialize the shoot with proper IP families for dual-stack tests + botanist.Shoot.SetInfo(&gardencorev1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shoot", + Namespace: "test-namespace", + }, + Spec: gardencorev1beta1.ShootSpec{ + Networking: &gardencorev1beta1.Networking{ + IPFamilies: []gardencorev1beta1.IPFamily{ + gardencorev1beta1.IPFamilyIPv4, + gardencorev1beta1.IPFamilyIPv6, + }, + }, + }, + }) }) Describe("#DetermineUpdateFunction", func() { @@ -54,6 +70,14 @@ var _ = Describe("DualStackMigration", func() { Namespace: "test-namespace", Annotations: map[string]string{}, }, + Spec: gardencorev1beta1.ShootSpec{ + Networking: &gardencorev1beta1.Networking{ + IPFamilies: []gardencorev1beta1.IPFamily{ + gardencorev1beta1.IPFamilyIPv4, + gardencorev1beta1.IPFamilyIPv6, + }, + }, + }, Status: gardencorev1beta1.ShootStatus{ Constraints: []gardencorev1beta1.Condition{ v1beta1helper.InitConditionWithClock(mockClock, gardencorev1beta1.ShootDualStackNodesMigrationReady), @@ -80,6 +104,14 @@ var _ = Describe("DualStackMigration", func() { } shoot := &gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Networking: &gardencorev1beta1.Networking{ + IPFamilies: []gardencorev1beta1.IPFamily{ + gardencorev1beta1.IPFamilyIPv4, + gardencorev1beta1.IPFamilyIPv6, + }, + }, + }, Status: gardencorev1beta1.ShootStatus{ Constraints: []gardencorev1beta1.Condition{}, }, @@ -93,7 +125,7 @@ var _ = Describe("DualStackMigration", func() { Expect(condition).NotTo(BeNil()) Expect(condition.Status).To(Equal(gardencorev1beta1.ConditionTrue)) Expect(condition.Reason).To(Equal("NodesMigrated")) - Expect(condition.Message).To(Equal("All nodes were migrated to dual-stack networking.")) + Expect(condition.Message).To(Equal("All node pod CIDRs migrated to target network stack.")) // Updated message }) It("Updates the constraint to ConditionFalse when not all nodes are dual-stack", func() { @@ -105,6 +137,14 @@ var _ = Describe("DualStackMigration", func() { } shoot := &gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Networking: &gardencorev1beta1.Networking{ + IPFamilies: []gardencorev1beta1.IPFamily{ + gardencorev1beta1.IPFamilyIPv4, + gardencorev1beta1.IPFamilyIPv6, + }, + }, + }, Status: gardencorev1beta1.ShootStatus{ Constraints: []gardencorev1beta1.Condition{}, }, @@ -118,7 +158,7 @@ var _ = Describe("DualStackMigration", func() { Expect(condition).NotTo(BeNil()) Expect(condition.Status).To(Equal(gardencorev1beta1.ConditionProgressing)) Expect(condition.Reason).To(Equal("NodesNotMigrated")) - Expect(condition.Message).To(Equal("The shoot is migrating to dual-stack networking.")) + Expect(condition.Message).To(Equal("Migrating node pod CIDRs to match target network stack.")) // Updated message }) It("Updates the constraint to ConditionProgressing if coredns pods not are migrated", func() { diff --git a/pkg/gardenlet/operation/shoot/shoot.go b/pkg/gardenlet/operation/shoot/shoot.go index 1be74c87d84..56c49395f13 100644 --- a/pkg/gardenlet/operation/shoot/shoot.go +++ b/pkg/gardenlet/operation/shoot/shoot.go @@ -521,21 +521,35 @@ func (s *Shoot) IsShootControlPlaneLoggingEnabled(c *gardenletconfigv1alpha1.Gar return s.Purpose != gardencorev1beta1.ShootPurposeTesting && gardenlethelper.IsLoggingEnabled(c) } -func sortByIPFamilies(ipfamilies []gardencorev1beta1.IPFamily, cidrs []net.IPNet) []net.IPNet { +// SortByIPFamilies sorts a slice of CIDRs according to the specified IP family order. +// For dual-stack configurations, CIDRs are ordered by the IP family preference. +// For single-stack configurations, matching CIDRs are placed first, followed by non-matching CIDRs. +func SortByIPFamilies(ipfamilies []gardencorev1beta1.IPFamily, cidrs []net.IPNet) []net.IPNet { + if len(ipfamilies) == 0 || len(cidrs) == 0 { + return cidrs + } + var result []net.IPNet - for _, ipfamily := range ipfamilies { - switch ipfamily { - case gardencorev1beta1.IPFamilyIPv4: - for _, cidr := range cidrs { - if cidr.IP.To4() != nil { - result = append(result, cidr) - } + + // Process each IP family in order + for _, family := range ipfamilies { + for _, cidr := range cidrs { + isIPv4 := cidr.IP.To4() != nil + if (family == gardencorev1beta1.IPFamilyIPv4 && isIPv4) || + (family == gardencorev1beta1.IPFamilyIPv6 && !isIPv4) { + result = append(result, cidr) } - case gardencorev1beta1.IPFamilyIPv6: - for _, cidr := range cidrs { - if cidr.IP.To4() == nil { - result = append(result, cidr) - } + } + } + + // For single-stack, append non-matching CIDRs at the end + if len(ipfamilies) == 1 { + primary := ipfamilies[0] + for _, cidr := range cidrs { + isIPv4 := cidr.IP.To4() != nil + if (primary == gardencorev1beta1.IPFamilyIPv4 && !isIPv4) || + (primary == gardencorev1beta1.IPFamilyIPv6 && isIPv4) { + result = append(result, cidr) } } } @@ -595,28 +609,30 @@ func ToNetworks(shoot *gardencorev1beta1.Shoot, workerless bool) (*Networks, err if result, err := copyUniqueCIDRs(shoot.Status.Networking.Pods, pods, "pod"); err != nil { return nil, err } else { - pods = sortByIPFamilies(shoot.Spec.Networking.IPFamilies, result) + pods = SortByIPFamilies(shoot.Spec.Networking.IPFamilies, result) } if result, err := copyUniqueCIDRs(shoot.Status.Networking.Services, services, "service"); err != nil { return nil, err } else { - services = sortByIPFamilies(shoot.Spec.Networking.IPFamilies, result) + services = SortByIPFamilies(shoot.Spec.Networking.IPFamilies, result) } if result, err := copyUniqueCIDRs(shoot.Status.Networking.Nodes, nodes, "node"); err != nil { return nil, err } else { - nodes = sortByIPFamilies(shoot.Spec.Networking.IPFamilies, result) + nodes = SortByIPFamilies(shoot.Spec.Networking.IPFamilies, result) } if result, err := copyUniqueCIDRs(shoot.Status.Networking.EgressCIDRs, egressCIDRs, "egressCIDRs"); err != nil { return nil, err } else { - egressCIDRs = sortByIPFamilies(shoot.Spec.Networking.IPFamilies, result) + egressCIDRs = SortByIPFamilies(shoot.Spec.Networking.IPFamilies, result) } } // During dual-stack migration, until nodes are migrated to dual-stack, we only use the primary addresses. condition := v1beta1helper.GetCondition(shoot.Status.Constraints, gardencorev1beta1.ShootDualStackNodesMigrationReady) - if condition != nil && condition.Status != gardencorev1beta1.ConditionTrue { + if condition != nil && + ((condition.Status != gardencorev1beta1.ConditionTrue && len(shoot.Spec.Networking.IPFamilies) == 2) || + (condition.Status == gardencorev1beta1.ConditionTrue && len(shoot.Spec.Networking.IPFamilies) == 1)) { nodes = getPrimaryCIDRs(nodes, shoot.Spec.Networking.IPFamilies) services = getPrimaryCIDRs(services, shoot.Spec.Networking.IPFamilies) pods = getPrimaryCIDRs(pods, shoot.Spec.Networking.IPFamilies) diff --git a/pkg/gardenlet/operation/shoot/shoot_test.go b/pkg/gardenlet/operation/shoot/shoot_test.go index 0410186f018..48b58340721 100644 --- a/pkg/gardenlet/operation/shoot/shoot_test.go +++ b/pkg/gardenlet/operation/shoot/shoot_test.go @@ -320,5 +320,49 @@ var _ = Describe("shoot", func() { Expect(shoot.HasManagedInfrastructure()).To(BeTrue()) }) }) + + Describe("#SortByIPFamilies", func() { + var ( + ipv4CIDR1 = net.IPNet{IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(24, 32)} + ipv4CIDR2 = net.IPNet{IP: net.ParseIP("192.168.1.0"), Mask: net.CIDRMask(24, 32)} + ipv6CIDR1 = net.IPNet{IP: net.ParseIP("2001:db8::"), Mask: net.CIDRMask(64, 128)} + ipv6CIDR2 = net.IPNet{IP: net.ParseIP("fd00::"), Mask: net.CIDRMask(48, 128)} + ) + + It("should sort CIDRs for single-stack IPv4 with IPv4 first", func() { + cidrs := []net.IPNet{ipv6CIDR1, ipv4CIDR1, ipv4CIDR2} + result := SortByIPFamilies([]gardencorev1beta1.IPFamily{gardencorev1beta1.IPFamilyIPv4}, cidrs) + expected := []net.IPNet{ipv4CIDR1, ipv4CIDR2, ipv6CIDR1} + Expect(result).To(Equal(expected)) + }) + + It("should sort CIDRs for single-stack IPv6 with IPv6 first", func() { + cidrs := []net.IPNet{ipv4CIDR1, ipv6CIDR1, ipv6CIDR2} + result := SortByIPFamilies([]gardencorev1beta1.IPFamily{gardencorev1beta1.IPFamilyIPv6}, cidrs) + expected := []net.IPNet{ipv6CIDR1, ipv6CIDR2, ipv4CIDR1} + Expect(result).To(Equal(expected)) + }) + + It("should sort CIDRs for dual-stack IPv4,IPv6", func() { + cidrs := []net.IPNet{ipv6CIDR1, ipv4CIDR1, ipv6CIDR2, ipv4CIDR2} + result := SortByIPFamilies([]gardencorev1beta1.IPFamily{gardencorev1beta1.IPFamilyIPv4, gardencorev1beta1.IPFamilyIPv6}, cidrs) + expected := []net.IPNet{ipv4CIDR1, ipv4CIDR2, ipv6CIDR1, ipv6CIDR2} + Expect(result).To(Equal(expected)) + }) + + It("should sort CIDRs for dual-stack IPv6,IPv4", func() { + cidrs := []net.IPNet{ipv4CIDR1, ipv6CIDR1, ipv4CIDR2, ipv6CIDR2} + result := SortByIPFamilies([]gardencorev1beta1.IPFamily{gardencorev1beta1.IPFamilyIPv6, gardencorev1beta1.IPFamilyIPv4}, cidrs) + expected := []net.IPNet{ipv6CIDR1, ipv6CIDR2, ipv4CIDR1, ipv4CIDR2} + Expect(result).To(Equal(expected)) + }) + + It("should handle mixed CIDRs with single-stack IPv4", func() { + cidrs := []net.IPNet{ipv6CIDR1, ipv4CIDR1} + result := SortByIPFamilies([]gardencorev1beta1.IPFamily{gardencorev1beta1.IPFamilyIPv4}, cidrs) + expected := []net.IPNet{ipv4CIDR1, ipv6CIDR1} + Expect(result).To(Equal(expected)) + }) + }) }) }) diff --git a/plugin/pkg/shoot/mutator/admission.go b/plugin/pkg/shoot/mutator/admission.go index 33cefdda30b..a4a3ef6a056 100644 --- a/plugin/pkg/shoot/mutator/admission.go +++ b/plugin/pkg/shoot/mutator/admission.go @@ -204,7 +204,7 @@ func (c *mutationContext) addMetadataAnnotations(a admission.Attributes) { } if !reflect.DeepEqual(c.oldShoot.Spec.Provider.InfrastructureConfig, c.shoot.Spec.Provider.InfrastructureConfig) || - c.oldShoot.Spec.Networking != nil && c.oldShoot.Spec.Networking.IPFamilies != nil && !reflect.DeepEqual(c.oldShoot.Spec.Networking.IPFamilies, c.shoot.Spec.Networking.IPFamilies) { + c.oldShoot.Spec.Networking != nil && c.oldShoot.Spec.Networking.IPFamilies != nil && len(c.oldShoot.Spec.Networking.IPFamilies) < len(c.shoot.Spec.Networking.IPFamilies) { addInfrastructureDeploymentTask(c.shoot) } From 496c8af9145728ba9d38ccbbf8d2754738d89ea9 Mon Sep 17 00:00:00 2001 From: Johannes Scheerer Date: Wed, 5 Nov 2025 22:33:17 +0100 Subject: [PATCH 047/176] Fix flaky `gardener-resource-manager` unit test by using old object tracker (#13368) --- .../gardener/resourcemanager/resource_manager_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/component/gardener/resourcemanager/resource_manager_test.go b/pkg/component/gardener/resourcemanager/resource_manager_test.go index c761b4ff1f1..dcb3e8f16b8 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager_test.go +++ b/pkg/component/gardener/resourcemanager/resource_manager_test.go @@ -32,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" kubernetesscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -40,6 +41,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component" . "github.com/gardener/gardener/pkg/component/gardener/resourcemanager" resourcemanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1" @@ -141,7 +143,10 @@ var _ = Describe("ResourceManager", func() { BeforeEach(func() { ctrl = gomock.NewController(GinkgoT()) c = mockclient.NewMockClient(ctrl) - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetesscheme.Scheme).Build() + // The new default object tracker coming with controller-runtime v0.22 does not seem to be thread-safe causing + // test failures with concurrent access. Use the old object tracker for the time being. + objectTracker := testing.NewObjectTracker(kubernetes.SeedScheme, kubernetesscheme.Codecs.UniversalDecoder()) + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetesscheme.Scheme).WithObjectTracker(objectTracker).Build() sm = fakesecretsmanager.New(fakeClient, deployNamespace) By("Create secrets managed outside of this package for whose secretsmanager.Get() will be called") From b6d7d12235ae05917a1b55bd107a1334f2d0b283 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 6 Nov 2025 00:21:18 +0100 Subject: [PATCH 048/176] Update module github.com/docker/cli to v28.5.2+incompatible (#13369) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 96718ac8ab4..d3bb48f7a5e 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/containerd/containerd v1.7.28 github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 - github.com/docker/cli v28.5.1+incompatible + github.com/docker/cli v28.5.2+incompatible github.com/fluent/fluent-operator/v3 v3.3.0 github.com/gardener/cert-management v0.19.0 github.com/gardener/dependency-watchdog v1.6.0 diff --git a/go.sum b/go.sum index bd38ba64b76..44eb0dfcf13 100644 --- a/go.sum +++ b/go.sum @@ -181,8 +181,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/cli v28.5.1+incompatible h1:ESutzBALAD6qyCLqbQSEf1a/U8Ybms5agw59yGVc+yY= -github.com/docker/cli v28.5.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v28.5.2+incompatible h1:XmG99IHcBmIAoC1PPg9eLBZPlTrNUAijsHLm8PjhBlg= +github.com/docker/cli v28.5.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= From 6d8cc127f7524ed094b081811988fe9ca41c8fc8 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 6 Nov 2025 02:21:18 +0100 Subject: [PATCH 049/176] Update go (#13371) --- .test-defs/CreateShoot.yaml | 2 +- .test-defs/DeleteShoot.yaml | 2 +- .test-defs/HibernateShoot.yaml | 2 +- .test-defs/MigrateShoot.yaml | 2 +- .test-defs/ReconcileShoots.yaml | 2 +- .test-defs/ShootKubernetesUpdateTest.yaml | 2 +- .test-defs/ShootMachineImageUpdateTest.yaml | 2 +- .test-defs/TestSuiteGardenerRelease.yaml | 2 +- .test-defs/TestSuiteShootBeta.yaml | 2 +- .test-defs/TestSuiteShootBetaSerial.yaml | 2 +- .test-defs/TestSuiteShootDefault.yaml | 2 +- .test-defs/TestSuiteShootDefaultSerial.yaml | 2 +- .test-defs/TestSuiteShootRelease.yaml | 2 +- .test-defs/WakeUpShoot.yaml | 2 +- Dockerfile | 2 +- hack/tools/image/variants.yaml | 4 ++-- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.test-defs/CreateShoot.yaml b/.test-defs/CreateShoot.yaml index 5855af4b867..765ad7f59c8 100644 --- a/.test-defs/CreateShoot.yaml +++ b/.test-defs/CreateShoot.yaml @@ -47,4 +47,4 @@ spec: # -machine-type=$MACHINE_TYPE # -external-domain= - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/DeleteShoot.yaml b/.test-defs/DeleteShoot.yaml index a2b93e7f74b..09b99b67699 100644 --- a/.test-defs/DeleteShoot.yaml +++ b/.test-defs/DeleteShoot.yaml @@ -16,4 +16,4 @@ spec: --shoot-name=$SHOOT_NAME --project-namespace=$PROJECT_NAMESPACE --kubecfg="$TM_KUBECONFIG_PATH/gardener.config" - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/HibernateShoot.yaml b/.test-defs/HibernateShoot.yaml index f8c0109be82..bbcc512c240 100644 --- a/.test-defs/HibernateShoot.yaml +++ b/.test-defs/HibernateShoot.yaml @@ -17,4 +17,4 @@ spec: --project-namespace=$PROJECT_NAMESPACE --kubecfg="$TM_KUBECONFIG_PATH/gardener.config" - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/MigrateShoot.yaml b/.test-defs/MigrateShoot.yaml index 80c2a7a386f..ba2f36ecadb 100644 --- a/.test-defs/MigrateShoot.yaml +++ b/.test-defs/MigrateShoot.yaml @@ -20,4 +20,4 @@ spec: -mr-exclude-list="$MR_EXCLUDE_LIST" -resources-with-generated-name="$RESOURCES_WITH_GENERATED_NAME" -add-test-run-taint="$ADD_TEST_RUN_TAINT" - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/ReconcileShoots.yaml b/.test-defs/ReconcileShoots.yaml index a08e1563c2d..2e955f73fb7 100644 --- a/.test-defs/ReconcileShoots.yaml +++ b/.test-defs/ReconcileShoots.yaml @@ -16,4 +16,4 @@ spec: -kubecfg=$TM_KUBECONFIG_PATH/gardener.config -version=$GARDENER_VERSION - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/ShootKubernetesUpdateTest.yaml b/.test-defs/ShootKubernetesUpdateTest.yaml index 68a58b8ac42..940e978147f 100644 --- a/.test-defs/ShootKubernetesUpdateTest.yaml +++ b/.test-defs/ShootKubernetesUpdateTest.yaml @@ -20,4 +20,4 @@ spec: -project-namespace=$PROJECT_NAMESPACE -version=$K8S_VERSION -version-worker-pools=$K8S_VERSION_WORKER_POOLS - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/ShootMachineImageUpdateTest.yaml b/.test-defs/ShootMachineImageUpdateTest.yaml index b4ff98440b3..ce497f3a4cd 100644 --- a/.test-defs/ShootMachineImageUpdateTest.yaml +++ b/.test-defs/ShootMachineImageUpdateTest.yaml @@ -19,4 +19,4 @@ spec: -shoot-name=$SHOOT_NAME -project-namespace=$PROJECT_NAMESPACE -machine-image-version=$MACHINE_IMAGE_VERSION - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/TestSuiteGardenerRelease.yaml b/.test-defs/TestSuiteGardenerRelease.yaml index 965524223ea..956e89e6288 100644 --- a/.test-defs/TestSuiteGardenerRelease.yaml +++ b/.test-defs/TestSuiteGardenerRelease.yaml @@ -20,4 +20,4 @@ spec: -ginkgo.focus="\[RELEASE\]" -ginkgo.skip="\[SERIAL\]" - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/TestSuiteShootBeta.yaml b/.test-defs/TestSuiteShootBeta.yaml index d9f1d0f4746..e2f065c56ad 100644 --- a/.test-defs/TestSuiteShootBeta.yaml +++ b/.test-defs/TestSuiteShootBeta.yaml @@ -22,4 +22,4 @@ spec: -ginkgo.focus="\[BETA\]" -ginkgo.skip="\[SERIAL\]" - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/TestSuiteShootBetaSerial.yaml b/.test-defs/TestSuiteShootBetaSerial.yaml index 04dacdf8266..e5eb1a97611 100644 --- a/.test-defs/TestSuiteShootBetaSerial.yaml +++ b/.test-defs/TestSuiteShootBetaSerial.yaml @@ -23,4 +23,4 @@ spec: -fenced=$FENCED -ginkgo.focus="\[BETA\].*\[SERIAL\]" - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/TestSuiteShootDefault.yaml b/.test-defs/TestSuiteShootDefault.yaml index 9242bf0a4b7..32033cceb4a 100644 --- a/.test-defs/TestSuiteShootDefault.yaml +++ b/.test-defs/TestSuiteShootDefault.yaml @@ -23,4 +23,4 @@ spec: -ginkgo.skip="\[SERIAL\]" -ginkgo.timeout=2h - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/TestSuiteShootDefaultSerial.yaml b/.test-defs/TestSuiteShootDefaultSerial.yaml index 439eb93a2ae..9e846fdebeb 100644 --- a/.test-defs/TestSuiteShootDefaultSerial.yaml +++ b/.test-defs/TestSuiteShootDefaultSerial.yaml @@ -24,4 +24,4 @@ spec: -ginkgo.focus="\[DEFAULT\].*\[SERIAL\]" -ginkgo.timeout=2h - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/TestSuiteShootRelease.yaml b/.test-defs/TestSuiteShootRelease.yaml index 18daf082273..5846b7f386f 100644 --- a/.test-defs/TestSuiteShootRelease.yaml +++ b/.test-defs/TestSuiteShootRelease.yaml @@ -22,4 +22,4 @@ spec: -ginkgo.focus="\[RELEASE\]" -ginkgo.skip="\[SERIAL\]" - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/.test-defs/WakeUpShoot.yaml b/.test-defs/WakeUpShoot.yaml index f4addc31448..00366a066cc 100644 --- a/.test-defs/WakeUpShoot.yaml +++ b/.test-defs/WakeUpShoot.yaml @@ -17,4 +17,4 @@ spec: --project-namespace=$PROJECT_NAMESPACE --kubecfg="$TM_KUBECONFIG_PATH/gardener.config" - image: golang:1.25.3 + image: golang:1.25.4 diff --git a/Dockerfile b/Dockerfile index 4140ee7393c..fd5b8ac0586 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # builder -FROM golang:1.25.3 AS builder +FROM golang:1.25.4 AS builder ARG GOPROXY=https://proxy.golang.org,direct ENV GOPROXY=$GOPROXY WORKDIR /go/src/github.com/gardener/gardener diff --git a/hack/tools/image/variants.yaml b/hack/tools/image/variants.yaml index d00079da42c..e448f687b08 100644 --- a/hack/tools/image/variants.yaml +++ b/hack/tools/image/variants.yaml @@ -4,6 +4,6 @@ # - https://github.com/gardener/ci-infra/blob/master/images/krte variants: "1.24": - image: golang:1.24.9-bookworm + image: golang:1.24.10-bookworm "1.25": - image: golang:1.25.3-bookworm + image: golang:1.25.4-bookworm From 3a01b96f0f4591d7d498b220fad264868f9d2e52 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Thu, 6 Nov 2025 13:39:20 +0200 Subject: [PATCH 050/176] Move defaulting of the Shoot Kubernetes versions from the `ShootValidator` to the `ShootMutator` admission plugin (part 5) (#13252) * Duplicate defaulting of the `.spec.kubernetes.version` field in the `ShootMutator` admission plugin * Duplicate unit tests for defaulting of the `.spec.kubernetes.version` field in the `ShootMutator` admission plugin * Clean up defaulting of the `.spec.kubernetes.version` field from the `ShootValidator` admission plugin * Duplicate defaulting of the `.spec.provider.workers[].kubernetes.version` field in the `ShootMutator` admission plugin * Clean up defaulting of the `.spec.provider.workers[].kubernetes.version` field from the `ShootValidator` admission plugin * Update `ShootMutator` docs --- docs/concepts/apiserver-admission-plugins.md | 3 +- plugin/pkg/shoot/mutator/admission.go | 123 +++++++++++++++++++ plugin/pkg/shoot/mutator/admission_test.go | 119 ++++++++++++++++++ plugin/pkg/shoot/validator/admission.go | 87 +------------ plugin/pkg/shoot/validator/admission_test.go | 97 --------------- 5 files changed, 246 insertions(+), 183 deletions(-) diff --git a/docs/concepts/apiserver-admission-plugins.md b/docs/concepts/apiserver-admission-plugins.md index 3637aac7fbc..24de5b45ef8 100644 --- a/docs/concepts/apiserver-admission-plugins.md +++ b/docs/concepts/apiserver-admission-plugins.md @@ -236,6 +236,7 @@ It mutates the `Shoot` in the following way: - It sets the `gardener.cloud/created-by=` annotation for newly created `Shoot` resources. - It maintains annotations used for Shoot lifecycle operations such as `shoot.gardener.cloud/tasks` and `maintenance.shoot.gardener.cloud/needs-retry-operation`. - It defaults Shoot `.spec.networking.pods` and `.spec.networking.services` fields in case they are not provided and the Seed specifies the `.spec.networks.shootDefaults` field. +- It defaults Shoot Kubernetes version (`.spec.kubernetes.version` and `.spec.provider.workers[].kubernetes.version`). Over time, the `ShootMutator` admission plugin will take over all the mutations that are performed by `ShootValidator`. For more details, see https://github.com/gardener/gardener/issues/2158. @@ -298,7 +299,7 @@ will not be affected by this admission plugin. This admission controller reacts on `CREATE`, `UPDATE` and `DELETE` operations for `Shoot`s. It validates certain configurations in the specification against the referred `CloudProfile` (e.g., machine images, machine types, used Kubernetes version, ...). Generally, it performs validations that cannot be handled by the static API validation due to their dynamic nature (e.g., when something needs to be checked against referred resources). -Additionally, it takes over certain defaulting tasks (e.g., default machine image for worker pools, default Kubernetes version). +Additionally, it takes over certain defaulting tasks (e.g., default machine image for worker pools). ## `ValidatingAdmissionPolicy` diff --git a/plugin/pkg/shoot/mutator/admission.go b/plugin/pkg/shoot/mutator/admission.go index a4a3ef6a056..e1beb04e5e3 100644 --- a/plugin/pkg/shoot/mutator/admission.go +++ b/plugin/pkg/shoot/mutator/admission.go @@ -13,11 +13,15 @@ import ( "net" "reflect" "slices" + "strconv" + "strings" + "github.com/Masterminds/semver/v3" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/admission" "k8s.io/utils/ptr" @@ -25,6 +29,7 @@ import ( "github.com/gardener/gardener/pkg/apis/core/helper" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" admissioninitializer "github.com/gardener/gardener/pkg/apiserver/admission/initializer" gardencoreinformers "github.com/gardener/gardener/pkg/client/core/informers/externalversions" gardencorev1beta1listers "github.com/gardener/gardener/pkg/client/core/listers/core/v1beta1" @@ -129,6 +134,8 @@ func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admissi var ( shoot *core.Shoot oldShoot = &core.Shoot{} + + allErrs field.ErrorList ) shoot, ok := a.GetObject().(*core.Shoot) @@ -172,6 +179,12 @@ func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admissi mutationContext.addMetadataAnnotations(a) mutationContext.defaultShootNetworks(helper.IsWorkerless(shoot)) + allErrs = append(allErrs, mutationContext.defaultKubernetes()...) + allErrs = append(allErrs, mutationContext.defaultKubernetesVersionForWorkers()...) + + if len(allErrs) > 0 { + return admission.NewForbidden(a, allErrs.ToAggregate()) + } return nil } @@ -318,3 +331,113 @@ func generateULAServicesCIDR() string { return fmt.Sprintf("fd%02x:%02x%02x:%02x%02x::/112", globalID[0], globalID[1], globalID[2], globalID[3], globalID[4]) } + +func (c *mutationContext) defaultKubernetes() field.ErrorList { + var path = field.NewPath("spec", "kubernetes") + + defaultVersion, errList := defaultKubernetesVersion(c.cloudProfileSpec.Kubernetes.Versions, c.shoot.Spec.Kubernetes.Version, path.Child("version")) + if len(errList) > 0 { + return errList + } + + if defaultVersion != nil { + c.shoot.Spec.Kubernetes.Version = *defaultVersion + } + + return nil +} + +func defaultKubernetesVersion(constraints []gardencorev1beta1.ExpirableVersion, shootVersion string, fldPath *field.Path) (*string, field.ErrorList) { + var ( + allErrs = field.ErrorList{} + shootVersionMajor *uint64 + shootVersionMinor *uint64 + versionParts = strings.Split(shootVersion, ".") + ) + + if len(versionParts) == 3 { + return nil, allErrs + } + if len(versionParts) == 2 && len(versionParts[1]) > 0 { + v, err := strconv.ParseUint(versionParts[1], 10, 0) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, versionParts[1], "must be a semantic version")) + return nil, allErrs + } + shootVersionMinor = ptr.To(v) + } + if len(versionParts) >= 1 && len(versionParts[0]) > 0 { + v, err := strconv.ParseUint(versionParts[0], 10, 0) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, versionParts[0], "must be a semantic version")) + return nil, allErrs + } + shootVersionMajor = ptr.To(v) + } + + if latestVersion := findLatestSupportedVersion(constraints, shootVersionMajor, shootVersionMinor); latestVersion != nil { + return ptr.To(latestVersion.String()), nil + } + + allErrs = append(allErrs, field.Invalid(fldPath, shootVersion, fmt.Sprintf("couldn't find a suitable version for %s. Suitable versions have a non-expired expiration date and are no 'preview' versions. 'Preview'-classified versions have to be selected explicitly", shootVersion))) + return nil, allErrs +} + +func findLatestSupportedVersion(constraints []gardencorev1beta1.ExpirableVersion, major, minor *uint64) *semver.Version { + var latestVersion *semver.Version + for _, versionConstraint := range constraints { + if v1beta1helper.CurrentLifecycleClassification(versionConstraint) != gardencorev1beta1.ClassificationSupported { + continue + } + + // CloudProfile cannot contain invalid semVer shootVersion + cpVersion := semver.MustParse(versionConstraint.Version) + + // defaulting on patch level: version has to have the same major and minor kubernetes version + if major != nil && cpVersion.Major() != *major { + continue + } + + if minor != nil && cpVersion.Minor() != *minor { + continue + } + + if latestVersion == nil || cpVersion.GreaterThan(latestVersion) { + latestVersion = cpVersion + } + } + + return latestVersion +} + +func (c *mutationContext) defaultKubernetesVersionForWorkers() field.ErrorList { + var path = field.NewPath("spec", "provider") + + for i, worker := range c.shoot.Spec.Provider.Workers { + idxPath := path.Child("workers").Index(i) + + if worker.Kubernetes != nil { + if errList := c.defaultKubernetesVersionForWorker(idxPath, worker); len(errList) > 0 { + return errList + } + } + } + + return nil +} + +func (c *mutationContext) defaultKubernetesVersionForWorker(idxPath *field.Path, worker core.Worker) field.ErrorList { + if worker.Kubernetes.Version == nil { + return nil + } + + defaultVersion, errList := defaultKubernetesVersion(c.cloudProfileSpec.Kubernetes.Versions, *worker.Kubernetes.Version, idxPath.Child("kubernetes", "version")) + if len(errList) > 0 { + return errList + } + + if defaultVersion != nil { + worker.Kubernetes.Version = defaultVersion + } + return nil +} diff --git a/plugin/pkg/shoot/mutator/admission_test.go b/plugin/pkg/shoot/mutator/admission_test.go index 95377e51a8a..c26bfe35444 100644 --- a/plugin/pkg/shoot/mutator/admission_test.go +++ b/plugin/pkg/shoot/mutator/admission_test.go @@ -103,6 +103,9 @@ var _ = Describe("mutator", func() { }, Spec: core.ShootSpec{ CloudProfileName: ptr.To("profile"), + Kubernetes: core.Kubernetes{ + Version: "1.6.4", + }, Provider: core.Provider{ Workers: []core.Worker{ { @@ -533,5 +536,121 @@ var _ = Describe("mutator", func() { Expect(shoot.Spec.Networking.Services).To(Equal(&servicesCIDR)) }) }) + + Context("kubernetes version", func() { + BeforeEach(func() { + cloudProfile.Spec.Kubernetes.Versions = []gardencorev1beta1.ExpirableVersion{ + {Version: "1.28.0", Classification: ptr.To(gardencorev1beta1.ClassificationPreview)}, + {Version: "1.27.3"}, + {Version: "1.27.2"}, + {Version: "1.26.8", Classification: ptr.To(gardencorev1beta1.ClassificationDeprecated), ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * -1000)}}, + {Version: "1.26.7"}, + {Version: "1.26.6"}, + {Version: "1.25.11"}, + {Version: "1.24.12", Classification: ptr.To(gardencorev1beta1.ClassificationDeprecated), ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * -1000)}}, + } + + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) + }) + + It("should throw an error because of an invalid major version", func() { + shoot.Spec.Kubernetes.Version = "foo" + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("must be a semantic version"))) + }) + + It("should throw an error because of an invalid minor version", func() { + shoot.Spec.Kubernetes.Version = "1.bar" + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("must be a semantic version"))) + }) + + It("should default a kubernetes version to latest major.minor.patch version", func() { + shoot.Spec.Kubernetes.Version = "" + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + Expect(shoot.Spec.Kubernetes.Version).To(Equal("1.27.3")) + }) + + It("should default a major kubernetes version to latest minor.patch version", func() { + shoot.Spec.Kubernetes.Version = "1" + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + Expect(shoot.Spec.Kubernetes.Version).To(Equal("1.27.3")) + }) + + It("should default a major.minor kubernetes version to latest patch version", func() { + shoot.Spec.Kubernetes.Version = "1.26" + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + Expect(shoot.Spec.Kubernetes.Version).To(Equal("1.26.7")) + }) + + It("should reject defaulting a major.minor kubernetes version if there is no higher non-preview version available for defaulting", func() { + shoot.Spec.Kubernetes.Version = "1.24" + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("couldn't find a suitable version for 1.24"))) + }) + + It("should be able to explicitly pick preview versions", func() { + shoot.Spec.Kubernetes.Version = "1.28.0" + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + }) + + It("should reject: default only exactly matching minor kubernetes version", func() { + shoot.Spec.Kubernetes.Version = "1.2" + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("couldn't find a suitable version for 1.2"))) + }) + + Context("worker kubernetes version", func() { + It("should not choose the default kubernetes version if version is not specified", func() { + shoot.Spec.Kubernetes.Version = "1.26" + shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{} + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Provider.Workers[0].Kubernetes.Version).To(BeNil()) + }) + + It("should choose the default kubernetes version if only major.minor is given in a worker group", func() { + shoot.Spec.Kubernetes.Version = "1.26" + shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: ptr.To("1.26")} + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(*shoot.Spec.Provider.Workers[0].Kubernetes.Version).To(Equal("1.26.7")) + }) + }) + }) }) }) diff --git a/plugin/pkg/shoot/validator/admission.go b/plugin/pkg/shoot/validator/admission.go index c35b0f309df..584001dc94a 100644 --- a/plugin/pkg/shoot/validator/admission.go +++ b/plugin/pkg/shoot/validator/admission.go @@ -936,18 +936,7 @@ func (c *validationContext) validateKubernetes(a admission.Attributes) field.Err return nil } - defaultVersion, errList := defaultKubernetesVersion(c.cloudProfileSpec.Kubernetes.Versions, c.shoot.Spec.Kubernetes.Version, path.Child("version")) - if len(errList) > 0 { - allErrs = append(allErrs, errList...) - } - - if defaultVersion != nil { - c.shoot.Spec.Kubernetes.Version = *defaultVersion - } else { - // We assume that the 'defaultVersion' is already calculated correctly, so only run validation if the version was not defaulted. - allErrs = append(allErrs, validateKubernetesVersionConstraints(a, c.cloudProfileSpec.Kubernetes.Versions, c.shoot.Spec.Kubernetes.Version, c.oldShoot.Spec.Kubernetes.Version, false, path.Child("version"))...) - } - + allErrs = append(allErrs, validateKubernetesVersionConstraints(a, c.cloudProfileSpec.Kubernetes.Versions, c.shoot.Spec.Kubernetes.Version, c.oldShoot.Spec.Kubernetes.Version, false, path.Child("version"))...) allErrs = append(allErrs, c.validateKubeAPIServerOIDCConfig(a)...) return allErrs @@ -1158,16 +1147,7 @@ func (c *validationContext) validateWorkerKubernetesVersion(idxPath *field.Path, oldWorkerKubernetesVersion = *oldWorker.Kubernetes.Version } - defaultVersion, errList := defaultKubernetesVersion(c.cloudProfileSpec.Kubernetes.Versions, *worker.Kubernetes.Version, idxPath.Child("kubernetes", "version")) - if len(errList) > 0 { - return errList - } - - if defaultVersion == nil { - return validateKubernetesVersionConstraints(a, c.cloudProfileSpec.Kubernetes.Versions, *worker.Kubernetes.Version, oldWorkerKubernetesVersion, isNewWorkerPool, idxPath.Child("kubernetes", "version")) - } - worker.Kubernetes.Version = defaultVersion - return nil + return validateKubernetesVersionConstraints(a, c.cloudProfileSpec.Kubernetes.Versions, *worker.Kubernetes.Version, oldWorkerKubernetesVersion, isNewWorkerPool, idxPath.Child("kubernetes", "version")) } func (c *validationContext) validateAPIVersionForRawExtensions() field.ErrorList { @@ -1332,69 +1312,6 @@ func hasDomainIntersection(domainA, domainB string) bool { return strings.HasSuffix(long, short) } -func defaultKubernetesVersion(constraints []gardencorev1beta1.ExpirableVersion, shootVersion string, fldPath *field.Path) (*string, field.ErrorList) { - var ( - allErrs = field.ErrorList{} - shootVersionMajor *uint64 - shootVersionMinor *uint64 - versionParts = strings.Split(shootVersion, ".") - ) - - if len(versionParts) == 3 { - return nil, allErrs - } - if len(versionParts) == 2 && len(versionParts[1]) > 0 { - v, err := strconv.ParseUint(versionParts[1], 10, 0) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, versionParts[1], "must be a semantic version")) - return nil, allErrs - } - shootVersionMinor = ptr.To(v) - } - if len(versionParts) >= 1 && len(versionParts[0]) > 0 { - v, err := strconv.ParseUint(versionParts[0], 10, 0) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, versionParts[0], "must be a semantic version")) - return nil, allErrs - } - shootVersionMajor = ptr.To(v) - } - - if latestVersion := findLatestSupportedVersion(constraints, shootVersionMajor, shootVersionMinor); latestVersion != nil { - return ptr.To(latestVersion.String()), nil - } - - allErrs = append(allErrs, field.Invalid(fldPath, shootVersion, fmt.Sprintf("couldn't find a suitable version for %s. Suitable versions have a non-expired expiration date and are no 'preview' versions. 'Preview'-classified versions have to be selected explicitly", shootVersion))) - return nil, allErrs -} - -func findLatestSupportedVersion(constraints []gardencorev1beta1.ExpirableVersion, major, minor *uint64) *semver.Version { - var latestVersion *semver.Version - for _, versionConstraint := range constraints { - if v1beta1helper.CurrentLifecycleClassification(versionConstraint) != gardencorev1beta1.ClassificationSupported { - continue - } - - // CloudProfile cannot contain invalid semVer shootVersion - cpVersion := semver.MustParse(versionConstraint.Version) - - // defaulting on patch level: version has to have the same major and minor kubernetes version - if major != nil && cpVersion.Major() != *major { - continue - } - - if minor != nil && cpVersion.Minor() != *minor { - continue - } - - if latestVersion == nil || cpVersion.GreaterThan(latestVersion) { - latestVersion = cpVersion - } - } - - return latestVersion -} - func validateKubernetesVersionConstraints(a admission.Attributes, constraints []gardencorev1beta1.ExpirableVersion, shootVersion, oldShootVersion string, isNewWorkerPool bool, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} diff --git a/plugin/pkg/shoot/validator/admission_test.go b/plugin/pkg/shoot/validator/admission_test.go index 01b6eea6a2a..dba039c9128 100644 --- a/plugin/pkg/shoot/validator/admission_test.go +++ b/plugin/pkg/shoot/validator/admission_test.go @@ -2753,81 +2753,6 @@ var _ = Describe("validator", func() { Expect(err).To(MatchError(ContainSubstring("Unsupported value: \"1.2.3\""))) }) - It("should throw an error because of an invalid major version", func() { - shoot.Spec.Kubernetes.Version = "foo" - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("must be a semantic version"))) - }) - - It("should throw an error because of an invalid minor version", func() { - shoot.Spec.Kubernetes.Version = "1.bar" - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("must be a semantic version"))) - }) - - It("should default a kubernetes version to latest major.minor.patch version", func() { - shoot.Spec.Kubernetes.Version = "" - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - Expect(shoot.Spec.Kubernetes.Version).To(Equal(highestSupportedVersion.Version)) - }) - - It("should default a major kubernetes version to latest minor.patch version", func() { - shoot.Spec.Kubernetes.Version = "1" - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - Expect(shoot.Spec.Kubernetes.Version).To(Equal(highestSupportedVersion.Version)) - }) - - It("should default a major.minor kubernetes version to latest patch version", func() { - shoot.Spec.Kubernetes.Version = "1.26" - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - Expect(shoot.Spec.Kubernetes.Version).To(Equal(highestSupported126Release.Version)) - }) - - It("should reject defaulting a major.minor kubernetes version if there is no higher non-preview version available for defaulting", func() { - shoot.Spec.Kubernetes.Version = "1.24" - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("couldn't find a suitable version for 1.24"))) - }) - - It("should be able to explicitly pick preview versions", func() { - shoot.Spec.Kubernetes.Version = highestPreviewVersion.Version - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - }) - - It("should reject: default only exactly matching minor kubernetes version", func() { - shoot.Spec.Kubernetes.Version = "1.2" - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("couldn't find a suitable version for 1.2"))) - }) - It("should reject to create a cluster with an expired kubernetes version", func() { shoot.Spec.Kubernetes.Version = expiredVersion.Version @@ -2856,28 +2781,6 @@ var _ = Describe("validator", func() { Expect(err).ToNot(HaveOccurred()) }) - It("should not choose the default kubernetes version if version is not specified", func() { - shoot.Spec.Kubernetes.Version = "1.26" - shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{} - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(shoot.Spec.Provider.Workers[0].Kubernetes.Version).To(BeNil()) - }) - - It("should choose the default kubernetes version if only major.minor is given in a worker group", func() { - shoot.Spec.Kubernetes.Version = "1.26" - shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: ptr.To("1.26")} - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(*shoot.Spec.Provider.Workers[0].Kubernetes.Version).To(Equal(highestSupported126Release.Version)) - }) - It("should work to create a cluster with a worker group kubernetes version set smaller than control plane version", func() { shoot.Spec.Kubernetes.Version = highestSupportedVersion.Version shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: ptr.To("1.26.6")} From b1fd9ceef7f75663282447f55073a2d7c909b7d6 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 6 Nov 2025 12:39:27 +0100 Subject: [PATCH 051/176] Update module github.com/containerd/containerd to v1.7.29 (#13372) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d3bb48f7a5e..77e2501e53a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ahmetb/gen-crd-api-reference-docs v0.3.0 github.com/andybalholm/brotli v1.2.0 github.com/bramvdbogaerde/go-scp v1.5.0 - github.com/containerd/containerd v1.7.28 + github.com/containerd/containerd v1.7.29 github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 github.com/docker/cli v28.5.2+incompatible diff --git a/go.sum b/go.sum index 44eb0dfcf13..10ee032a51a 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,8 @@ github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/containerd v1.7.28 h1:Nsgm1AtcmEh4AHAJ4gGlNSaKgXiNccU270Dnf81FQ3c= -github.com/containerd/containerd v1.7.28/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs= +github.com/containerd/containerd v1.7.29 h1:90fWABQsaN9mJhGkoVnuzEY+o1XDPbg9BTC9QTAHnuE= +github.com/containerd/containerd v1.7.29/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs= github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= github.com/containerd/continuity v0.4.4 h1:/fNVfTJ7wIl/YPMHjf+5H32uFhl63JucB34PlCpMKII= From d5071c800378616eb6bb2c7662b4b28f4cfe7406 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 6 Nov 2025 14:21:20 +0100 Subject: [PATCH 052/176] Update quay.io/kiwigrid/k8s-sidecar Docker tag to v2.1.1 (#13374) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 4d2cb5a9054..c30df330f11 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -352,7 +352,7 @@ images: - name: plutono-data-refresher sourceRepository: github.com/kiwigrid/k8s-sidecar repository: quay.io/kiwigrid/k8s-sidecar - tag: "2.0.3" + tag: "2.1.1" labels: - name: 'gardener.cloud/cve-categorisation' value: From 8401f805438478f2a062ed7cad2f6017faf9bc6d Mon Sep 17 00:00:00 2001 From: Axel Siebenborn Date: Thu, 6 Nov 2025 17:07:20 +0100 Subject: [PATCH 053/176] Make migration code more robust. (#13381) --- .../operation/botanist/dualstackmigration.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/gardenlet/operation/botanist/dualstackmigration.go b/pkg/gardenlet/operation/botanist/dualstackmigration.go index ee3cae5d4c4..636c53b9058 100644 --- a/pkg/gardenlet/operation/botanist/dualstackmigration.go +++ b/pkg/gardenlet/operation/botanist/dualstackmigration.go @@ -9,6 +9,7 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -198,10 +199,18 @@ func (b *Botanist) UpdateDualStackMigrationConditionIfNeeded(ctx context.Context } if constraint := v1beta1helper.GetCondition(shoot.Status.Constraints, gardencorev1beta1.ShootDualStackNodesMigrationReady); constraint == nil { - network, _ := b.Shoot.Components.Extensions.Network.Get(ctx) - if network == nil { - return nil // Network not yet created, nothing to do + network, err := b.Shoot.Components.Extensions.Network.Get(ctx) + if err != nil { + if apierrors.IsNotFound(err) { + return nil // Network not yet created, nothing to do + } + return fmt.Errorf("failed to get network resource: %w", err) } + + if network == nil || network.Status.IPFamilies == nil { + return nil + } + networkReadyForDualStackMigration := len(network.Status.IPFamilies) == len(b.Shoot.GetInfo().Spec.Networking.IPFamilies) updateFunction := b.DetermineUpdateFunction(networkReadyForDualStackMigration, nodeList) if err := b.Shoot.UpdateInfoStatus(ctx, b.GardenClient, true, false, updateFunction); err != nil { From a956428aebceca3cdf6bb46b07cb7ec3a091a16e Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 7 Nov 2025 10:07:21 +0100 Subject: [PATCH 054/176] Update quay.io/kiwigrid/k8s-sidecar Docker tag to v2.1.2 (#13384) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index c30df330f11..acfef962dc2 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -352,7 +352,7 @@ images: - name: plutono-data-refresher sourceRepository: github.com/kiwigrid/k8s-sidecar repository: quay.io/kiwigrid/k8s-sidecar - tag: "2.1.1" + tag: "2.1.2" labels: - name: 'gardener.cloud/cve-categorisation' value: From 2b4c4f5df0d1b7e00891964ccc53cec39e05ecd1 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Fri, 7 Nov 2025 14:51:23 +0100 Subject: [PATCH 055/176] Support setting `loadBalancerClass` in `Seed` and `ExposureClass` (#13305) * Add docs for `Seed` and `ExposureClass` `loadBalancerClass` * Add `Seed.spec.settings.loadBalancerServices.class` field * Add `GardenletConfiguration.exposureClassHandlers[].loadBalancerService.class` field * Require non-empty class if field is specified * Handle new setting in istio component * Align `loadBalancerClass` validation with Kubernetes --- docs/api-reference/core.md | 13 + docs/operations/seed_settings.md | 7 + docs/usage/networking/exposureclasses.md | 6 + example/50-seed.yaml | 1 + pkg/apis/core/types_seed.go | 3 + pkg/apis/core/v1beta1/generated.pb.go | 1862 +++++++++-------- pkg/apis/core/v1beta1/generated.proto | 5 + pkg/apis/core/v1beta1/types_seed.go | 4 + .../core/v1beta1/zz_generated.conversion.go | 2 + .../core/v1beta1/zz_generated.deepcopy.go | 5 + pkg/apis/core/validation/seed.go | 4 + pkg/apis/core/validation/seed_test.go | 50 + pkg/apis/core/zz_generated.deepcopy.go | 5 + pkg/apiserver/openapi/openapi_generated.go | 7 + .../istio-ingress/templates/service.yaml | 3 + .../charts/istio/istio-ingress/values.yaml | 1 + .../networking/istio/ingress_gateway.go | 2 + pkg/component/networking/istio/istio_test.go | 35 + .../test_charts/ingress_service_class.yaml | 25 + pkg/component/shared/istio.go | 4 + pkg/component/shared/istio_test.go | 17 + pkg/gardenlet/apis/config/v1alpha1/types.go | 5 + .../config/v1alpha1/validation/validation.go | 40 +- .../v1alpha1/validation/validation_test.go | 44 + .../config/v1alpha1/zz_generated.deepcopy.go | 5 + .../controller/seed/seed/components.go | 4 + pkg/gardenlet/operation/seed/seed.go | 10 + pkg/gardenlet/operation/seed/seed_test.go | 42 + .../controller/garden/garden/components.go | 1 + .../validation/kubernetes/core/validation.go | 10 + .../kubernetes/core/validation_test.go | 18 + 31 files changed, 1318 insertions(+), 922 deletions(-) create mode 100644 pkg/component/networking/istio/test_charts/ingress_service_class.yaml diff --git a/docs/api-reference/core.md b/docs/api-reference/core.md index 9e07bec8a6d..b670753d337 100644 --- a/docs/api-reference/core.md +++ b/docs/api-reference/core.md @@ -11454,6 +11454,19 @@ SeedSettingLoadBalancerServicesZonalIngress Defaults to true.

+ + +class
+ +string + + + +(Optional) +

Class configures the Service.spec.loadBalancerClass field for the load balancer services on the seed. +Note that changing the loadBalancerClass of existing LoadBalancer services is denied by Kubernetes.

+ +

SeedSettingLoadBalancerServicesZonalIngress diff --git a/docs/operations/seed_settings.md b/docs/operations/seed_settings.md index 62e016d1912..234442ccc36 100644 --- a/docs/operations/seed_settings.md +++ b/docs/operations/seed_settings.md @@ -72,6 +72,13 @@ In most cases, the cloud-controller-manager (responsible for managing these load By setting the `.spec.settings.loadBalancerServices.annotations` field the Gardener administrator can specify a list of annotations, which will be injected into the `Service`s of type `LoadBalancer`. +### Load Balancer Class + +By default, Gardener creates `Services` without the `spec.loadBalancerClass` field set, meaning that the default load balancer implementation of the underlying cloud infrastructure is used (implemented by the `Service` controller of cloud-controller-manager). +If a non-default load balancer implementation should be used for load balancer services in the seed cluster, the `spec.settings.loadBalancerServices.loadBalancerClass` field can be configured accordingly to set the `spec.loadBalancerClass` on the created `Service` objects. +Note that changing the `loadBalancerClass` of existing load balancer services is denied by Kubernetes, i.e., this setting can only be applied automatically to newly created load balancer services. +If an existing load balancer service should use a different load balancer class, the migration needs to be performed manually by the operator. + ### External Traffic Policy Setting the [external traffic policy](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip) to `Local` can be beneficial as it diff --git a/docs/usage/networking/exposureclasses.md b/docs/usage/networking/exposureclasses.md index 03ede51d83c..c00e49c49f9 100644 --- a/docs/usage/networking/exposureclasses.md +++ b/docs/usage/networking/exposureclasses.md @@ -128,6 +128,7 @@ exposureClassHandlers: loadbalancer/network: internet - name: internal-config loadBalancerService: + class: internal-loadbalancer annotations: loadbalancer/network: internal sni: @@ -146,6 +147,11 @@ The load balancer service needs to be configured in a way that it is reachable f Therefore, the configuration of load balancer service need to be specified, which can be done via the `.loadBalancerService` section. The common way to influence load balancer service behaviour is via annotations where the respective cloud-controller-manager will react on and configure the infrastructure load balancer accordingly. +To select a non-default load balancer implementation, the `class` field can be set to configure the `spec.loadBalancerClass` on the created `Service` objects. +If the `class` field is unset, `spec.loadBalancerClass` is not configured and the default load balancer implementation of the underlying cloud infrastructure is used (implemented by the `Service` controller of cloud-controller-manager). +Note that changing the `loadBalancerClass` of existing load balancer services is denied by Kubernetes, i.e., this setting can only be applied automatically to newly created load balancer services. +If an existing load balancer service should use a different load balancer class, the migration needs to be performed manually by the operator. + The control planes on a `Seed` will be exposed via a central load balancer and with Envoy via TLS SNI passthrough proxy. In this case, the gardenlet will install a dedicated ingress gateway (Envoy + load balancer + respective configuration) for each handler on the `Seed`. The configuration of the ingress gateways can be controlled via the `.sni` section in the same way like for the default ingress gateways. diff --git a/example/50-seed.yaml b/example/50-seed.yaml index 3e10a09c632..fdc699bc284 100644 --- a/example/50-seed.yaml +++ b/example/50-seed.yaml @@ -93,6 +93,7 @@ spec: # loadBalancerServices: # annotations: # foo: bar + # class: non-default-load-balancer-class # externalTrafficPolicy: Local # proxyProtocol: # allowed: true diff --git a/pkg/apis/core/types_seed.go b/pkg/apis/core/types_seed.go index e647cbe2296..7b634a3b458 100644 --- a/pkg/apis/core/types_seed.go +++ b/pkg/apis/core/types_seed.go @@ -267,6 +267,9 @@ type SeedSettingLoadBalancerServices struct { // ZonalIngress controls whether ingress gateways are deployed per availability zone. // Defaults to true. ZonalIngress *SeedSettingLoadBalancerServicesZonalIngress + // Class configures the Service.spec.loadBalancerClass field for the load balancer services on the seed. + // Note that changing the loadBalancerClass of existing LoadBalancer services is denied by Kubernetes. + Class *string } // SeedSettingLoadBalancerServicesZones controls settings, which are specific to the single-zone load balancers in a diff --git a/pkg/apis/core/v1beta1/generated.pb.go b/pkg/apis/core/v1beta1/generated.pb.go index 849860f37b9..3da202d663a 100644 --- a/pkg/apis/core/v1beta1/generated.pb.go +++ b/pkg/apis/core/v1beta1/generated.pb.go @@ -5969,926 +5969,927 @@ func init() { } var fileDescriptor_ca37af0df9a5bbd2 = []byte{ - // 14698 bytes of a gzipped FileDescriptorProto + // 14705 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x6b, 0x70, 0x24, 0xd9, 0x55, 0x20, 0xec, 0x2c, 0xbd, 0x8f, 0x1e, 0x2d, 0xdd, 0x56, 0x77, 0xab, 0xd5, 0x8f, 0x6a, 0xe7, 0x8c, 0x4d, 0x0f, 0x63, 0xab, 0x99, 0xf1, 0xd8, 0xe3, 0x19, 0x33, 0x0f, 0xa9, 0x4a, 0xea, 0x2e, 0xb7, 0xa4, 0xae, 0xb9, 0x25, 0xcd, 0x8c, 0x67, 0x60, 0x4c, 0x2a, 0xeb, 0xaa, 0x94, 0xd3, 0x59, 0x99, 0x35, 0x99, 0x59, 0x6a, 0xa9, 0xc7, 0xc6, 0x0f, 0x30, 0x9f, 0x6d, 0x0c, 0x1f, 0xf0, 0x01, - 0xfe, 0x6c, 0xb3, 0xc1, 0x60, 0x02, 0xb3, 0xbb, 0xb0, 0x66, 0x03, 0x82, 0x8d, 0x00, 0x02, 0x76, - 0x21, 0x82, 0xc5, 0x10, 0x40, 0x10, 0x18, 0x76, 0xcd, 0x02, 0x02, 0x6b, 0xbd, 0xb0, 0x11, 0xfb, - 0x8c, 0x25, 0x62, 0x89, 0xe8, 0x25, 0xd8, 0x8d, 0xfb, 0xc8, 0xcc, 0x9b, 0xaf, 0x52, 0x29, 0x4b, - 0x92, 0x67, 0x16, 0xff, 0x92, 0xea, 0x9e, 0x7b, 0xcf, 0xb9, 0xaf, 0x3c, 0xf7, 0x9c, 0x73, 0xcf, - 0x3d, 0x07, 0x16, 0x1a, 0x86, 0xb7, 0xd5, 0xde, 0x98, 0xd3, 0xed, 0xe6, 0xb5, 0x86, 0xe6, 0xd4, - 0x89, 0x45, 0x9c, 0xf0, 0x9f, 0xd6, 0xed, 0xc6, 0x35, 0xad, 0x65, 0xb8, 0xd7, 0x74, 0xdb, 0x21, - 0xd7, 0xb6, 0x1f, 0xda, 0x20, 0x9e, 0xf6, 0xd0, 0xb5, 0x06, 0x85, 0x69, 0x1e, 0xa9, 0xcf, 0xb5, - 0x1c, 0xdb, 0xb3, 0xd1, 0xc3, 0x21, 0x8e, 0x39, 0xbf, 0x69, 0xf8, 0x4f, 0xeb, 0x76, 0x63, 0x8e, - 0xe2, 0x98, 0xa3, 0x38, 0xe6, 0x04, 0x8e, 0xd9, 0xb7, 0xcb, 0x74, 0xed, 0x86, 0x7d, 0x8d, 0xa1, - 0xda, 0x68, 0x6f, 0xb2, 0x5f, 0xec, 0x07, 0xfb, 0x8f, 0x93, 0x98, 0x7d, 0xe0, 0xf6, 0xbb, 0xdd, - 0x39, 0xc3, 0xa6, 0x9d, 0xb9, 0xa6, 0xb5, 0x3d, 0xdb, 0xd5, 0x35, 0xd3, 0xb0, 0x1a, 0xd7, 0xb6, - 0x13, 0xbd, 0x99, 0x55, 0xa5, 0xaa, 0xa2, 0xdb, 0x1d, 0xeb, 0x38, 0x1b, 0x9a, 0x9e, 0x56, 0xe7, - 0x46, 0x58, 0x87, 0xec, 0x78, 0xc4, 0x72, 0x0d, 0xdb, 0x72, 0xdf, 0x4e, 0x47, 0x42, 0x9c, 0x6d, - 0x79, 0x6e, 0x22, 0x15, 0xd2, 0x30, 0x3d, 0x12, 0x62, 0x6a, 0x6a, 0xfa, 0x96, 0x61, 0x11, 0x67, - 0xd7, 0x6f, 0x7e, 0xcd, 0x21, 0xae, 0xdd, 0x76, 0x74, 0x72, 0xa8, 0x56, 0xee, 0xb5, 0x26, 0xf1, - 0xb4, 0x34, 0x5a, 0xd7, 0xb2, 0x5a, 0x39, 0x6d, 0xcb, 0x33, 0x9a, 0x49, 0x32, 0xef, 0x3a, 0xa8, - 0x81, 0xab, 0x6f, 0x91, 0xa6, 0x96, 0x68, 0xf7, 0x8e, 0xac, 0x76, 0x6d, 0xcf, 0x30, 0xaf, 0x19, - 0x96, 0xe7, 0x7a, 0x4e, 0xbc, 0x91, 0xfa, 0x49, 0x05, 0x26, 0xe7, 0xab, 0x95, 0x1a, 0x9b, 0xc1, - 0x65, 0xbb, 0xd1, 0x30, 0xac, 0x06, 0x7a, 0x10, 0x46, 0xb6, 0x89, 0xb3, 0x61, 0xbb, 0x86, 0xb7, - 0x3b, 0xa3, 0x5c, 0x51, 0xae, 0x0e, 0x2c, 0x8c, 0xef, 0xef, 0x15, 0x47, 0x9e, 0xf5, 0x0b, 0x71, - 0x08, 0x47, 0x15, 0x38, 0xbd, 0xe5, 0x79, 0xad, 0x79, 0x5d, 0x27, 0xae, 0x1b, 0xd4, 0x98, 0x29, - 0xb0, 0x66, 0xe7, 0xf6, 0xf7, 0x8a, 0xa7, 0x6f, 0xac, 0xad, 0x55, 0x63, 0x60, 0x9c, 0xd6, 0x46, - 0xfd, 0x79, 0x05, 0xa6, 0x82, 0xce, 0x60, 0xf2, 0x4a, 0x9b, 0xb8, 0x9e, 0x8b, 0x30, 0x9c, 0x6d, - 0x6a, 0x3b, 0xab, 0xb6, 0xb5, 0xd2, 0xf6, 0x34, 0xcf, 0xb0, 0x1a, 0x15, 0x6b, 0xd3, 0x34, 0x1a, - 0x5b, 0x9e, 0xe8, 0xda, 0xec, 0xfe, 0x5e, 0xf1, 0xec, 0x4a, 0x6a, 0x0d, 0x9c, 0xd1, 0x92, 0x76, - 0xba, 0xa9, 0xed, 0x24, 0x10, 0x4a, 0x9d, 0x5e, 0x49, 0x82, 0x71, 0x5a, 0x1b, 0xf5, 0x9d, 0x30, - 0xc5, 0xc7, 0x81, 0x89, 0xeb, 0x39, 0x86, 0xee, 0x19, 0xb6, 0x85, 0xae, 0x40, 0xbf, 0xa5, 0x35, - 0x09, 0xeb, 0xe1, 0xc8, 0xc2, 0xd8, 0x97, 0xf6, 0x8a, 0x6f, 0xda, 0xdf, 0x2b, 0xf6, 0xaf, 0x6a, - 0x4d, 0x82, 0x19, 0x44, 0xfd, 0x9f, 0x05, 0xb8, 0x98, 0x68, 0xf7, 0x9c, 0xe1, 0x6d, 0xdd, 0x6a, - 0xd1, 0xff, 0x5c, 0xf4, 0xfd, 0x0a, 0x4c, 0x69, 0xf1, 0x0a, 0x0c, 0xe1, 0xe8, 0xc3, 0x8b, 0x73, - 0x87, 0xff, 0xc0, 0xe7, 0x12, 0xd4, 0x16, 0xce, 0x8b, 0x7e, 0x25, 0x07, 0x80, 0x93, 0xa4, 0xd1, - 0xc7, 0x15, 0x18, 0xb2, 0x79, 0xe7, 0x66, 0x0a, 0x57, 0xfa, 0xae, 0x8e, 0x3e, 0xfc, 0xed, 0x47, - 0xd2, 0x0d, 0x69, 0xd0, 0x73, 0xe2, 0xef, 0xa2, 0xe5, 0x39, 0xbb, 0x0b, 0xa7, 0x44, 0xf7, 0x86, - 0x44, 0x29, 0xf6, 0xc9, 0xcf, 0x3e, 0x0e, 0x63, 0x72, 0x4d, 0x34, 0x09, 0x7d, 0xb7, 0x09, 0xdf, - 0xaa, 0x23, 0x98, 0xfe, 0x8b, 0xa6, 0x61, 0x60, 0x5b, 0x33, 0xdb, 0x84, 0x2d, 0xe9, 0x08, 0xe6, - 0x3f, 0x1e, 0x2f, 0xbc, 0x5b, 0x51, 0x1f, 0x86, 0x81, 0xf9, 0x7a, 0xdd, 0xb6, 0xd0, 0x03, 0x30, - 0x44, 0x2c, 0x6d, 0xc3, 0x24, 0x75, 0xd6, 0x70, 0x38, 0xa4, 0xb7, 0xc8, 0x8b, 0xb1, 0x0f, 0x57, - 0x7f, 0xa4, 0x00, 0x83, 0xac, 0x91, 0x8b, 0x7e, 0x48, 0x81, 0xd3, 0xb7, 0xdb, 0x1b, 0xc4, 0xb1, - 0x88, 0x47, 0xdc, 0xb2, 0xe6, 0x6e, 0x6d, 0xd8, 0x9a, 0x53, 0x17, 0x0b, 0x73, 0x3d, 0xcf, 0x8c, - 0xdc, 0x4c, 0xa2, 0xe3, 0x7b, 0x30, 0x05, 0x80, 0xd3, 0x88, 0xa3, 0x6d, 0x18, 0xb3, 0x1a, 0x86, - 0xb5, 0x53, 0xb1, 0x1a, 0x0e, 0x71, 0x5d, 0x36, 0xe8, 0xd1, 0x87, 0x9f, 0xce, 0xd3, 0x99, 0x55, - 0x09, 0xcf, 0xc2, 0xe4, 0xfe, 0x5e, 0x71, 0x4c, 0x2e, 0xc1, 0x11, 0x3a, 0xea, 0xdf, 0x2b, 0x70, - 0x6a, 0xbe, 0xde, 0x34, 0x5c, 0xca, 0x69, 0xab, 0x66, 0xbb, 0x61, 0x74, 0xb1, 0xf5, 0xd1, 0x33, - 0x30, 0xa8, 0xdb, 0xd6, 0xa6, 0xd1, 0x10, 0xfd, 0x7c, 0xfb, 0x1c, 0xe7, 0x5c, 0x73, 0x32, 0xe7, - 0x62, 0xdd, 0x13, 0x1c, 0x6f, 0x0e, 0x6b, 0x77, 0x16, 0x7d, 0x86, 0xbe, 0x00, 0xfb, 0x7b, 0xc5, - 0xc1, 0x12, 0x43, 0x80, 0x05, 0x22, 0x74, 0x15, 0x86, 0xeb, 0x86, 0xcb, 0x17, 0xb3, 0x8f, 0x2d, - 0xe6, 0xd8, 0xfe, 0x5e, 0x71, 0xb8, 0x2c, 0xca, 0x70, 0x00, 0x45, 0xcb, 0x30, 0x4d, 0x67, 0x90, - 0xb7, 0xab, 0x11, 0xdd, 0x21, 0x1e, 0xed, 0xda, 0x4c, 0x3f, 0xeb, 0xee, 0xcc, 0xfe, 0x5e, 0x71, - 0xfa, 0x66, 0x0a, 0x1c, 0xa7, 0xb6, 0x52, 0x97, 0x60, 0x78, 0xde, 0x24, 0x0e, 0x65, 0x08, 0xe8, - 0x71, 0x98, 0x20, 0x4d, 0xcd, 0x30, 0x31, 0xd1, 0x89, 0xb1, 0x4d, 0x1c, 0x77, 0x46, 0xb9, 0xd2, - 0x77, 0x75, 0x64, 0x01, 0xed, 0xef, 0x15, 0x27, 0x16, 0x23, 0x10, 0x1c, 0xab, 0xa9, 0x7e, 0x44, - 0x81, 0xd1, 0xf9, 0x76, 0xdd, 0xf0, 0xf8, 0xb8, 0x90, 0x03, 0xa3, 0x1a, 0xfd, 0x59, 0xb5, 0x4d, - 0x43, 0xdf, 0x15, 0x9b, 0xeb, 0xa9, 0x5c, 0x9f, 0x5b, 0x88, 0x66, 0xe1, 0xd4, 0xfe, 0x5e, 0x71, - 0x54, 0x2a, 0xc0, 0x32, 0x11, 0x75, 0x0b, 0x64, 0x18, 0x7a, 0x1f, 0x8c, 0xf1, 0xe1, 0xae, 0x68, - 0x2d, 0x4c, 0x36, 0x45, 0x1f, 0xee, 0x93, 0xd6, 0xca, 0x27, 0x34, 0x77, 0x6b, 0xe3, 0x65, 0xa2, - 0x7b, 0x98, 0x6c, 0x12, 0x87, 0x58, 0x3a, 0xe1, 0xdb, 0xa6, 0x24, 0x35, 0xc6, 0x11, 0x54, 0xea, - 0xff, 0xa7, 0xc0, 0xa5, 0xf9, 0xb6, 0xb7, 0x65, 0x3b, 0xc6, 0x5d, 0xe2, 0x84, 0xd3, 0x1d, 0x60, - 0x40, 0x4f, 0xc2, 0x84, 0x16, 0x54, 0x58, 0x0d, 0xb7, 0xd3, 0x59, 0xb1, 0x9d, 0x26, 0xe6, 0x23, - 0x50, 0x1c, 0xab, 0x8d, 0x1e, 0x06, 0x70, 0xc3, 0xb5, 0x65, 0x3c, 0x60, 0x01, 0x89, 0xb6, 0x20, - 0xad, 0xaa, 0x54, 0x4b, 0xfd, 0x0b, 0x7a, 0x14, 0x6e, 0x6b, 0x86, 0xa9, 0x6d, 0x18, 0xa6, 0xe1, - 0xed, 0xbe, 0x60, 0x5b, 0xa4, 0x8b, 0xdd, 0xbc, 0x0e, 0xe7, 0xda, 0x96, 0xc6, 0xdb, 0x99, 0x64, - 0x85, 0xef, 0xdf, 0xb5, 0xdd, 0x16, 0xe1, 0x5c, 0x72, 0x64, 0xe1, 0xc2, 0xfe, 0x5e, 0xf1, 0xdc, - 0x7a, 0x7a, 0x15, 0x9c, 0xd5, 0x96, 0x9e, 0x7a, 0x12, 0xe8, 0x59, 0xdb, 0x6c, 0x37, 0x05, 0xd6, - 0x3e, 0x86, 0x95, 0x9d, 0x7a, 0xeb, 0xa9, 0x35, 0x70, 0x46, 0x4b, 0xf5, 0x0b, 0x05, 0x18, 0x5c, - 0xd0, 0xf4, 0xdb, 0xed, 0x16, 0x7a, 0x1b, 0x0c, 0xb7, 0x1c, 0x7b, 0xdb, 0xa8, 0x13, 0x47, 0x8c, - 0x6d, 0x52, 0x8c, 0x6d, 0xb8, 0x2a, 0xca, 0x71, 0x50, 0x03, 0x19, 0x30, 0xe1, 0xff, 0x5f, 0xea, - 0xe1, 0xcb, 0x65, 0x5f, 0x42, 0x35, 0x82, 0x08, 0xc7, 0x10, 0x23, 0x15, 0x06, 0x1d, 0xd2, 0xa0, - 0x47, 0x5d, 0x1f, 0xeb, 0x16, 0xfb, 0xda, 0x31, 0x2b, 0xc1, 0x02, 0x82, 0xde, 0x0f, 0x13, 0xba, - 0x43, 0xea, 0xc4, 0xf2, 0x0c, 0xcd, 0x74, 0xe9, 0xe6, 0x1c, 0xe8, 0x7e, 0x73, 0xb2, 0x4e, 0x94, - 0x22, 0xcd, 0x71, 0x0c, 0x9d, 0xfa, 0xa5, 0x02, 0x8c, 0xf1, 0x89, 0x5a, 0x68, 0xeb, 0xb7, 0x89, - 0x87, 0xbe, 0x03, 0x86, 0xa9, 0x7c, 0x57, 0xd7, 0x3c, 0x4d, 0x7c, 0x08, 0xdf, 0x92, 0x39, 0x74, - 0xf6, 0x0d, 0xd2, 0xda, 0x21, 0xf5, 0x15, 0xe2, 0x69, 0xe1, 0xfe, 0x0b, 0xcb, 0x70, 0x80, 0x15, - 0x6d, 0x42, 0xbf, 0xdb, 0x22, 0xba, 0x98, 0xd8, 0x72, 0x9e, 0x4f, 0x5d, 0xee, 0x71, 0xad, 0x45, - 0xf4, 0x70, 0xbb, 0xd2, 0x5f, 0x98, 0xe1, 0x47, 0x16, 0x0c, 0xba, 0x9e, 0xe6, 0xb5, 0x5d, 0x36, - 0xbf, 0xa3, 0x0f, 0x2f, 0xf5, 0x4c, 0x89, 0x61, 0x5b, 0x98, 0x10, 0xb4, 0x06, 0xf9, 0x6f, 0x2c, - 0xa8, 0xa8, 0xff, 0x56, 0x81, 0x49, 0xb9, 0xfa, 0xb2, 0xe1, 0x7a, 0xe8, 0xdb, 0x12, 0xd3, 0x39, - 0xd7, 0xdd, 0x74, 0xd2, 0xd6, 0x6c, 0x32, 0x83, 0xdd, 0xea, 0x97, 0x48, 0x53, 0x49, 0x60, 0xc0, - 0xf0, 0x48, 0xd3, 0x97, 0x52, 0x9e, 0xee, 0x75, 0x84, 0x0b, 0xe3, 0x82, 0xd8, 0x40, 0x85, 0xa2, - 0xc5, 0x1c, 0xbb, 0xfa, 0x1d, 0x30, 0x2d, 0xd7, 0xf2, 0xf7, 0x35, 0x65, 0x19, 0xde, 0x6e, 0x2b, - 0xc1, 0x32, 0xe8, 0x27, 0x88, 0x19, 0x04, 0xbd, 0x35, 0xd8, 0xe3, 0x9c, 0x33, 0x05, 0x73, 0x17, - 0xdd, 0xe7, 0xea, 0x87, 0xfb, 0xa2, 0x73, 0x47, 0x97, 0x11, 0x6d, 0xc7, 0xbe, 0xdc, 0xd1, 0x87, - 0x6f, 0xf4, 0x3a, 0x40, 0xbf, 0xeb, 0xaf, 0x17, 0x1e, 0x70, 0x15, 0x86, 0x5d, 0x42, 0xea, 0xd2, - 0xb9, 0xcc, 0x4e, 0xf3, 0x9a, 0x28, 0xc3, 0x01, 0xf4, 0xf8, 0x39, 0xc1, 0x6b, 0xfd, 0x80, 0x92, - 0xbb, 0x5d, 0x9e, 0x0c, 0x5e, 0x22, 0x96, 0xa2, 0x97, 0xc9, 0x10, 0x1f, 0x4e, 0x0c, 0x31, 0xba, - 0x0b, 0xe3, 0xa6, 0xe6, 0x7a, 0xb7, 0x5a, 0x54, 0x6f, 0xf3, 0xf7, 0xcc, 0xe8, 0xc3, 0xf3, 0x79, + 0xfe, 0x6c, 0xf3, 0x05, 0x83, 0x09, 0xcc, 0xb7, 0xcb, 0xae, 0xd9, 0x80, 0x60, 0x23, 0x80, 0x80, + 0x5d, 0x88, 0x60, 0x31, 0x04, 0x26, 0x08, 0x0c, 0xbb, 0x66, 0x01, 0x81, 0xb5, 0x5e, 0xd8, 0x88, + 0x7d, 0xc6, 0x12, 0xb1, 0x44, 0xf4, 0x12, 0xec, 0xc6, 0x7d, 0x64, 0xe6, 0xcd, 0x57, 0xa9, 0x94, + 0x25, 0xc9, 0x33, 0x8b, 0x7f, 0x49, 0x75, 0xcf, 0xbd, 0xe7, 0xdc, 0x57, 0x9e, 0x7b, 0xee, 0xb9, + 0xe7, 0x01, 0x0b, 0x0d, 0xc3, 0xdb, 0x6a, 0x6f, 0xcc, 0xe9, 0x76, 0xf3, 0x5a, 0x43, 0x73, 0xea, + 0xc4, 0x22, 0x4e, 0xf8, 0x4f, 0xeb, 0x76, 0xe3, 0x9a, 0xd6, 0x32, 0xdc, 0x6b, 0xba, 0xed, 0x90, + 0x6b, 0xdb, 0x0f, 0x6d, 0x10, 0x4f, 0x7b, 0xe8, 0x5a, 0x83, 0xc2, 0x34, 0x8f, 0xd4, 0xe7, 0x5a, + 0x8e, 0xed, 0xd9, 0xe8, 0xe1, 0x10, 0xc7, 0x9c, 0xdf, 0x34, 0xfc, 0xa7, 0x75, 0xbb, 0x31, 0x47, + 0x71, 0xcc, 0x51, 0x1c, 0x73, 0x02, 0xc7, 0xec, 0xdb, 0x65, 0xba, 0x76, 0xc3, 0xbe, 0xc6, 0x50, + 0x6d, 0xb4, 0x37, 0xd9, 0x2f, 0xf6, 0x83, 0xfd, 0xc7, 0x49, 0xcc, 0x3e, 0x70, 0xfb, 0xdd, 0xee, + 0x9c, 0x61, 0xd3, 0xce, 0x5c, 0xd3, 0xda, 0x9e, 0xed, 0xea, 0x9a, 0x69, 0x58, 0x8d, 0x6b, 0xdb, + 0x89, 0xde, 0xcc, 0xaa, 0x52, 0x55, 0xd1, 0xed, 0x8e, 0x75, 0x9c, 0x0d, 0x4d, 0x4f, 0xab, 0x73, + 0x23, 0xac, 0x43, 0x76, 0x3c, 0x62, 0xb9, 0x86, 0x6d, 0xb9, 0x6f, 0xa7, 0x23, 0x21, 0xce, 0xb6, + 0x3c, 0x37, 0x91, 0x0a, 0x69, 0x98, 0x1e, 0x09, 0x31, 0x35, 0x35, 0x7d, 0xcb, 0xb0, 0x88, 0xb3, + 0xeb, 0x37, 0xbf, 0xe6, 0x10, 0xd7, 0x6e, 0x3b, 0x3a, 0x39, 0x54, 0x2b, 0xf7, 0x5a, 0x93, 0x78, + 0x5a, 0x1a, 0xad, 0x6b, 0x59, 0xad, 0x9c, 0xb6, 0xe5, 0x19, 0xcd, 0x24, 0x99, 0x77, 0x1d, 0xd4, + 0xc0, 0xd5, 0xb7, 0x48, 0x53, 0x4b, 0xb4, 0x7b, 0x47, 0x56, 0xbb, 0xb6, 0x67, 0x98, 0xd7, 0x0c, + 0xcb, 0x73, 0x3d, 0x27, 0xde, 0x48, 0xfd, 0xa4, 0x02, 0x93, 0xf3, 0xd5, 0x4a, 0x8d, 0xcd, 0xe0, + 0xb2, 0xdd, 0x68, 0x18, 0x56, 0x03, 0x3d, 0x08, 0x23, 0xdb, 0xc4, 0xd9, 0xb0, 0x5d, 0xc3, 0xdb, + 0x9d, 0x51, 0xae, 0x28, 0x57, 0x07, 0x16, 0xc6, 0xf7, 0xf7, 0x8a, 0x23, 0xcf, 0xfa, 0x85, 0x38, + 0x84, 0xa3, 0x0a, 0x9c, 0xde, 0xf2, 0xbc, 0xd6, 0xbc, 0xae, 0x13, 0xd7, 0x0d, 0x6a, 0xcc, 0x14, + 0x58, 0xb3, 0x73, 0xfb, 0x7b, 0xc5, 0xd3, 0x37, 0xd6, 0xd6, 0xaa, 0x31, 0x30, 0x4e, 0x6b, 0xa3, + 0xfe, 0x82, 0x02, 0x53, 0x41, 0x67, 0x30, 0x79, 0xa5, 0x4d, 0x5c, 0xcf, 0x45, 0x18, 0xce, 0x36, + 0xb5, 0x9d, 0x55, 0xdb, 0x5a, 0x69, 0x7b, 0x9a, 0x67, 0x58, 0x8d, 0x8a, 0xb5, 0x69, 0x1a, 0x8d, + 0x2d, 0x4f, 0x74, 0x6d, 0x76, 0x7f, 0xaf, 0x78, 0x76, 0x25, 0xb5, 0x06, 0xce, 0x68, 0x49, 0x3b, + 0xdd, 0xd4, 0x76, 0x12, 0x08, 0xa5, 0x4e, 0xaf, 0x24, 0xc1, 0x38, 0xad, 0x8d, 0xfa, 0x4e, 0x98, + 0xe2, 0xe3, 0xc0, 0xc4, 0xf5, 0x1c, 0x43, 0xf7, 0x0c, 0xdb, 0x42, 0x57, 0xa0, 0xdf, 0xd2, 0x9a, + 0x84, 0xf5, 0x70, 0x64, 0x61, 0xec, 0x4b, 0x7b, 0xc5, 0x37, 0xed, 0xef, 0x15, 0xfb, 0x57, 0xb5, + 0x26, 0xc1, 0x0c, 0xa2, 0xfe, 0x8f, 0x02, 0x5c, 0x4c, 0xb4, 0x7b, 0xce, 0xf0, 0xb6, 0x6e, 0xb5, + 0xe8, 0x7f, 0x2e, 0xfa, 0x41, 0x05, 0xa6, 0xb4, 0x78, 0x05, 0x86, 0x70, 0xf4, 0xe1, 0xc5, 0xb9, + 0xc3, 0x7f, 0xe0, 0x73, 0x09, 0x6a, 0x0b, 0xe7, 0x45, 0xbf, 0x92, 0x03, 0xc0, 0x49, 0xd2, 0xe8, + 0xe3, 0x0a, 0x0c, 0xd9, 0xbc, 0x73, 0x33, 0x85, 0x2b, 0x7d, 0x57, 0x47, 0x1f, 0xfe, 0xce, 0x23, + 0xe9, 0x86, 0x34, 0xe8, 0x39, 0xf1, 0x77, 0xd1, 0xf2, 0x9c, 0xdd, 0x85, 0x53, 0xa2, 0x7b, 0x43, + 0xa2, 0x14, 0xfb, 0xe4, 0x67, 0x1f, 0x87, 0x31, 0xb9, 0x26, 0x9a, 0x84, 0xbe, 0xdb, 0x84, 0x6f, + 0xd5, 0x11, 0x4c, 0xff, 0x45, 0xd3, 0x30, 0xb0, 0xad, 0x99, 0x6d, 0xc2, 0x96, 0x74, 0x04, 0xf3, + 0x1f, 0x8f, 0x17, 0xde, 0xad, 0xa8, 0x0f, 0xc3, 0xc0, 0x7c, 0xbd, 0x6e, 0x5b, 0xe8, 0x01, 0x18, + 0x22, 0x96, 0xb6, 0x61, 0x92, 0x3a, 0x6b, 0x38, 0x1c, 0xd2, 0x5b, 0xe4, 0xc5, 0xd8, 0x87, 0xab, + 0x3f, 0x56, 0x80, 0x41, 0xd6, 0xc8, 0x45, 0x3f, 0xa2, 0xc0, 0xe9, 0xdb, 0xed, 0x0d, 0xe2, 0x58, + 0xc4, 0x23, 0x6e, 0x59, 0x73, 0xb7, 0x36, 0x6c, 0xcd, 0xa9, 0x8b, 0x85, 0xb9, 0x9e, 0x67, 0x46, + 0x6e, 0x26, 0xd1, 0xf1, 0x3d, 0x98, 0x02, 0xc0, 0x69, 0xc4, 0xd1, 0x36, 0x8c, 0x59, 0x0d, 0xc3, + 0xda, 0xa9, 0x58, 0x0d, 0x87, 0xb8, 0x2e, 0x1b, 0xf4, 0xe8, 0xc3, 0x4f, 0xe7, 0xe9, 0xcc, 0xaa, + 0x84, 0x67, 0x61, 0x72, 0x7f, 0xaf, 0x38, 0x26, 0x97, 0xe0, 0x08, 0x1d, 0xf5, 0xef, 0x15, 0x38, + 0x35, 0x5f, 0x6f, 0x1a, 0x2e, 0xe5, 0xb4, 0x55, 0xb3, 0xdd, 0x30, 0xba, 0xd8, 0xfa, 0xe8, 0x19, + 0x18, 0xd4, 0x6d, 0x6b, 0xd3, 0x68, 0x88, 0x7e, 0xbe, 0x7d, 0x8e, 0x73, 0xae, 0x39, 0x99, 0x73, + 0xb1, 0xee, 0x09, 0x8e, 0x37, 0x87, 0xb5, 0x3b, 0x8b, 0x3e, 0x43, 0x5f, 0x80, 0xfd, 0xbd, 0xe2, + 0x60, 0x89, 0x21, 0xc0, 0x02, 0x11, 0xba, 0x0a, 0xc3, 0x75, 0xc3, 0xe5, 0x8b, 0xd9, 0xc7, 0x16, + 0x73, 0x6c, 0x7f, 0xaf, 0x38, 0x5c, 0x16, 0x65, 0x38, 0x80, 0xa2, 0x65, 0x98, 0xa6, 0x33, 0xc8, + 0xdb, 0xd5, 0x88, 0xee, 0x10, 0x8f, 0x76, 0x6d, 0xa6, 0x9f, 0x75, 0x77, 0x66, 0x7f, 0xaf, 0x38, + 0x7d, 0x33, 0x05, 0x8e, 0x53, 0x5b, 0xa9, 0x4b, 0x30, 0x3c, 0x6f, 0x12, 0x87, 0x32, 0x04, 0xf4, + 0x38, 0x4c, 0x90, 0xa6, 0x66, 0x98, 0x98, 0xe8, 0xc4, 0xd8, 0x26, 0x8e, 0x3b, 0xa3, 0x5c, 0xe9, + 0xbb, 0x3a, 0xb2, 0x80, 0xf6, 0xf7, 0x8a, 0x13, 0x8b, 0x11, 0x08, 0x8e, 0xd5, 0x54, 0x3f, 0xa2, + 0xc0, 0xe8, 0x7c, 0xbb, 0x6e, 0x78, 0x7c, 0x5c, 0xc8, 0x81, 0x51, 0x8d, 0xfe, 0xac, 0xda, 0xa6, + 0xa1, 0xef, 0x8a, 0xcd, 0xf5, 0x54, 0xae, 0xcf, 0x2d, 0x44, 0xb3, 0x70, 0x6a, 0x7f, 0xaf, 0x38, + 0x2a, 0x15, 0x60, 0x99, 0x88, 0xba, 0x05, 0x32, 0x0c, 0xbd, 0x0f, 0xc6, 0xf8, 0x70, 0x57, 0xb4, + 0x16, 0x26, 0x9b, 0xa2, 0x0f, 0xf7, 0x49, 0x6b, 0xe5, 0x13, 0x9a, 0xbb, 0xb5, 0xf1, 0x32, 0xd1, + 0x3d, 0x4c, 0x36, 0x89, 0x43, 0x2c, 0x9d, 0xf0, 0x6d, 0x53, 0x92, 0x1a, 0xe3, 0x08, 0x2a, 0xf5, + 0xff, 0x51, 0xe0, 0xd2, 0x7c, 0xdb, 0xdb, 0xb2, 0x1d, 0xe3, 0x2e, 0x71, 0xc2, 0xe9, 0x0e, 0x30, + 0xa0, 0x27, 0x61, 0x42, 0x0b, 0x2a, 0xac, 0x86, 0xdb, 0xe9, 0xac, 0xd8, 0x4e, 0x13, 0xf3, 0x11, + 0x28, 0x8e, 0xd5, 0x46, 0x0f, 0x03, 0xb8, 0xe1, 0xda, 0x32, 0x1e, 0xb0, 0x80, 0x44, 0x5b, 0x90, + 0x56, 0x55, 0xaa, 0xa5, 0xfe, 0x05, 0x3d, 0x0a, 0xb7, 0x35, 0xc3, 0xd4, 0x36, 0x0c, 0xd3, 0xf0, + 0x76, 0x5f, 0xb0, 0x2d, 0xd2, 0xc5, 0x6e, 0x5e, 0x87, 0x73, 0x6d, 0x4b, 0xe3, 0xed, 0x4c, 0xb2, + 0xc2, 0xf7, 0xef, 0xda, 0x6e, 0x8b, 0x70, 0x2e, 0x39, 0xb2, 0x70, 0x61, 0x7f, 0xaf, 0x78, 0x6e, + 0x3d, 0xbd, 0x0a, 0xce, 0x6a, 0x4b, 0x4f, 0x3d, 0x09, 0xf4, 0xac, 0x6d, 0xb6, 0x9b, 0x02, 0x6b, + 0x1f, 0xc3, 0xca, 0x4e, 0xbd, 0xf5, 0xd4, 0x1a, 0x38, 0xa3, 0xa5, 0xfa, 0x85, 0x02, 0x0c, 0x2e, + 0x68, 0xfa, 0xed, 0x76, 0x0b, 0xbd, 0x0d, 0x86, 0x5b, 0x8e, 0xbd, 0x6d, 0xd4, 0x89, 0x23, 0xc6, + 0x36, 0x29, 0xc6, 0x36, 0x5c, 0x15, 0xe5, 0x38, 0xa8, 0x81, 0x0c, 0x98, 0xf0, 0xff, 0x2f, 0xf5, + 0xf0, 0xe5, 0xb2, 0x2f, 0xa1, 0x1a, 0x41, 0x84, 0x63, 0x88, 0x91, 0x0a, 0x83, 0x0e, 0x69, 0xd0, + 0xa3, 0xae, 0x8f, 0x75, 0x8b, 0x7d, 0xed, 0x98, 0x95, 0x60, 0x01, 0x41, 0xef, 0x87, 0x09, 0xdd, + 0x21, 0x75, 0x62, 0x79, 0x86, 0x66, 0xba, 0x74, 0x73, 0x0e, 0x74, 0xbf, 0x39, 0x59, 0x27, 0x4a, + 0x91, 0xe6, 0x38, 0x86, 0x4e, 0xfd, 0x52, 0x01, 0xc6, 0xf8, 0x44, 0x2d, 0xb4, 0xf5, 0xdb, 0xc4, + 0x43, 0xdf, 0x05, 0xc3, 0x54, 0xbe, 0xab, 0x6b, 0x9e, 0x26, 0x3e, 0x84, 0x6f, 0xcb, 0x1c, 0x3a, + 0xfb, 0x06, 0x69, 0xed, 0x90, 0xfa, 0x0a, 0xf1, 0xb4, 0x70, 0xff, 0x85, 0x65, 0x38, 0xc0, 0x8a, + 0x36, 0xa1, 0xdf, 0x6d, 0x11, 0x5d, 0x4c, 0x6c, 0x39, 0xcf, 0xa7, 0x2e, 0xf7, 0xb8, 0xd6, 0x22, + 0x7a, 0xb8, 0x5d, 0xe9, 0x2f, 0xcc, 0xf0, 0x23, 0x0b, 0x06, 0x5d, 0x4f, 0xf3, 0xda, 0x2e, 0x9b, + 0xdf, 0xd1, 0x87, 0x97, 0x7a, 0xa6, 0xc4, 0xb0, 0x2d, 0x4c, 0x08, 0x5a, 0x83, 0xfc, 0x37, 0x16, + 0x54, 0xd4, 0x7f, 0xa3, 0xc0, 0xa4, 0x5c, 0x7d, 0xd9, 0x70, 0x3d, 0xf4, 0x1d, 0x89, 0xe9, 0x9c, + 0xeb, 0x6e, 0x3a, 0x69, 0x6b, 0x36, 0x99, 0xc1, 0x6e, 0xf5, 0x4b, 0xa4, 0xa9, 0x24, 0x30, 0x60, + 0x78, 0xa4, 0xe9, 0x4b, 0x29, 0x4f, 0xf7, 0x3a, 0xc2, 0x85, 0x71, 0x41, 0x6c, 0xa0, 0x42, 0xd1, + 0x62, 0x8e, 0x5d, 0xfd, 0x2e, 0x98, 0x96, 0x6b, 0xf9, 0xfb, 0x9a, 0xb2, 0x0c, 0x6f, 0xb7, 0x95, + 0x60, 0x19, 0xf4, 0x13, 0xc4, 0x0c, 0x82, 0xde, 0x1a, 0xec, 0x71, 0xce, 0x99, 0x82, 0xb9, 0x8b, + 0xee, 0x73, 0xf5, 0xc3, 0x7d, 0xd1, 0xb9, 0xa3, 0xcb, 0x88, 0xb6, 0x63, 0x5f, 0xee, 0xe8, 0xc3, + 0x37, 0x7a, 0x1d, 0xa0, 0xdf, 0xf5, 0xd7, 0x0b, 0x0f, 0xb8, 0x0a, 0xc3, 0x2e, 0x21, 0x75, 0xe9, + 0x5c, 0x66, 0xa7, 0x79, 0x4d, 0x94, 0xe1, 0x00, 0x7a, 0xfc, 0x9c, 0xe0, 0xb5, 0x7e, 0x40, 0xc9, + 0xdd, 0x2e, 0x4f, 0x06, 0x2f, 0x11, 0x4b, 0xd1, 0xcb, 0x64, 0x88, 0x0f, 0x27, 0x86, 0x18, 0xdd, + 0x85, 0x71, 0x53, 0x73, 0xbd, 0x5b, 0x2d, 0x7a, 0x6f, 0xf3, 0xf7, 0xcc, 0xe8, 0xc3, 0xf3, 0x79, 0x16, 0x7d, 0x59, 0x46, 0xb4, 0x30, 0xb5, 0xbf, 0x57, 0x1c, 0x8f, 0x14, 0xe1, 0x28, 0x29, 0xf4, 0x32, 0x8c, 0xd0, 0x82, 0x45, 0xc7, 0xb1, 0x1d, 0xc1, 0x2f, 0x9e, 0xc8, 0x4b, 0x97, 0x21, 0xe1, - 0x7a, 0x64, 0xf0, 0x13, 0x87, 0xe8, 0xd1, 0x7b, 0x01, 0xd9, 0x1b, 0x4c, 0x93, 0xaf, 0x5f, 0xe7, - 0x4a, 0x2a, 0x1d, 0x2c, 0x5d, 0xfe, 0xbe, 0x85, 0x59, 0xb1, 0x2f, 0xd1, 0xad, 0x44, 0x0d, 0x9c, - 0xd2, 0x0a, 0xdd, 0x06, 0x14, 0x28, 0xba, 0xfc, 0xb4, 0x3f, 0x60, 0x6b, 0x04, 0x95, 0xc4, 0xd6, - 0x38, 0x4b, 0x89, 0x5d, 0x4f, 0xa0, 0xc0, 0x29, 0x68, 0xd5, 0xdf, 0x2c, 0xc0, 0x28, 0xdf, 0x22, - 0x5c, 0x19, 0x39, 0xfe, 0xb3, 0x82, 0x44, 0xce, 0x8a, 0x52, 0xfe, 0xcf, 0x9f, 0x75, 0x38, 0xf3, - 0xa8, 0x68, 0xc6, 0x8e, 0x8a, 0xc5, 0x5e, 0x09, 0x75, 0x3e, 0x29, 0xfe, 0x58, 0x81, 0x53, 0x52, - 0xed, 0x13, 0x38, 0x28, 0xea, 0xd1, 0x83, 0xe2, 0xa9, 0x1e, 0xc7, 0x97, 0x71, 0x4e, 0xd8, 0x91, - 0x61, 0x31, 0x1e, 0xfe, 0x30, 0xc0, 0x06, 0x63, 0x27, 0x92, 0x68, 0x1b, 0x2c, 0xf9, 0x42, 0x00, - 0xc1, 0x52, 0xad, 0x08, 0x53, 0x2c, 0x74, 0x62, 0x8a, 0xea, 0x7f, 0xe8, 0x83, 0xa9, 0xc4, 0xb4, - 0x27, 0xf9, 0x88, 0xf2, 0x75, 0xe2, 0x23, 0x85, 0xaf, 0x07, 0x1f, 0xe9, 0xcb, 0xc5, 0x47, 0xba, - 0x3f, 0x88, 0x1c, 0x40, 0x4d, 0xa3, 0xc1, 0x9b, 0xd5, 0x3c, 0xcd, 0xf1, 0xd6, 0x8c, 0x26, 0x11, - 0x1c, 0xe7, 0x9b, 0xbb, 0xdb, 0xb2, 0xb4, 0x05, 0x67, 0x3c, 0x2b, 0x09, 0x4c, 0x38, 0x05, 0xbb, - 0xfa, 0x5d, 0x05, 0x18, 0x5a, 0xd0, 0x5c, 0xd6, 0xd3, 0x0f, 0xc2, 0x98, 0x40, 0x5d, 0x69, 0x6a, - 0x0d, 0xd2, 0x8b, 0x39, 0x42, 0xa0, 0x5c, 0x91, 0xd0, 0x71, 0x8d, 0x4e, 0x2e, 0xc1, 0x11, 0x72, - 0x68, 0x17, 0x46, 0x9b, 0xa1, 0xf6, 0x22, 0x96, 0x78, 0xa9, 0x77, 0xea, 0x14, 0x1b, 0x57, 0x5b, - 0xa5, 0x02, 0x2c, 0xd3, 0x52, 0x5f, 0x82, 0xd3, 0x29, 0x3d, 0xee, 0x42, 0x71, 0x7b, 0x0b, 0x0c, - 0x51, 0xdd, 0x3b, 0x14, 0xc3, 0x46, 0xf7, 0xf7, 0x8a, 0x43, 0xcf, 0xf2, 0x22, 0xec, 0xc3, 0xd4, - 0x77, 0x51, 0x01, 0x20, 0xde, 0xa7, 0x6e, 0x0c, 0x7c, 0x03, 0x00, 0xa5, 0x79, 0x6c, 0x7b, 0x7c, - 0x2b, 0x3d, 0x05, 0x03, 0xad, 0x2d, 0xcd, 0xf5, 0x5b, 0x3c, 0xe0, 0xb3, 0x8a, 0x2a, 0x2d, 0xbc, - 0xb7, 0x57, 0x9c, 0x91, 0x05, 0x11, 0xd1, 0x88, 0xc1, 0x30, 0x6f, 0x47, 0x77, 0x18, 0xdd, 0xe4, - 0x25, 0xbb, 0xd9, 0x32, 0x09, 0x85, 0xb2, 0x1d, 0x56, 0xc8, 0xb7, 0xc3, 0x96, 0x13, 0x98, 0x70, - 0x0a, 0x76, 0x9f, 0x66, 0xc5, 0x32, 0x3c, 0x43, 0x0b, 0x68, 0xf6, 0xe5, 0xa7, 0x19, 0xc5, 0x84, - 0x53, 0xb0, 0xa3, 0x4f, 0x2a, 0x30, 0x1b, 0x2d, 0x5e, 0x32, 0x2c, 0xc3, 0xdd, 0x22, 0x75, 0x46, - 0xbc, 0xff, 0xd0, 0xc4, 0x2f, 0xef, 0xef, 0x15, 0x67, 0x97, 0x33, 0x31, 0xe2, 0x0e, 0xd4, 0xd0, - 0xf7, 0x29, 0x70, 0x21, 0x36, 0x2f, 0x8e, 0xd1, 0x68, 0x10, 0x47, 0xf4, 0xe6, 0xf0, 0x1f, 0x78, - 0x71, 0x7f, 0xaf, 0x78, 0x61, 0x39, 0x1b, 0x25, 0xee, 0x44, 0x0f, 0xfd, 0x84, 0x02, 0x67, 0x5b, - 0xc4, 0xaa, 0x1b, 0x56, 0xe3, 0x39, 0xdb, 0xb9, 0x4d, 0x1c, 0x17, 0xdb, 0xa6, 0x69, 0xb7, 0x3d, - 0x77, 0x66, 0x90, 0x9d, 0x61, 0x95, 0x3c, 0xdf, 0x5c, 0x35, 0x0d, 0xe3, 0xc2, 0x65, 0xb1, 0x45, - 0xcf, 0xa6, 0x82, 0x5d, 0x9c, 0xd1, 0x11, 0xf5, 0x37, 0x14, 0xe8, 0x2b, 0xe1, 0x0a, 0x7a, 0x30, - 0xf2, 0x89, 0x9c, 0x93, 0x3f, 0x91, 0x7b, 0x7b, 0xc5, 0xa1, 0x12, 0xae, 0x48, 0x1f, 0xe3, 0xf7, - 0x29, 0x30, 0xa5, 0xdb, 0x96, 0xa7, 0xd1, 0xb9, 0xc3, 0x5c, 0x56, 0xf6, 0xcf, 0xe5, 0x5c, 0xca, - 0x70, 0x29, 0x86, 0x2c, 0x34, 0x76, 0xc7, 0x21, 0x2e, 0x4e, 0x52, 0x56, 0x3f, 0xaf, 0xc0, 0x74, - 0x49, 0x6b, 0x09, 0x53, 0x50, 0x99, 0x6c, 0x1a, 0x74, 0x83, 0x74, 0x63, 0xd9, 0x47, 0x5b, 0x30, - 0xc8, 0xac, 0xcd, 0x6e, 0x2f, 0xba, 0x7c, 0x48, 0xfb, 0x59, 0x86, 0x8b, 0xdb, 0x41, 0xf8, 0xff, - 0x58, 0xe0, 0x57, 0x9f, 0x80, 0xc9, 0x78, 0x3d, 0x54, 0xf4, 0x65, 0x1a, 0x6e, 0x7c, 0x1c, 0x89, - 0x8b, 0x23, 0x8f, 0x0f, 0xff, 0xff, 0xaf, 0x15, 0xdf, 0xf4, 0xe1, 0x3f, 0xbb, 0xf2, 0x26, 0xf5, - 0x2b, 0x0a, 0x8c, 0x95, 0x4c, 0xbb, 0x5d, 0xaf, 0x3a, 0xf6, 0xa6, 0x61, 0x92, 0x37, 0x86, 0x95, - 0x43, 0xee, 0x71, 0x96, 0xe8, 0xca, 0xac, 0x0e, 0x72, 0xc5, 0x37, 0x88, 0xd5, 0x41, 0xee, 0x72, - 0x86, 0x34, 0xf9, 0x22, 0x9c, 0x91, 0x6b, 0x85, 0x26, 0xd3, 0x2b, 0xd0, 0x7f, 0xdb, 0xb0, 0xea, - 0xf1, 0x8d, 0x79, 0xd3, 0xb0, 0xea, 0x98, 0x41, 0x82, 0xad, 0x5b, 0xc8, 0x3c, 0xb3, 0xfe, 0x66, - 0x24, 0x3a, 0x6d, 0x4c, 0x58, 0xbd, 0x0a, 0xc3, 0xba, 0xb6, 0xd0, 0xb6, 0xea, 0x66, 0xb0, 0xeb, - 0xe9, 0x14, 0x94, 0xe6, 0x79, 0x19, 0x0e, 0xa0, 0xe8, 0x2e, 0x40, 0x78, 0x3b, 0xd1, 0x8b, 0x10, - 0x10, 0x5e, 0x7c, 0xd4, 0x88, 0xe7, 0x19, 0x56, 0xc3, 0x0d, 0xf7, 0x55, 0x08, 0xc3, 0x12, 0x35, - 0xf4, 0x41, 0x18, 0x97, 0x25, 0x12, 0x6e, 0x26, 0xcd, 0xb9, 0x0c, 0x11, 0xd1, 0xe7, 0x8c, 0x20, - 0x3c, 0x2e, 0x97, 0xba, 0x38, 0x4a, 0x0d, 0xed, 0x06, 0xf2, 0x17, 0x37, 0xd2, 0xf6, 0xe7, 0xd7, - 0x28, 0x64, 0xd1, 0x67, 0x5a, 0x10, 0x1f, 0x8b, 0x18, 0x8d, 0x23, 0xa4, 0x52, 0x0c, 0x33, 0x03, - 0xc7, 0x65, 0x98, 0x21, 0x30, 0xc4, 0x4d, 0x53, 0xfe, 0x71, 0xf3, 0x78, 0x9e, 0x01, 0x72, 0x2b, - 0x57, 0x78, 0xdd, 0xc6, 0x7f, 0xbb, 0xd8, 0xc7, 0x8d, 0xb6, 0x61, 0x8c, 0x0a, 0xd6, 0x35, 0x62, - 0x12, 0xdd, 0xb3, 0x9d, 0x99, 0xa1, 0xfc, 0xd7, 0x59, 0x35, 0x09, 0x0f, 0x97, 0x62, 0xe5, 0x12, - 0x1c, 0xa1, 0x13, 0x58, 0xee, 0x86, 0x33, 0x2d, 0x77, 0x6d, 0x18, 0xdd, 0x96, 0x4c, 0xf1, 0x23, - 0x6c, 0x12, 0x9e, 0xcc, 0xd3, 0xb1, 0xd0, 0x2e, 0xbf, 0x70, 0x5a, 0x10, 0x1a, 0x95, 0x6d, 0xf8, - 0x32, 0x1d, 0xb4, 0x01, 0x43, 0x1b, 0x5c, 0x06, 0x9d, 0x01, 0x36, 0x17, 0xef, 0xe9, 0x41, 0xb4, - 0xe6, 0x72, 0xae, 0xf8, 0x81, 0x7d, 0xc4, 0xe8, 0x25, 0x18, 0x34, 0x8d, 0xa6, 0xe1, 0xb9, 0x33, - 0xa3, 0x8c, 0x44, 0xae, 0xa5, 0x5d, 0x66, 0x18, 0xf8, 0x61, 0xc5, 0xff, 0xc7, 0x02, 0x2b, 0xfa, - 0xb4, 0x02, 0xa7, 0xc5, 0x36, 0x0c, 0x0e, 0x2d, 0x83, 0xb8, 0x33, 0x63, 0x6c, 0x0e, 0x6f, 0xf4, - 0x76, 0x48, 0x86, 0x07, 0xf4, 0xc2, 0x05, 0x31, 0x9b, 0xa7, 0x57, 0x92, 0xc4, 0x70, 0x5a, 0x0f, - 0xd4, 0x7f, 0x39, 0x0e, 0x53, 0x25, 0xb3, 0xed, 0x7a, 0xc4, 0x99, 0x17, 0x9e, 0x2c, 0xc4, 0x41, - 0x1f, 0x55, 0xe0, 0x2c, 0xfb, 0xb7, 0x6c, 0xdf, 0xb1, 0xca, 0xc4, 0xd4, 0x76, 0xe7, 0x37, 0x69, - 0x8d, 0x7a, 0xfd, 0x70, 0x87, 0x47, 0xb9, 0x2d, 0xd4, 0x64, 0x76, 0x63, 0x53, 0x4b, 0xc5, 0x88, - 0x33, 0x28, 0xa1, 0xef, 0x55, 0xe0, 0x7c, 0x0a, 0xa8, 0x4c, 0x4c, 0xe2, 0xf9, 0xc2, 0xff, 0x61, - 0xfb, 0x71, 0x69, 0x7f, 0xaf, 0x78, 0xbe, 0x96, 0x85, 0x14, 0x67, 0xd3, 0x43, 0xdf, 0xaf, 0xc0, - 0x6c, 0x0a, 0x74, 0x49, 0x33, 0xcc, 0xb6, 0xe3, 0xeb, 0x05, 0x87, 0xed, 0x0e, 0x13, 0xcf, 0x6b, - 0x99, 0x58, 0x71, 0x07, 0x8a, 0xe8, 0x43, 0x70, 0x26, 0x80, 0xae, 0x5b, 0x16, 0x21, 0xf5, 0x88, - 0x96, 0x70, 0xd8, 0xae, 0x9c, 0xdf, 0xdf, 0x2b, 0x9e, 0xa9, 0xa5, 0x21, 0xc4, 0xe9, 0x74, 0x50, - 0x03, 0x2e, 0x85, 0x00, 0xcf, 0x30, 0x8d, 0xbb, 0x5c, 0x91, 0xd9, 0x72, 0x88, 0xbb, 0x65, 0x9b, - 0x75, 0xc6, 0x8a, 0x95, 0x85, 0x37, 0xef, 0xef, 0x15, 0x2f, 0xd5, 0x3a, 0x55, 0xc4, 0x9d, 0xf1, - 0xa0, 0x3a, 0x8c, 0xb9, 0xba, 0x66, 0x55, 0x2c, 0x8f, 0x38, 0xdb, 0x9a, 0x39, 0x33, 0x98, 0x6b, - 0x80, 0x9c, 0x01, 0x4a, 0x78, 0x70, 0x04, 0x2b, 0x7a, 0x37, 0x0c, 0x93, 0x9d, 0x96, 0x66, 0xd5, - 0x09, 0x67, 0xba, 0x23, 0x0b, 0x17, 0xe9, 0x51, 0xbf, 0x28, 0xca, 0xee, 0xed, 0x15, 0xc7, 0xfc, - 0xff, 0x57, 0xec, 0x3a, 0xc1, 0x41, 0x6d, 0xf4, 0x01, 0x98, 0x66, 0xae, 0x36, 0x75, 0xc2, 0x8e, - 0x10, 0xd7, 0xd7, 0x15, 0x87, 0x73, 0xf5, 0x93, 0x5d, 0xc3, 0xaf, 0xa4, 0xe0, 0xc3, 0xa9, 0x54, - 0xe8, 0x32, 0x34, 0xb5, 0x9d, 0xeb, 0x8e, 0xa6, 0x93, 0xcd, 0xb6, 0xb9, 0x46, 0x9c, 0xa6, 0x61, - 0x71, 0x63, 0x09, 0xd1, 0x6d, 0xab, 0x4e, 0x19, 0xb5, 0x72, 0x75, 0x80, 0x2f, 0xc3, 0x4a, 0xa7, - 0x8a, 0xb8, 0x33, 0x1e, 0xf4, 0x08, 0x8c, 0x19, 0x0d, 0xcb, 0x76, 0xc8, 0x9a, 0x66, 0x58, 0x9e, - 0x3b, 0x03, 0x4c, 0xc8, 0x66, 0xd3, 0x5a, 0x91, 0xca, 0x71, 0xa4, 0x16, 0xda, 0x06, 0x64, 0x91, - 0x3b, 0x55, 0xbb, 0xce, 0xb6, 0xc0, 0x7a, 0x8b, 0x6d, 0x64, 0xc1, 0x66, 0x0f, 0x3b, 0x35, 0x4c, - 0x95, 0x5e, 0x4d, 0x60, 0xc3, 0x29, 0x14, 0xd0, 0x12, 0xa0, 0xa6, 0xb6, 0xb3, 0xd8, 0x6c, 0x79, - 0xbb, 0x0b, 0x6d, 0xf3, 0xb6, 0xe0, 0x1a, 0x63, 0x6c, 0x2e, 0xb8, 0xa1, 0x29, 0x01, 0xc5, 0x29, - 0x2d, 0x90, 0x06, 0x17, 0xf8, 0x78, 0xca, 0x1a, 0x69, 0xda, 0x96, 0x4b, 0x3c, 0x57, 0xda, 0xa4, - 0x33, 0xe3, 0xcc, 0xe1, 0x82, 0x29, 0xb6, 0x95, 0xec, 0x6a, 0xb8, 0x13, 0x8e, 0xa8, 0xcb, 0xd9, - 0xc4, 0x01, 0x2e, 0x67, 0x8f, 0xc2, 0xb8, 0xeb, 0x69, 0x8e, 0xd7, 0x6e, 0x89, 0x65, 0x38, 0xc5, - 0x96, 0x81, 0xd9, 0x21, 0x6b, 0x32, 0x00, 0x47, 0xeb, 0xd1, 0xe5, 0xe3, 0xc6, 0x66, 0xd1, 0x6e, - 0x32, 0x5c, 0xbe, 0x9a, 0x54, 0x8e, 0x23, 0xb5, 0xd0, 0x3a, 0x9c, 0x6b, 0x6a, 0x3b, 0xc1, 0xe7, - 0x5b, 0xd5, 0x1c, 0xcd, 0x34, 0x89, 0x69, 0xb8, 0xcd, 0x99, 0x29, 0xd6, 0x53, 0x76, 0xc3, 0xbf, - 0x92, 0x5e, 0x05, 0x67, 0xb5, 0x15, 0x3e, 0x68, 0x65, 0x47, 0x33, 0x22, 0x28, 0x51, 0xc4, 0x07, - 0x2d, 0x0e, 0xc6, 0x69, 0x6d, 0xd4, 0xff, 0xd1, 0x0f, 0x33, 0x89, 0x13, 0xcc, 0x77, 0x24, 0x3b, - 0x90, 0x47, 0x29, 0x47, 0xc4, 0xa3, 0x5a, 0x70, 0x25, 0xa8, 0x70, 0xbd, 0xd5, 0x4e, 0xa5, 0x55, - 0x60, 0xb4, 0xee, 0xdf, 0xdf, 0x2b, 0x5e, 0xa9, 0x1d, 0x50, 0x17, 0x1f, 0x88, 0x2d, 0x9b, 0xff, - 0xf7, 0x9d, 0x10, 0xff, 0xff, 0x00, 0x4c, 0x4b, 0x00, 0x87, 0x68, 0xf5, 0xdd, 0x1e, 0xce, 0x1f, - 0xc6, 0xf6, 0x6a, 0x29, 0xf8, 0x70, 0x2a, 0x95, 0x4c, 0xa6, 0x3b, 0x70, 0x12, 0x4c, 0x57, 0xdd, - 0xeb, 0x83, 0x91, 0x92, 0x6d, 0xd5, 0xb9, 0x5d, 0xe4, 0xa1, 0xc8, 0xad, 0xf7, 0x25, 0x59, 0x76, - 0xbe, 0xb7, 0x57, 0x1c, 0x0f, 0x2a, 0x4a, 0xc2, 0xf4, 0x63, 0xc1, 0xfd, 0x12, 0xd7, 0x48, 0xdf, - 0x1c, 0xbd, 0x18, 0xba, 0xb7, 0x57, 0x3c, 0x15, 0x34, 0x8b, 0xde, 0x15, 0x51, 0x8e, 0x6a, 0x6a, - 0xae, 0xb7, 0xe6, 0x68, 0x96, 0x6b, 0xf4, 0x60, 0x98, 0x0c, 0x2e, 0x04, 0x96, 0x13, 0xd8, 0x70, - 0x0a, 0x05, 0xf4, 0x32, 0x4c, 0xd0, 0xd2, 0xf5, 0x56, 0x5d, 0xf3, 0x48, 0x4e, 0x7b, 0x64, 0xe0, - 0xc3, 0xb4, 0x1c, 0xc1, 0x84, 0x63, 0x98, 0xb9, 0x97, 0x80, 0xe6, 0xda, 0x16, 0x5b, 0xcf, 0x88, - 0x97, 0x00, 0x2d, 0xc5, 0x02, 0x8a, 0x1e, 0x80, 0xa1, 0x26, 0x71, 0x5d, 0xad, 0x41, 0x98, 0x54, - 0x30, 0x12, 0x2a, 0x56, 0x2b, 0xbc, 0x18, 0xfb, 0x70, 0xf4, 0x36, 0x18, 0xd0, 0xed, 0x3a, 0x71, - 0x67, 0x86, 0x18, 0xe3, 0xa3, 0x67, 0xc0, 0x40, 0x89, 0x16, 0xdc, 0xdb, 0x2b, 0x8e, 0xb0, 0xeb, - 0x13, 0xfa, 0x0b, 0xf3, 0x4a, 0xea, 0x8f, 0x2b, 0x30, 0x19, 0x37, 0x96, 0x75, 0xe1, 0xdd, 0x70, - 0x72, 0x8e, 0x02, 0xea, 0xa7, 0x15, 0x18, 0xa3, 0x3d, 0x74, 0x6c, 0xb3, 0x6a, 0x6a, 0x16, 0x41, - 0xdf, 0xa3, 0xc0, 0xe4, 0x96, 0xd1, 0xd8, 0x92, 0xfd, 0xb8, 0x84, 0xb8, 0x9e, 0xcb, 0xd8, 0x74, - 0x23, 0x86, 0x6b, 0x61, 0x7a, 0x7f, 0xaf, 0x38, 0x19, 0x2f, 0xc5, 0x09, 0x9a, 0xea, 0x5f, 0x16, - 0xe0, 0x9c, 0xdc, 0xb3, 0xf9, 0xd0, 0x45, 0x1e, 0xfd, 0xb1, 0x02, 0xd0, 0x34, 0xac, 0x79, 0xd3, - 0xb4, 0xef, 0x30, 0xe7, 0x53, 0xaa, 0x00, 0xbd, 0x98, 0xd7, 0xc8, 0x99, 0x42, 0x61, 0x6e, 0x25, - 0xc0, 0xce, 0x2f, 0x26, 0x9f, 0xf7, 0x8d, 0x27, 0x21, 0xe0, 0xde, 0x5e, 0xb1, 0x98, 0xf4, 0xcb, - 0x9f, 0xc3, 0xc2, 0xf9, 0x7d, 0xd9, 0x70, 0xbd, 0x8f, 0xfe, 0x45, 0xc7, 0x2a, 0xfc, 0x86, 0x32, - 0x1c, 0xc8, 0x6c, 0x13, 0x4e, 0xc5, 0x08, 0xa7, 0x38, 0xe6, 0x96, 0x65, 0xc7, 0xdc, 0x03, 0x98, - 0xd4, 0x9c, 0xef, 0x8a, 0x3f, 0xf7, 0x4c, 0x5b, 0xb3, 0x3c, 0x3a, 0xd3, 0x92, 0x23, 0xef, 0x9f, - 0x17, 0x60, 0x5a, 0x4c, 0x80, 0x49, 0x55, 0x94, 0x96, 0x69, 0xef, 0x36, 0x89, 0x75, 0x12, 0xce, - 0x5a, 0xfe, 0x47, 0x50, 0xc8, 0xfc, 0x08, 0x9a, 0x89, 0x8f, 0xa0, 0x2f, 0xcf, 0x47, 0x10, 0xf0, - 0x8a, 0x03, 0x0c, 0x33, 0x18, 0xce, 0x1a, 0x16, 0xed, 0xe8, 0x75, 0xb6, 0x61, 0x42, 0x97, 0x4a, - 0xc6, 0x9f, 0x86, 0xb9, 0xee, 0x59, 0x49, 0xad, 0x81, 0x33, 0x5a, 0xaa, 0x7f, 0xad, 0xc0, 0x4c, - 0xda, 0xfc, 0x9e, 0x80, 0x2d, 0xb5, 0x19, 0xb5, 0xa5, 0xde, 0xe8, 0xe1, 0xdb, 0x88, 0x74, 0x3d, - 0xc3, 0xa6, 0xfa, 0x57, 0x05, 0x38, 0x1b, 0x56, 0xaf, 0x58, 0xae, 0xa7, 0x99, 0x26, 0x97, 0x4b, - 0x8f, 0x7f, 0x2f, 0xb5, 0x22, 0x26, 0xf1, 0xd5, 0xde, 0x86, 0x2a, 0xf7, 0x3d, 0xd3, 0xaf, 0x63, - 0x27, 0xe6, 0xd7, 0x51, 0x3d, 0x42, 0x9a, 0x9d, 0x5d, 0x3c, 0xfe, 0x93, 0x02, 0xb3, 0xe9, 0x0d, - 0x4f, 0x60, 0x53, 0xd9, 0xd1, 0x4d, 0xf5, 0xde, 0xa3, 0x1b, 0x75, 0xc6, 0xb6, 0xfa, 0xf9, 0x42, - 0xd6, 0x68, 0x99, 0x5d, 0x7d, 0x13, 0x4e, 0x39, 0xa4, 0x61, 0xb8, 0x9e, 0x70, 0x40, 0x38, 0x9c, - 0x8f, 0xb5, 0x7f, 0x9f, 0x76, 0x0a, 0x47, 0x71, 0xe0, 0x38, 0x52, 0xb4, 0x0a, 0x43, 0x2e, 0x21, - 0x75, 0x8a, 0xbf, 0xd0, 0x3d, 0xfe, 0x40, 0x88, 0xa8, 0xf1, 0xb6, 0xd8, 0x47, 0x82, 0xbe, 0x0d, - 0xc6, 0xeb, 0xc1, 0x17, 0x45, 0xb1, 0xf6, 0x75, 0x8f, 0x95, 0xa9, 0x68, 0x65, 0xb9, 0x35, 0x8e, - 0x22, 0x53, 0xff, 0x4e, 0x81, 0x8b, 0x9d, 0xf6, 0x16, 0x7a, 0x05, 0x40, 0xf7, 0xa5, 0x42, 0x57, - 0x1c, 0x9e, 0x4f, 0xe4, 0x5c, 0x4b, 0x8e, 0x25, 0xfc, 0x40, 0x83, 0x22, 0x17, 0x4b, 0x44, 0x52, - 0xbc, 0xfd, 0x0a, 0xc7, 0xe4, 0xed, 0xa7, 0xfe, 0x67, 0x45, 0x66, 0x45, 0xf2, 0xda, 0xbe, 0xd1, - 0x58, 0x91, 0xdc, 0xf7, 0xcc, 0x7b, 0xba, 0x2f, 0x17, 0xe0, 0x4a, 0x7a, 0x13, 0xe9, 0x3c, 0x7f, - 0x1a, 0x06, 0x5b, 0xfc, 0x1d, 0x04, 0x77, 0x09, 0xbf, 0x4a, 0x39, 0x0b, 0x7f, 0xa5, 0x70, 0x6f, - 0xaf, 0x38, 0x9b, 0xc6, 0xe8, 0xc5, 0xfb, 0x06, 0xd1, 0x0e, 0x19, 0xb1, 0x0b, 0x05, 0x2e, 0xb4, - 0xbf, 0xa3, 0x4b, 0xe6, 0xa2, 0x6d, 0x10, 0xb3, 0xeb, 0x3b, 0x84, 0x8f, 0x28, 0x30, 0x11, 0xd9, - 0xd1, 0xee, 0xcc, 0x00, 0xdb, 0xa3, 0xb9, 0x1c, 0xad, 0x22, 0x9f, 0x4a, 0x28, 0x0d, 0x44, 0x8a, - 0x5d, 0x1c, 0x23, 0x18, 0x63, 0xb3, 0xf2, 0xac, 0xbe, 0xe1, 0xd8, 0xac, 0xdc, 0xf9, 0x0c, 0x36, - 0xfb, 0x63, 0x85, 0xac, 0xd1, 0x32, 0x36, 0x7b, 0x07, 0x46, 0x7c, 0x31, 0xd2, 0x67, 0x17, 0x4b, - 0xbd, 0xf6, 0x89, 0xa3, 0x5b, 0x98, 0x12, 0xfd, 0x19, 0xf1, 0x4b, 0x5c, 0x1c, 0xd2, 0x42, 0xdf, - 0xad, 0x00, 0x84, 0x0b, 0x23, 0x3e, 0xaa, 0xb5, 0xa3, 0x9b, 0x0e, 0x49, 0xac, 0x99, 0xa0, 0x9f, - 0xb4, 0xb4, 0x29, 0x24, 0xba, 0xea, 0xbf, 0xe9, 0x07, 0x94, 0xec, 0x7b, 0x77, 0xd7, 0xc5, 0x07, - 0x08, 0xb9, 0x2d, 0x98, 0x74, 0xa8, 0xb4, 0xa8, 0x1b, 0x26, 0x53, 0x59, 0xed, 0xb6, 0x97, 0xd3, - 0xf2, 0xc1, 0xd4, 0x2a, 0x1c, 0xc3, 0x85, 0x13, 0xd8, 0xd1, 0x5b, 0x60, 0xa8, 0xe5, 0x18, 0x4d, - 0xcd, 0xd9, 0x65, 0x4a, 0xf1, 0x30, 0xbf, 0xcb, 0xaa, 0xf2, 0x22, 0xec, 0xc3, 0xd0, 0x07, 0x60, - 0xc4, 0x34, 0x36, 0x89, 0xbe, 0xab, 0x9b, 0x44, 0x98, 0xca, 0x6f, 0x1d, 0xcd, 0x9a, 0x2f, 0xfb, - 0x68, 0x85, 0x07, 0xa2, 0xff, 0x13, 0x87, 0x04, 0x51, 0x05, 0x4e, 0xdf, 0x61, 0x3e, 0x31, 0x26, - 0x71, 0xdd, 0x5a, 0xbb, 0xd5, 0xb2, 0x1d, 0x8f, 0xd4, 0x99, 0x41, 0x7d, 0x98, 0x1b, 0xf6, 0x9e, - 0x4b, 0x82, 0x71, 0x5a, 0x1b, 0xf4, 0x04, 0x80, 0xd6, 0xf6, 0x6c, 0xfe, 0x20, 0x71, 0x66, 0x98, - 0x69, 0xed, 0x97, 0xe8, 0x62, 0xcf, 0x07, 0xa5, 0xf7, 0xf6, 0x8a, 0xa3, 0xc2, 0xf6, 0xc7, 0x96, - 0x46, 0x6a, 0x80, 0x5e, 0x84, 0x69, 0x9d, 0x83, 0x4a, 0x76, 0xb3, 0xa5, 0x79, 0x86, 0xd0, 0x88, - 0x47, 0x18, 0xa2, 0x6f, 0xda, 0xdf, 0x2b, 0x4e, 0x97, 0x52, 0xe0, 0x71, 0x94, 0xa9, 0x48, 0xd4, - 0x4f, 0x16, 0xe0, 0x42, 0x87, 0x09, 0x42, 0x98, 0x7e, 0x78, 0x62, 0xfd, 0xc4, 0x36, 0x7b, 0x84, - 0x7f, 0x2c, 0xa2, 0xf0, 0xde, 0x5e, 0xf1, 0xbe, 0x0e, 0x08, 0x6a, 0x74, 0x9f, 0x93, 0xc6, 0x2e, - 0x0e, 0xd1, 0xa0, 0x0a, 0x0c, 0xd6, 0xc3, 0xbb, 0xaf, 0x91, 0x85, 0x87, 0xe8, 0x51, 0xc0, 0xad, - 0xd4, 0xdd, 0x62, 0x13, 0x08, 0xd0, 0x32, 0x0c, 0x71, 0x9f, 0x4a, 0x22, 0x8e, 0x95, 0x87, 0x99, - 0xc9, 0x84, 0x17, 0x75, 0x8b, 0xcc, 0x47, 0xa1, 0xfe, 0xad, 0x02, 0x43, 0x25, 0xdb, 0x21, 0xe5, - 0xd5, 0x1a, 0xda, 0x85, 0x51, 0xe9, 0x45, 0xbc, 0x60, 0xb1, 0x39, 0x79, 0x0e, 0xc3, 0x28, 0xa9, - 0xf6, 0xfe, 0x1b, 0xbe, 0xa0, 0x00, 0xcb, 0xb4, 0xd0, 0x2b, 0x74, 0xce, 0xef, 0x38, 0x86, 0x47, - 0x09, 0xf7, 0xe2, 0x64, 0xc3, 0x09, 0x63, 0x1f, 0x17, 0xdf, 0xed, 0xc1, 0x4f, 0x1c, 0x52, 0x51, - 0xab, 0x94, 0xbd, 0xc4, 0xbb, 0x89, 0x1e, 0x87, 0xfe, 0xa6, 0x5d, 0xf7, 0xd7, 0xfd, 0xad, 0x3e, - 0xf3, 0x58, 0xb1, 0xeb, 0x74, 0x6e, 0xcf, 0x26, 0x5b, 0xb0, 0xfb, 0x24, 0xd6, 0x46, 0x5d, 0x85, - 0xc9, 0x38, 0x7d, 0xf4, 0x38, 0x4c, 0xe8, 0x76, 0xb3, 0x69, 0x5b, 0xb5, 0xf6, 0xe6, 0xa6, 0xb1, - 0x43, 0x22, 0x8f, 0x2b, 0x4b, 0x11, 0x08, 0x8e, 0xd5, 0x54, 0x3f, 0xa7, 0x40, 0x1f, 0x5d, 0x17, - 0x15, 0x06, 0xeb, 0x76, 0x53, 0x33, 0x2c, 0xd1, 0x2b, 0x76, 0x4b, 0x5d, 0x66, 0x25, 0x58, 0x40, - 0x50, 0x0b, 0x46, 0x7c, 0x89, 0xac, 0x27, 0xb7, 0xf0, 0xf2, 0x6a, 0x2d, 0x78, 0x55, 0x13, 0x1c, - 0x13, 0x7e, 0x89, 0x8b, 0x43, 0x22, 0xaa, 0x06, 0x53, 0xe5, 0xd5, 0x5a, 0xc5, 0xd2, 0xcd, 0x76, - 0x9d, 0x2c, 0xee, 0xb0, 0x3f, 0x94, 0xcf, 0x19, 0xbc, 0x44, 0x8c, 0x93, 0xf1, 0x39, 0x51, 0x09, - 0xfb, 0x30, 0x5a, 0x8d, 0xf0, 0x16, 0xe2, 0xad, 0x21, 0xab, 0x26, 0x90, 0x60, 0x1f, 0xa6, 0x7e, - 0xa5, 0x00, 0xa3, 0x52, 0x87, 0x90, 0x09, 0x43, 0x7c, 0xb8, 0x6e, 0x2f, 0xef, 0xc9, 0x13, 0xbd, - 0xe6, 0xd4, 0xf9, 0x84, 0xba, 0xd8, 0x27, 0x21, 0xf3, 0xec, 0x42, 0x07, 0x9e, 0x3d, 0x17, 0x79, - 0xb2, 0xc9, 0x3f, 0xc9, 0x89, 0xec, 0xe7, 0x9a, 0xe8, 0xa2, 0x38, 0x9e, 0xb8, 0x5f, 0xf6, 0x70, - 0xec, 0x68, 0xda, 0x84, 0x81, 0xbb, 0xb6, 0x45, 0x5c, 0x61, 0x0b, 0x3f, 0xa2, 0x01, 0x32, 0x6f, - 0xba, 0x17, 0x28, 0x5e, 0xcc, 0xd1, 0xab, 0x3f, 0xa1, 0x00, 0x94, 0x35, 0x4f, 0xe3, 0xae, 0x1b, - 0x5d, 0x78, 0x07, 0x5e, 0x8c, 0x9c, 0xaa, 0xc3, 0x89, 0x97, 0x61, 0xfd, 0xae, 0x71, 0xd7, 0x1f, - 0x7e, 0x20, 0xad, 0x73, 0xec, 0x35, 0xe3, 0x2e, 0xc1, 0x0c, 0x8e, 0x1e, 0x84, 0x11, 0x62, 0xe9, - 0xce, 0x6e, 0x8b, 0x1e, 0x2c, 0xdc, 0xc4, 0xc3, 0xbe, 0xd0, 0x45, 0xbf, 0x10, 0x87, 0x70, 0xf5, - 0x21, 0x88, 0xaa, 0x5c, 0x5d, 0x38, 0x2f, 0xff, 0xbd, 0x02, 0xe7, 0xca, 0x6d, 0xcd, 0x9c, 0x6f, - 0xd1, 0x8d, 0xaa, 0x99, 0x4b, 0x36, 0xf7, 0x01, 0xa0, 0x7a, 0xc8, 0xdb, 0x60, 0xd8, 0x17, 0x72, - 0xe2, 0x4f, 0x47, 0x7d, 0x46, 0x89, 0x83, 0x1a, 0x48, 0x83, 0x61, 0xd7, 0x17, 0xbb, 0x0b, 0x3d, - 0x88, 0xdd, 0x3e, 0x89, 0x40, 0xec, 0x0e, 0xd0, 0x72, 0xe3, 0x17, 0x5b, 0xa0, 0x1a, 0x71, 0xb6, - 0x0d, 0x9d, 0xcc, 0xeb, 0xba, 0xdd, 0xb6, 0x3c, 0x57, 0x3c, 0x05, 0x17, 0xc6, 0xaf, 0xb4, 0x1a, - 0x38, 0xa3, 0xa5, 0xfa, 0x25, 0x05, 0xfa, 0x17, 0xd7, 0x4a, 0x65, 0xf4, 0x6d, 0xd0, 0x1f, 0xb0, - 0x8c, 0x9c, 0xae, 0x3e, 0x14, 0x0f, 0xb7, 0xd3, 0xf1, 0xf5, 0x5e, 0xa1, 0x0c, 0x87, 0x61, 0x45, - 0x1b, 0x30, 0x48, 0xb6, 0x09, 0xed, 0x6a, 0xe1, 0x48, 0xf0, 0x33, 0x96, 0xb6, 0xc8, 0x30, 0x62, - 0x81, 0x59, 0xfd, 0x94, 0x02, 0x10, 0x56, 0x41, 0xdf, 0x99, 0x76, 0x3a, 0xdd, 0x3c, 0x42, 0xeb, - 0x73, 0xe7, 0x23, 0x4a, 0xfd, 0xb5, 0x01, 0x38, 0x4f, 0xbb, 0x23, 0xb6, 0xaa, 0x61, 0x5b, 0x37, - 0xc9, 0xee, 0x37, 0xdc, 0xe4, 0xbf, 0xe1, 0x26, 0x7f, 0x84, 0x6e, 0xf2, 0x2f, 0xc2, 0x79, 0xba, - 0xe5, 0x04, 0x98, 0x30, 0x9f, 0xa1, 0xaa, 0x43, 0x5a, 0x9a, 0x43, 0xea, 0x4c, 0x1f, 0x18, 0xe6, - 0x5e, 0x50, 0xf3, 0x59, 0x95, 0x70, 0x76, 0x7b, 0xf5, 0x29, 0x98, 0x0c, 0xf7, 0xae, 0xf8, 0xa8, - 0x1e, 0x8c, 0x2b, 0x99, 0x23, 0xbe, 0xc4, 0x94, 0x54, 0x0c, 0xd5, 0x7b, 0x0a, 0x4c, 0x2e, 0xee, - 0xb4, 0x0c, 0x87, 0xbd, 0xcf, 0xe7, 0xcf, 0x4c, 0xd0, 0x03, 0xe1, 0x6b, 0x14, 0x25, 0x7a, 0x8b, - 0x17, 0x7f, 0x91, 0x82, 0x36, 0x61, 0x82, 0xb0, 0xe6, 0x4c, 0x0b, 0xd4, 0xbc, 0x3c, 0xdb, 0x9b, - 0x07, 0xa5, 0x88, 0x60, 0xc1, 0x31, 0xac, 0xa8, 0x06, 0x13, 0xba, 0xa9, 0xb9, 0xae, 0xb1, 0x69, - 0xe8, 0xe1, 0x2b, 0xaa, 0x91, 0x85, 0x07, 0x99, 0xcc, 0x15, 0x81, 0xdc, 0xdb, 0x2b, 0x9e, 0x11, - 0xfd, 0x8c, 0x02, 0x70, 0x0c, 0x85, 0xfa, 0x99, 0x02, 0x8c, 0x2f, 0xee, 0xb4, 0x6c, 0xb7, 0xed, - 0x10, 0x56, 0xf5, 0x04, 0xec, 0x5a, 0x0f, 0xc0, 0xd0, 0x96, 0x66, 0xd5, 0x4d, 0xe2, 0x88, 0x63, - 0x37, 0x98, 0xdb, 0x1b, 0xbc, 0x18, 0xfb, 0x70, 0xf4, 0x2a, 0x80, 0xab, 0x6f, 0x91, 0x7a, 0x9b, - 0x31, 0xc7, 0xbe, 0xfc, 0xcc, 0x31, 0x32, 0xc6, 0x5a, 0x80, 0x52, 0x88, 0x34, 0xc1, 0x6f, 0x2c, - 0x91, 0x53, 0xff, 0x44, 0x81, 0xa9, 0x48, 0xbb, 0x13, 0x30, 0xd7, 0x6c, 0x46, 0xcd, 0x35, 0xf3, - 0x3d, 0x8f, 0x35, 0xc3, 0x4a, 0xf3, 0xf1, 0x02, 0x9c, 0xcb, 0x98, 0x93, 0x84, 0xbf, 0xaf, 0x72, - 0x42, 0xfe, 0xbe, 0x6d, 0x18, 0xf5, 0x6c, 0x53, 0x3c, 0xf6, 0xf3, 0x67, 0x20, 0xd7, 0x11, 0xbc, - 0x16, 0xa0, 0x09, 0xbd, 0x79, 0xc3, 0x32, 0x17, 0xcb, 0x74, 0xd4, 0xdf, 0x50, 0x60, 0x24, 0xb0, - 0x0a, 0xbf, 0xae, 0x2e, 0xd4, 0xbb, 0x8f, 0xa3, 0xa3, 0xfe, 0x6e, 0x01, 0xce, 0x06, 0xb8, 0x7d, - 0x36, 0x57, 0xf3, 0x28, 0xdf, 0x38, 0xd8, 0xb4, 0x74, 0x31, 0xf2, 0x12, 0x61, 0x38, 0xf9, 0x30, - 0xaf, 0xd5, 0x76, 0x5a, 0xb6, 0xeb, 0xcb, 0xc1, 0x5c, 0x61, 0xe0, 0x45, 0xd8, 0x87, 0xa1, 0x55, - 0x18, 0x70, 0x29, 0x3d, 0x71, 0xd6, 0x1d, 0x72, 0x36, 0x98, 0x28, 0xcf, 0xfa, 0x8b, 0x39, 0x1a, - 0xf4, 0xaa, 0xcc, 0xc3, 0x07, 0xf2, 0x1b, 0x2f, 0xe9, 0x48, 0xea, 0x81, 0x24, 0x1c, 0x5e, 0x7d, - 0x74, 0x32, 0x16, 0xaa, 0xcb, 0x30, 0x29, 0x9c, 0x5a, 0xf9, 0xb6, 0xb1, 0x74, 0x82, 0xde, 0x1d, - 0xd9, 0x19, 0xf7, 0xc7, 0x5c, 0x6a, 0xa6, 0xe3, 0xf5, 0xc3, 0x1d, 0xa3, 0xba, 0x30, 0x7c, 0x5d, - 0x74, 0x12, 0xcd, 0x42, 0xc1, 0xf0, 0xd7, 0x02, 0x04, 0x8e, 0x42, 0xa5, 0x8c, 0x0b, 0x46, 0x17, - 0x2f, 0x42, 0xe4, 0x63, 0xa9, 0xaf, 0xf3, 0xb1, 0xa4, 0x7e, 0xad, 0x00, 0xd3, 0x3e, 0x55, 0x7f, - 0x8c, 0x65, 0x71, 0x5b, 0x7e, 0x80, 0x52, 0x74, 0xb0, 0xa9, 0xf1, 0x16, 0xf4, 0x33, 0x06, 0x98, - 0xeb, 0x16, 0x3d, 0x40, 0x48, 0xbb, 0x83, 0x19, 0x22, 0xf4, 0x01, 0x18, 0x34, 0xa9, 0x86, 0xe1, - 0x3f, 0xd5, 0xc8, 0x65, 0x98, 0x4d, 0x1b, 0x2e, 0x57, 0x5c, 0x44, 0x08, 0xb3, 0xe0, 0x22, 0x94, - 0x17, 0x62, 0x41, 0x73, 0xf6, 0x31, 0x18, 0x95, 0xaa, 0x1d, 0x2a, 0x7e, 0xd9, 0xe7, 0x0a, 0x30, - 0x73, 0x83, 0x98, 0xcd, 0x54, 0xd7, 0x87, 0x22, 0x0c, 0xe8, 0x5b, 0x9a, 0xc3, 0x43, 0xe3, 0x8d, - 0xf1, 0x4d, 0x5e, 0xa2, 0x05, 0x98, 0x97, 0x53, 0x85, 0x23, 0xf2, 0x38, 0xed, 0x49, 0x69, 0x26, - 0xc3, 0x98, 0x89, 0xef, 0x0f, 0x82, 0x2a, 0x86, 0x03, 0x8f, 0x54, 0xa0, 0xc7, 0xcb, 0x7b, 0x6b, - 0xb7, 0x56, 0xd3, 0x9e, 0xa5, 0xa1, 0xbb, 0x30, 0x6e, 0xeb, 0x06, 0x26, 0x2d, 0xdb, 0x35, 0x3c, - 0xdb, 0xd9, 0x15, 0x8b, 0x96, 0xeb, 0x68, 0xb9, 0x55, 0xaa, 0x84, 0x88, 0xf8, 0xf5, 0x61, 0xa4, - 0x08, 0x47, 0x49, 0xa9, 0x5f, 0x54, 0x60, 0xf4, 0x86, 0xb1, 0x41, 0x1c, 0xee, 0xb7, 0xcb, 0x2c, - 0x24, 0x91, 0x20, 0x6f, 0xa3, 0x69, 0x01, 0xde, 0xd0, 0x0e, 0x8c, 0x88, 0x73, 0x38, 0x78, 0x75, - 0x78, 0x3d, 0x9f, 0xbf, 0x50, 0x40, 0x5a, 0x9c, 0x6f, 0xd2, 0x87, 0xef, 0x97, 0xb8, 0x38, 0x24, - 0xa6, 0xbe, 0x0a, 0xa7, 0x53, 0x1a, 0xd1, 0x85, 0x64, 0xae, 0xab, 0xe2, 0xa3, 0xf1, 0xb9, 0x15, - 0x5d, 0x48, 0x56, 0x8e, 0xce, 0x43, 0x1f, 0xb1, 0xea, 0xe2, 0x8b, 0x19, 0xda, 0xdf, 0x2b, 0xf6, - 0x2d, 0x5a, 0x75, 0x4c, 0xcb, 0x28, 0x13, 0x37, 0xed, 0x88, 0xc4, 0xc6, 0x98, 0xf8, 0xb2, 0x28, - 0xc3, 0x01, 0x94, 0x79, 0x78, 0xc5, 0x9d, 0x99, 0xa8, 0x66, 0x31, 0xb9, 0x19, 0xe3, 0x2d, 0xbd, - 0xf8, 0x50, 0xc5, 0xf9, 0xd4, 0xc2, 0x8c, 0x98, 0x90, 0x04, 0xc7, 0xc3, 0x09, 0xba, 0xea, 0x2f, - 0xf7, 0xc3, 0xa5, 0x1b, 0xb6, 0x63, 0xdc, 0xb5, 0x2d, 0x4f, 0x33, 0xab, 0x76, 0x3d, 0xf4, 0x6f, - 0x15, 0x47, 0xd6, 0xc7, 0x14, 0x38, 0xa7, 0xb7, 0xda, 0x5c, 0x33, 0xf1, 0x5d, 0x44, 0xab, 0xc4, - 0x31, 0xec, 0xbc, 0x0f, 0x35, 0x98, 0x3b, 0x6f, 0xa9, 0xba, 0x9e, 0x86, 0x12, 0x67, 0xd1, 0x62, - 0xef, 0x45, 0xea, 0xf6, 0x1d, 0x8b, 0x75, 0xae, 0xe6, 0xb1, 0xd9, 0xbc, 0x1b, 0x2e, 0x42, 0xce, - 0xf7, 0x22, 0xe5, 0x54, 0x8c, 0x38, 0x83, 0x12, 0xfa, 0x10, 0x9c, 0x31, 0x78, 0xe7, 0x30, 0xd1, - 0xea, 0x86, 0x45, 0x5c, 0x97, 0x3b, 0x9b, 0xf7, 0xf0, 0x20, 0xa2, 0x92, 0x86, 0x10, 0xa7, 0xd3, - 0x41, 0x2f, 0x01, 0xb8, 0xbb, 0x96, 0x2e, 0xe6, 0x3f, 0x9f, 0x23, 0x2a, 0x17, 0x91, 0x03, 0x2c, - 0x58, 0xc2, 0x48, 0x15, 0x2d, 0x2f, 0xd8, 0x94, 0x83, 0xcc, 0x99, 0x98, 0x29, 0x5a, 0xe1, 0x1e, - 0x0a, 0xe1, 0xea, 0xa7, 0x14, 0x98, 0xa8, 0x58, 0x55, 0x53, 0xd3, 0x09, 0xf7, 0xab, 0x74, 0xd1, - 0x35, 0x18, 0x71, 0x83, 0x1b, 0x19, 0xce, 0x11, 0xc2, 0xef, 0x33, 0xb8, 0x8b, 0x09, 0xeb, 0x30, - 0x2f, 0x6d, 0xc3, 0x12, 0x87, 0xdd, 0x92, 0xed, 0x70, 0x44, 0xe2, 0xbb, 0xe3, 0x5e, 0xda, 0x49, - 0x30, 0x4e, 0x6b, 0xa3, 0xfe, 0x9c, 0x02, 0xd3, 0xd1, 0xee, 0x08, 0x97, 0x86, 0x1f, 0x55, 0x60, - 0x3a, 0xf2, 0x98, 0x5a, 0x80, 0x7b, 0x89, 0xef, 0x54, 0x4d, 0xc1, 0xc7, 0x5d, 0x7c, 0xd3, 0x20, - 0x38, 0x95, 0xbe, 0xfa, 0xb3, 0x0a, 0x0c, 0x89, 0x58, 0x8f, 0xe8, 0xad, 0x31, 0xeb, 0x79, 0x70, - 0xb4, 0xc5, 0x2c, 0xe8, 0xbb, 0xcc, 0x3f, 0x43, 0x1c, 0x4d, 0xe2, 0x94, 0xc9, 0x65, 0x7e, 0x15, - 0x84, 0xc3, 0x73, 0x2e, 0xe2, 0xa7, 0xe1, 0x5f, 0xcd, 0x48, 0xc4, 0xd4, 0xd7, 0x14, 0x98, 0x4a, - 0xb4, 0xea, 0x42, 0x1c, 0x3d, 0x41, 0x8f, 0xd5, 0x2f, 0xf7, 0xd3, 0x2d, 0xe9, 0x51, 0x7e, 0x6f, - 0x72, 0xc3, 0xf6, 0x09, 0xe8, 0xbf, 0x0f, 0xc2, 0x88, 0xd1, 0x6c, 0xb6, 0x3d, 0x76, 0x89, 0x38, - 0x10, 0x5a, 0x8b, 0x2b, 0x7e, 0x21, 0x0e, 0xe1, 0xc8, 0x12, 0x92, 0x16, 0x3f, 0x05, 0x97, 0xf3, - 0xad, 0x9c, 0x3c, 0xc0, 0x39, 0x2a, 0x15, 0x71, 0x71, 0x28, 0x4d, 0x10, 0xfb, 0x1e, 0x05, 0xc0, - 0xf5, 0x1c, 0xc3, 0x6a, 0xd0, 0x42, 0x21, 0x8d, 0xe1, 0x23, 0x20, 0x5b, 0x0b, 0x90, 0x72, 0xe2, - 0x61, 0xfc, 0xc7, 0x00, 0x80, 0x25, 0xca, 0x68, 0x5e, 0x08, 0xa1, 0xfc, 0xc8, 0x7c, 0x7b, 0x4c, - 0xdc, 0xbe, 0x94, 0xe2, 0x09, 0xcb, 0x09, 0x85, 0x52, 0xea, 0xec, 0xa3, 0x30, 0x12, 0xd0, 0x3b, - 0x48, 0xa8, 0x1b, 0x93, 0x84, 0xba, 0xd9, 0x27, 0xe0, 0x54, 0xac, 0xbb, 0x87, 0x92, 0x09, 0xff, - 0x54, 0x01, 0x14, 0x1d, 0xfd, 0x09, 0x58, 0x0e, 0x1a, 0x51, 0xcb, 0xc1, 0x42, 0xef, 0x4b, 0x96, - 0x61, 0x3a, 0xf8, 0x69, 0x04, 0x2c, 0x14, 0x6e, 0x10, 0x1a, 0x5a, 0x9c, 0xfc, 0x54, 0x50, 0x09, - 0x5f, 0x80, 0x8b, 0x2f, 0xb7, 0x07, 0x41, 0xe5, 0x66, 0x0c, 0x57, 0x28, 0xa8, 0xc4, 0x21, 0x38, - 0x41, 0x17, 0x7d, 0x42, 0x81, 0x49, 0x2d, 0x1a, 0x0a, 0xd7, 0x9f, 0x99, 0x5c, 0x01, 0xba, 0x62, - 0x61, 0x75, 0xc3, 0xbe, 0xc4, 0x00, 0x2e, 0x4e, 0x90, 0x45, 0x8f, 0xc0, 0x98, 0xd6, 0x32, 0xe6, - 0xdb, 0x75, 0x83, 0x6a, 0x9e, 0x7e, 0xc4, 0x50, 0x66, 0x0d, 0x99, 0xaf, 0x56, 0x82, 0x72, 0x1c, - 0xa9, 0x15, 0xc4, 0x9c, 0x2d, 0x85, 0x8e, 0xc3, 0xbd, 0xc4, 0x9c, 0x15, 0x73, 0x18, 0xc6, 0x9c, - 0x15, 0x53, 0x27, 0x13, 0x41, 0x16, 0x80, 0x6d, 0xd4, 0x75, 0x41, 0x72, 0x30, 0xff, 0x1d, 0xc8, - 0xad, 0x4a, 0xb9, 0x24, 0x28, 0x32, 0xf1, 0x21, 0xfc, 0x8d, 0x25, 0x0a, 0xe8, 0xd3, 0x0a, 0x8c, - 0x0b, 0xde, 0x2d, 0x68, 0x0e, 0xb1, 0x25, 0x7a, 0x21, 0xef, 0x7e, 0x89, 0xed, 0xc9, 0x39, 0x2c, - 0x23, 0xe7, 0x7c, 0x27, 0x08, 0x20, 0x10, 0x81, 0xe1, 0x68, 0x3f, 0x98, 0x0c, 0xe0, 0x46, 0x2e, - 0xa1, 0x44, 0x07, 0x87, 0xf3, 0xcb, 0x00, 0xb5, 0x14, 0x7c, 0xe2, 0x91, 0x51, 0x0a, 0x04, 0xa7, - 0xd2, 0xa7, 0x72, 0xed, 0xa9, 0x3b, 0x9a, 0xa7, 0x6f, 0x95, 0x34, 0x7d, 0x8b, 0xdd, 0x41, 0xf2, - 0xe7, 0x94, 0x39, 0xf7, 0xf5, 0x73, 0x51, 0x54, 0x0b, 0xa7, 0xf7, 0xf7, 0x8a, 0xa7, 0x62, 0x85, - 0x38, 0x4e, 0x10, 0xd9, 0x30, 0xec, 0x88, 0x78, 0xf0, 0xe2, 0x05, 0x7c, 0xbe, 0x10, 0xe8, 0xf1, - 0xe0, 0xf2, 0x5c, 0x33, 0xf2, 0x7f, 0xe1, 0x80, 0x08, 0x6a, 0xc0, 0x25, 0xae, 0x1b, 0xce, 0x5b, - 0xb6, 0xb5, 0xdb, 0xb4, 0xdb, 0xee, 0x7c, 0xdb, 0xdb, 0x22, 0x96, 0xe7, 0x9b, 0xc2, 0x47, 0xd9, - 0x31, 0xca, 0x1e, 0xcd, 0x2d, 0x76, 0xaa, 0x88, 0x3b, 0xe3, 0x41, 0xcf, 0xc3, 0x30, 0xbb, 0xa7, - 0x5b, 0x5b, 0x5b, 0x66, 0x2f, 0x33, 0x0f, 0x2f, 0x2e, 0xb3, 0x21, 0x2c, 0x0a, 0x1c, 0x38, 0xc0, - 0x86, 0x6e, 0xc3, 0x90, 0xc9, 0x03, 0xfa, 0xb3, 0x17, 0x9a, 0x39, 0x99, 0x62, 0x3c, 0x39, 0x00, - 0x57, 0xa0, 0xc5, 0x0f, 0xec, 0x53, 0x40, 0x2d, 0xb8, 0x52, 0x27, 0x9b, 0x5a, 0xdb, 0xf4, 0x56, - 0x6d, 0x0f, 0xb3, 0x17, 0x6a, 0x81, 0xc5, 0xd3, 0x7f, 0x84, 0x3b, 0xc1, 0x62, 0xb0, 0xb1, 0xb7, - 0x7f, 0xe5, 0x03, 0xea, 0xe2, 0x03, 0xb1, 0xa1, 0x5d, 0xb8, 0x4f, 0xd4, 0x61, 0x4f, 0xe2, 0xf4, - 0x2d, 0x3a, 0xcb, 0x49, 0xa2, 0xa7, 0x18, 0xd1, 0x6f, 0xda, 0xdf, 0x2b, 0xde, 0x57, 0x3e, 0xb8, - 0x3a, 0xee, 0x06, 0x27, 0x7b, 0x65, 0x44, 0x62, 0x57, 0x40, 0x33, 0x93, 0xf9, 0xe7, 0x38, 0x7e, - 0x9d, 0xc4, 0xdd, 0xe1, 0xe2, 0xa5, 0x38, 0x41, 0x13, 0x7d, 0x41, 0x81, 0x19, 0xd7, 0x73, 0xda, - 0xba, 0xd7, 0x76, 0x48, 0x3d, 0xb6, 0x43, 0xa7, 0x58, 0x87, 0x72, 0x09, 0x70, 0xb5, 0x0c, 0x9c, - 0xec, 0x39, 0xf8, 0x4c, 0x16, 0x14, 0x67, 0xf6, 0x05, 0x7d, 0x5e, 0x81, 0x73, 0x51, 0x20, 0xd5, - 0xe9, 0x79, 0x3f, 0x51, 0xfe, 0x4b, 0x96, 0x5a, 0x3a, 0x4a, 0xae, 0xc1, 0x67, 0x00, 0x71, 0x56, - 0x47, 0xe2, 0x37, 0xe3, 0xa7, 0x4f, 0xf8, 0x66, 0x7c, 0xf6, 0x69, 0x40, 0xc9, 0xe3, 0xe3, 0x20, - 0x39, 0x70, 0x58, 0x96, 0x03, 0x3f, 0x3b, 0x00, 0x17, 0xe8, 0xa9, 0x14, 0x6a, 0x3f, 0x2b, 0x9a, - 0xa5, 0x35, 0x5e, 0x9f, 0x12, 0xd3, 0x17, 0x15, 0x38, 0xb7, 0x95, 0x6e, 0xda, 0x11, 0xfa, 0xd7, - 0x33, 0xb9, 0x4c, 0x70, 0x9d, 0xac, 0x45, 0x9c, 0x61, 0x77, 0xac, 0x82, 0xb3, 0x3a, 0x85, 0x9e, - 0x86, 0x49, 0xcb, 0xae, 0x93, 0x52, 0xa5, 0x8c, 0x57, 0x34, 0xf7, 0x76, 0xcd, 0x77, 0xd4, 0x19, - 0xe0, 0xdf, 0xeb, 0x6a, 0x0c, 0x86, 0x13, 0xb5, 0xd1, 0x36, 0xa0, 0x96, 0x5d, 0x5f, 0xdc, 0xe6, - 0x89, 0x2f, 0x7a, 0x73, 0x99, 0x65, 0xce, 0x02, 0xd5, 0x04, 0x36, 0x9c, 0x42, 0x81, 0xd9, 0xa6, - 0x68, 0x67, 0x56, 0x6c, 0xcb, 0xf0, 0x6c, 0x87, 0x05, 0x38, 0xe8, 0xc9, 0x44, 0xc3, 0x6c, 0x53, - 0xab, 0xa9, 0x18, 0x71, 0x06, 0x25, 0xf5, 0xbf, 0x2b, 0x70, 0x8a, 0x6e, 0x8b, 0xaa, 0x63, 0xef, - 0xec, 0xbe, 0x1e, 0x37, 0xe4, 0x03, 0xc2, 0x67, 0x91, 0xdb, 0x76, 0xce, 0x48, 0xfe, 0x8a, 0x23, - 0xac, 0xcf, 0xa1, 0x8b, 0xa2, 0x6c, 0x56, 0xee, 0xcb, 0x36, 0x2b, 0xab, 0x9f, 0x2e, 0x70, 0xcd, - 0xc5, 0x37, 0xeb, 0xbe, 0x2e, 0xbf, 0xc3, 0x47, 0x61, 0x9c, 0x96, 0xad, 0x68, 0x3b, 0xd5, 0xf2, - 0xb3, 0xb6, 0xe9, 0xbf, 0xc6, 0x66, 0xb6, 0xf6, 0x9b, 0x32, 0x00, 0x47, 0xeb, 0xa1, 0xc7, 0x61, - 0xa8, 0xc5, 0xe3, 0x84, 0x09, 0x9d, 0xf9, 0x0a, 0x77, 0xec, 0x63, 0x45, 0xf7, 0xf6, 0x8a, 0x53, - 0xe1, 0x15, 0xaf, 0x1f, 0xad, 0xcc, 0x6f, 0xa0, 0xfe, 0xf2, 0x59, 0x60, 0xc8, 0x4d, 0xe2, 0xbd, - 0x1e, 0xe7, 0xe4, 0x21, 0x18, 0xd5, 0x5b, 0xed, 0xd2, 0x52, 0xed, 0x99, 0xb6, 0xcd, 0x6c, 0x21, - 0x2c, 0x3f, 0x0c, 0xe5, 0xde, 0xa5, 0xea, 0xba, 0x5f, 0x8c, 0xe5, 0x3a, 0x94, 0x3b, 0xe8, 0xad, - 0xb6, 0xe0, 0xb7, 0x55, 0xf9, 0xbd, 0x0a, 0xe3, 0x0e, 0xa5, 0xea, 0x7a, 0x04, 0x86, 0x13, 0xb5, - 0xd1, 0x87, 0x60, 0x8c, 0x88, 0x0f, 0xf7, 0x86, 0xe6, 0xd4, 0x05, 0x5f, 0xa8, 0xe4, 0x1d, 0x7c, - 0x30, 0xb5, 0x3e, 0x37, 0xe0, 0x1a, 0xe0, 0xa2, 0x44, 0x02, 0x47, 0x08, 0xa2, 0x17, 0xe1, 0xbc, - 0xff, 0x9b, 0xae, 0xb2, 0x5d, 0x8f, 0x33, 0x8a, 0x01, 0xee, 0x36, 0xb3, 0x98, 0x55, 0x09, 0x67, - 0xb7, 0x47, 0x3f, 0xa3, 0xc0, 0xd9, 0x00, 0x6a, 0x58, 0x46, 0xb3, 0xdd, 0xc4, 0x44, 0x37, 0x35, - 0xa3, 0x29, 0xf4, 0xbe, 0xe7, 0x8e, 0x6c, 0xa0, 0x51, 0xf4, 0x9c, 0x59, 0xa5, 0xc3, 0x70, 0x46, - 0x97, 0xd0, 0x6b, 0x0a, 0x5c, 0xf1, 0x41, 0x55, 0x87, 0xb8, 0x6e, 0xdb, 0x21, 0x61, 0x2c, 0x00, - 0x31, 0x25, 0x43, 0xb9, 0x78, 0x27, 0x13, 0x80, 0x17, 0x0f, 0xc0, 0x8d, 0x0f, 0xa4, 0x2e, 0x6f, - 0x97, 0x9a, 0xbd, 0xe9, 0x09, 0x45, 0xf1, 0xb8, 0xb6, 0x0b, 0x25, 0x81, 0x23, 0x04, 0xd1, 0xcf, - 0x29, 0x70, 0x4e, 0x2e, 0x90, 0x77, 0x0b, 0xd7, 0x10, 0x9f, 0x3f, 0xb2, 0xce, 0xc4, 0xf0, 0x73, - 0x09, 0x2f, 0x03, 0x88, 0xb3, 0x7a, 0x45, 0xd9, 0x76, 0x93, 0x6d, 0x4c, 0xae, 0x45, 0x0e, 0x70, - 0xb6, 0xcd, 0xf7, 0xaa, 0x8b, 0x7d, 0x18, 0x7a, 0x04, 0xc6, 0x5a, 0x76, 0xbd, 0x6a, 0xd4, 0x5d, - 0x16, 0xc3, 0x8c, 0xe9, 0x7a, 0x7d, 0x7c, 0x3a, 0xaa, 0x76, 0xbd, 0x5a, 0x29, 0xf3, 0x72, 0x1c, - 0xa9, 0x85, 0xe6, 0x00, 0x36, 0x35, 0xc3, 0xac, 0xdd, 0xd1, 0x5a, 0xb7, 0xfc, 0xa0, 0x38, 0xcc, - 0x16, 0xb1, 0x14, 0x94, 0x62, 0xa9, 0x06, 0x5d, 0x3f, 0xca, 0x77, 0x30, 0xe1, 0x61, 0xa7, 0x99, - 0x7a, 0x74, 0x14, 0xeb, 0xe7, 0x23, 0xe4, 0x1d, 0xbe, 0x29, 0x91, 0xc0, 0x11, 0x82, 0xe8, 0x63, - 0x0a, 0x4c, 0xb8, 0xbb, 0xae, 0x47, 0x9a, 0x41, 0x1f, 0x4e, 0x1d, 0x75, 0x1f, 0x98, 0x4d, 0xbc, - 0x16, 0x21, 0x82, 0x63, 0x44, 0x59, 0x78, 0xa1, 0xa6, 0xd6, 0x20, 0xd7, 0x4b, 0x37, 0x8c, 0xc6, - 0x56, 0x10, 0xdd, 0xa5, 0x4a, 0x1c, 0x9d, 0x58, 0x1e, 0x53, 0xac, 0x06, 0x44, 0x78, 0xa1, 0xec, - 0x6a, 0xb8, 0x13, 0x0e, 0xf4, 0x12, 0xcc, 0x0a, 0xf0, 0xb2, 0x7d, 0x27, 0x41, 0x81, 0x47, 0xf1, - 0x61, 0x0e, 0x90, 0x95, 0xcc, 0x5a, 0xb8, 0x03, 0x06, 0x54, 0x81, 0xd3, 0x2e, 0x71, 0xd8, 0x9d, - 0x20, 0x8f, 0x08, 0x59, 0x6d, 0x9b, 0xa6, 0xcb, 0x54, 0x1b, 0xf1, 0xe4, 0xa7, 0x96, 0x04, 0xe3, - 0xb4, 0x36, 0xe8, 0x89, 0xe0, 0x59, 0xf0, 0x2e, 0x2d, 0x78, 0xa6, 0x5a, 0x63, 0x9a, 0xc8, 0x00, - 0xb7, 0x94, 0xe0, 0x28, 0x08, 0xc7, 0xeb, 0xd2, 0xd3, 0xdc, 0x2f, 0x5a, 0x68, 0x3b, 0xae, 0x37, - 0x33, 0xcd, 0x1a, 0xb3, 0xd3, 0x1c, 0xcb, 0x00, 0x1c, 0xad, 0x87, 0x1e, 0x87, 0x09, 0x97, 0xe8, - 0xba, 0xdd, 0x6c, 0x09, 0x3d, 0x79, 0xe6, 0x0c, 0xeb, 0x3d, 0x5f, 0xc1, 0x08, 0x04, 0xc7, 0x6a, - 0xa2, 0x5d, 0x38, 0x1d, 0x84, 0xd0, 0x5d, 0xb6, 0x1b, 0x2b, 0xda, 0x0e, 0x13, 0x8e, 0xcf, 0xe6, - 0x09, 0xf1, 0xc0, 0xa7, 0xab, 0x94, 0x44, 0x87, 0xd3, 0x68, 0xa0, 0x65, 0x98, 0x8e, 0x15, 0x2f, - 0x19, 0x26, 0x71, 0x67, 0xce, 0xb1, 0x61, 0x33, 0x63, 0x57, 0x29, 0x05, 0x8e, 0x53, 0x5b, 0xa1, - 0x5b, 0x70, 0xa6, 0xe5, 0xd8, 0x1e, 0xd1, 0xbd, 0x9b, 0x54, 0x20, 0x30, 0xc5, 0x00, 0xdd, 0x99, - 0x19, 0x36, 0x17, 0xec, 0x3e, 0xb4, 0x9a, 0x56, 0x01, 0xa7, 0xb7, 0x43, 0x9f, 0x55, 0xe0, 0xb2, - 0xeb, 0x39, 0x44, 0x6b, 0x1a, 0x56, 0xa3, 0x64, 0x5b, 0x16, 0x61, 0x8c, 0xa9, 0x52, 0x0f, 0x5f, - 0xcc, 0x9d, 0xcf, 0x75, 0x8a, 0xa8, 0xfb, 0x7b, 0xc5, 0xcb, 0xb5, 0x8e, 0x98, 0xf1, 0x01, 0x94, - 0xd1, 0xab, 0x00, 0x4d, 0xd2, 0xb4, 0x9d, 0x5d, 0xca, 0x91, 0x66, 0x66, 0xf3, 0xeb, 0xbb, 0x2b, - 0x01, 0x16, 0xfe, 0xf9, 0x47, 0x6e, 0x72, 0x43, 0x20, 0x96, 0xc8, 0xd1, 0xa9, 0xa6, 0xfc, 0x56, - 0x44, 0xb1, 0x92, 0x3e, 0x9a, 0x0b, 0x6c, 0xe5, 0xd8, 0x54, 0xaf, 0xa4, 0x55, 0xc0, 0xe9, 0xed, - 0x90, 0x0b, 0x53, 0xec, 0x0b, 0x15, 0x27, 0xf9, 0xf5, 0xd2, 0x7c, 0x83, 0xcc, 0x5c, 0xcc, 0x35, - 0xb9, 0x54, 0xf6, 0x9f, 0xaa, 0xc4, 0x91, 0xe1, 0x24, 0xfe, 0x90, 0xa8, 0xb6, 0x13, 0x12, 0xbd, - 0xd4, 0x2b, 0x51, 0x09, 0x19, 0x4e, 0xe2, 0x57, 0xf7, 0x0a, 0x70, 0x26, 0xf5, 0x94, 0xa4, 0xcc, - 0x83, 0x4f, 0xf1, 0xbc, 0x9f, 0xff, 0x4b, 0x5c, 0x7b, 0x32, 0xe6, 0xb1, 0x12, 0x05, 0xe1, 0x78, - 0x5d, 0x2a, 0xc3, 0x32, 0x6a, 0x4b, 0xb5, 0xb0, 0x7d, 0x21, 0x94, 0x61, 0x2b, 0x31, 0x18, 0x4e, - 0xd4, 0x46, 0x25, 0x31, 0x1f, 0x4b, 0xb5, 0x0a, 0x55, 0x03, 0xdd, 0x25, 0x87, 0xf8, 0xda, 0x41, - 0x38, 0x3e, 0x19, 0x88, 0x93, 0xf5, 0xe9, 0x28, 0xe8, 0x0f, 0xb9, 0x17, 0xfd, 0xe1, 0x28, 0x56, - 0xa3, 0x20, 0x1c, 0xaf, 0xeb, 0xeb, 0xe9, 0x91, 0x2e, 0x0c, 0x84, 0xa3, 0x58, 0x8d, 0xc1, 0x70, - 0xa2, 0xb6, 0xfa, 0x67, 0xfd, 0x70, 0x5f, 0x17, 0x92, 0x25, 0x6a, 0xa6, 0x4f, 0xf7, 0xe1, 0x79, - 0x5e, 0x77, 0xcb, 0xd3, 0xca, 0x58, 0x9e, 0xc3, 0xd3, 0xeb, 0x76, 0x39, 0xdd, 0xac, 0xe5, 0x3c, - 0x3c, 0xc9, 0xee, 0x97, 0xbf, 0x99, 0xbe, 0xfc, 0x39, 0x67, 0xf5, 0xc0, 0xed, 0xd2, 0xca, 0xd8, - 0x2e, 0x39, 0x67, 0xb5, 0x8b, 0xed, 0xf5, 0xe7, 0xfd, 0x70, 0x7f, 0x37, 0x52, 0x6e, 0xce, 0xfd, - 0x95, 0xc2, 0x5b, 0x8e, 0x75, 0x7f, 0x65, 0xbd, 0xe7, 0x3e, 0xc6, 0xfd, 0xd5, 0x91, 0x7d, 0x1e, - 0xcf, 0xfe, 0xca, 0x9a, 0xd5, 0xe3, 0xda, 0x5f, 0x59, 0xb3, 0xda, 0xc5, 0xfe, 0xfa, 0x9b, 0xf8, - 0xf9, 0x10, 0x88, 0xda, 0x15, 0xe8, 0xd3, 0x5b, 0xed, 0x9c, 0x4c, 0x8a, 0x79, 0x19, 0x96, 0xaa, - 0xeb, 0x98, 0xe2, 0x40, 0x18, 0x06, 0xf9, 0xfe, 0xc9, 0xc9, 0x82, 0x98, 0xe7, 0x28, 0xdf, 0x92, - 0x58, 0x60, 0xa2, 0x53, 0x45, 0x5a, 0x5b, 0xa4, 0x49, 0x1c, 0xcd, 0xac, 0x79, 0xb6, 0xa3, 0x35, - 0xf2, 0x72, 0x1b, 0x7e, 0x83, 0x12, 0xc3, 0x85, 0x13, 0xd8, 0xe9, 0x84, 0xb4, 0x8c, 0x7a, 0x4e, - 0xfe, 0xc2, 0x26, 0xa4, 0x5a, 0x29, 0x63, 0x8a, 0x43, 0xfd, 0xb5, 0x61, 0x90, 0x82, 0xd3, 0xa3, - 0x4f, 0x2a, 0x30, 0xa5, 0xc7, 0x63, 0x72, 0xf6, 0xe2, 0x0f, 0x95, 0x08, 0xf0, 0xc9, 0xb7, 0x7c, - 0xa2, 0x18, 0x27, 0xc9, 0xa2, 0x0f, 0x2b, 0xdc, 0xc8, 0x17, 0xdc, 0xe6, 0x89, 0x69, 0xbd, 0x7e, - 0x44, 0xf7, 0xde, 0xa1, 0xb5, 0x30, 0xbc, 0x62, 0x8d, 0x12, 0x44, 0xaf, 0x29, 0x70, 0xe6, 0x76, - 0xda, 0xdd, 0x84, 0x98, 0xfc, 0x5b, 0x79, 0xbb, 0x92, 0x71, 0xd9, 0xc1, 0x25, 0xc8, 0xd4, 0x0a, - 0x38, 0xbd, 0x23, 0xc1, 0x2c, 0x05, 0xe6, 0x5a, 0xf1, 0x9d, 0xe6, 0x9e, 0xa5, 0x98, 0xdd, 0x37, - 0x9c, 0xa5, 0x00, 0x80, 0xa3, 0x04, 0x51, 0x0b, 0x46, 0x6e, 0xfb, 0x36, 0x72, 0x61, 0x17, 0x2b, - 0xe5, 0xa5, 0x2e, 0x19, 0xda, 0xb9, 0xbf, 0x57, 0x50, 0x88, 0x43, 0x22, 0x68, 0x0b, 0x86, 0x6e, - 0x73, 0x5e, 0x21, 0xec, 0x59, 0xf3, 0x3d, 0x6b, 0xff, 0xdc, 0xac, 0x22, 0x8a, 0xb0, 0x8f, 0x5e, - 0x7e, 0x4b, 0x30, 0x7c, 0xc0, 0x13, 0xb7, 0xcf, 0x2a, 0x70, 0x66, 0x9b, 0x38, 0x9e, 0xa1, 0xc7, - 0x6f, 0x86, 0x46, 0xf2, 0x5b, 0x28, 0x9e, 0x4d, 0x43, 0xc8, 0xb7, 0x49, 0x2a, 0x08, 0xa7, 0x77, - 0x01, 0x3d, 0x0b, 0xfd, 0xc4, 0xd3, 0xeb, 0x22, 0x80, 0xf3, 0xbb, 0xf3, 0x3e, 0xd9, 0xe5, 0xef, - 0x5e, 0xe8, 0x7f, 0x98, 0xe1, 0x53, 0xff, 0x4a, 0x81, 0x84, 0x6d, 0x1a, 0xfd, 0x80, 0x02, 0x63, - 0x9b, 0x44, 0xf3, 0xda, 0x0e, 0xb9, 0x2e, 0x7c, 0x42, 0xfb, 0xae, 0x8e, 0x3e, 0xfc, 0xec, 0x51, - 0x98, 0xc4, 0xe7, 0x96, 0x24, 0xc4, 0xdc, 0x59, 0x25, 0x48, 0x38, 0x21, 0x83, 0x70, 0xa4, 0x07, - 0xb3, 0x4f, 0xc1, 0x54, 0xa2, 0xe1, 0xa1, 0xae, 0x29, 0x7f, 0x55, 0x81, 0xb4, 0xdc, 0xe6, 0xe8, - 0x25, 0x18, 0xd0, 0xea, 0xf5, 0x20, 0xc5, 0xe5, 0x63, 0xf9, 0xfc, 0xa6, 0xea, 0x72, 0xa4, 0x20, - 0xf6, 0x13, 0x73, 0xb4, 0x68, 0x09, 0x90, 0x16, 0xb9, 0x97, 0x5e, 0x09, 0x43, 0x54, 0xb0, 0xeb, - 0xb4, 0xf9, 0x04, 0x14, 0xa7, 0xb4, 0x50, 0x3f, 0xae, 0x00, 0x4a, 0xa6, 0x28, 0x41, 0x0e, 0x0c, - 0x8b, 0xfd, 0xeb, 0xaf, 0x52, 0x39, 0xe7, 0x6b, 0xba, 0xc8, 0xd3, 0xd0, 0xd0, 0x09, 0x4f, 0x14, - 0xb8, 0x38, 0xa0, 0xa3, 0xfe, 0x9d, 0x02, 0x61, 0x1a, 0x3c, 0xf4, 0x4e, 0x18, 0xad, 0x13, 0x57, - 0x77, 0x8c, 0x96, 0x17, 0x3e, 0x24, 0x0d, 0x1e, 0xa4, 0x95, 0x43, 0x10, 0x96, 0xeb, 0x21, 0x15, - 0x06, 0x3d, 0xcd, 0xbd, 0x5d, 0x29, 0x0b, 0x65, 0x8f, 0x1d, 0xcd, 0x6b, 0xac, 0x04, 0x0b, 0x48, - 0x18, 0x3a, 0xb6, 0xaf, 0x8b, 0xd0, 0xb1, 0x68, 0xf3, 0x08, 0xe2, 0xe4, 0xa2, 0x83, 0x63, 0xe4, - 0xaa, 0x3f, 0x55, 0x80, 0x53, 0xb4, 0xca, 0x8a, 0x66, 0x58, 0x1e, 0xb1, 0xd8, 0xb3, 0xa9, 0x9c, - 0x93, 0xd0, 0x80, 0x71, 0x2f, 0xf2, 0x68, 0xf9, 0xf0, 0x8f, 0x6a, 0x03, 0x4f, 0xaf, 0xe8, 0x53, - 0xe5, 0x28, 0x5e, 0xf4, 0x98, 0xff, 0x6e, 0x8d, 0xab, 0xc5, 0xf7, 0xf9, 0x5b, 0x95, 0x3d, 0x46, - 0xbb, 0x27, 0x5e, 0x80, 0x07, 0xb9, 0x13, 0x23, 0x4f, 0xd4, 0x1e, 0x85, 0x71, 0xf1, 0x42, 0x82, - 0xc7, 0x00, 0x16, 0x6a, 0x31, 0x3b, 0x56, 0x96, 0x64, 0x00, 0x8e, 0xd6, 0x53, 0xff, 0xb0, 0x00, - 0xd1, 0x0c, 0x8d, 0x79, 0x67, 0x29, 0x19, 0x00, 0xb9, 0x70, 0x6c, 0x01, 0x90, 0x79, 0x8e, 0x72, - 0xe6, 0xce, 0x2d, 0xee, 0xd9, 0xe5, 0xfc, 0xc4, 0xac, 0x1c, 0x07, 0x35, 0xc2, 0x69, 0xed, 0x3f, - 0xf4, 0xb4, 0xbe, 0x53, 0x78, 0xfe, 0x0e, 0x44, 0xc2, 0x50, 0xfb, 0x9e, 0xbf, 0x53, 0x91, 0x86, - 0xd2, 0x2b, 0xbb, 0x79, 0x10, 0x39, 0x4e, 0xe8, 0xba, 0x88, 0x10, 0xd9, 0xee, 0x9a, 0xed, 0x69, - 0x26, 0x9b, 0x4e, 0x61, 0x74, 0x5d, 0x91, 0x01, 0x38, 0x5a, 0x4f, 0x5d, 0x85, 0x37, 0x2f, 0xdb, - 0x5a, 0x7d, 0x41, 0x33, 0xe9, 0xd6, 0x75, 0x84, 0x5b, 0x9e, 0xcb, 0x4e, 0xe6, 0xaa, 0x63, 0x7b, - 0xb6, 0x6e, 0x9b, 0xf4, 0xdc, 0xd4, 0x82, 0x58, 0xc1, 0xec, 0x22, 0xd2, 0x3f, 0x37, 0x45, 0xac, - 0x5d, 0xec, 0xc3, 0xd5, 0xdf, 0x56, 0x60, 0x48, 0xe4, 0x3d, 0xe9, 0xe2, 0x61, 0xe9, 0x26, 0x0c, - 0x30, 0xed, 0xa8, 0x17, 0xa9, 0xb4, 0xb6, 0x65, 0xdb, 0x5e, 0x24, 0x61, 0x12, 0x4f, 0x39, 0xc6, - 0x92, 0x44, 0x72, 0xf4, 0xcc, 0x1f, 0xd5, 0xd1, 0xb7, 0x0c, 0x8f, 0x30, 0xb7, 0x1b, 0xb1, 0xf1, - 0xb9, 0x3f, 0xaa, 0x54, 0x8e, 0x23, 0xb5, 0xd4, 0xff, 0x36, 0x00, 0x57, 0xfc, 0x1c, 0x2e, 0x71, - 0x51, 0x2d, 0xe0, 0xb9, 0xbb, 0x41, 0x52, 0x19, 0x16, 0x14, 0xdf, 0xb7, 0xa9, 0xe6, 0xd3, 0x92, - 0xcf, 0x49, 0x69, 0x63, 0x64, 0x74, 0x38, 0x8d, 0x06, 0x8f, 0xbe, 0xce, 0x8a, 0x6f, 0x10, 0xcd, - 0xf4, 0xb6, 0x7c, 0xda, 0x85, 0x5e, 0xa2, 0xaf, 0x27, 0xf1, 0xe1, 0x54, 0x2a, 0xcc, 0xa5, 0xc3, - 0x4f, 0x66, 0xe3, 0x10, 0x4d, 0xf6, 0x27, 0xe9, 0xe1, 0xb9, 0xd1, 0x4a, 0x2a, 0x46, 0x9c, 0x41, - 0x89, 0x99, 0x1b, 0xb5, 0x1d, 0x66, 0xbd, 0xc0, 0xc4, 0x73, 0x0c, 0x96, 0xf8, 0x2a, 0xb8, 0xab, - 0x58, 0x89, 0x82, 0x70, 0xbc, 0x2e, 0x7a, 0x1c, 0x26, 0x98, 0x8b, 0x4c, 0x18, 0xce, 0x73, 0x20, - 0x0c, 0xea, 0xb4, 0x1a, 0x81, 0xe0, 0x58, 0x4d, 0xf4, 0x5d, 0x0a, 0x4c, 0x1b, 0xf2, 0x63, 0x1a, - 0x7f, 0xf4, 0xf9, 0x32, 0xa3, 0xb0, 0xeb, 0x25, 0x7f, 0x1b, 0xa7, 0xa0, 0xc5, 0xa9, 0xc4, 0xd0, - 0x32, 0x4c, 0x8b, 0x17, 0xd1, 0xd1, 0x3d, 0xc0, 0x63, 0xbd, 0xb1, 0x35, 0x2d, 0xa7, 0xc0, 0x71, - 0x6a, 0x2b, 0xf5, 0x23, 0x05, 0x18, 0x3b, 0x64, 0x12, 0xd3, 0xb6, 0x24, 0x73, 0xf4, 0xf0, 0x6e, - 0x51, 0xa6, 0xda, 0x85, 0xd8, 0x81, 0x9e, 0x87, 0x89, 0x36, 0x9b, 0x08, 0x3f, 0x12, 0x9a, 0xf8, - 0xa6, 0xbf, 0x85, 0xae, 0xdc, 0x7a, 0x04, 0x72, 0x6f, 0xaf, 0x38, 0x2b, 0xa3, 0x8f, 0x42, 0x71, - 0x0c, 0x8f, 0xfa, 0xeb, 0x05, 0x40, 0x72, 0xf5, 0x25, 0x53, 0xdb, 0xb6, 0x1d, 0xf4, 0x79, 0x05, - 0xc6, 0x74, 0x39, 0x6d, 0x14, 0x17, 0xb0, 0x9e, 0xef, 0x75, 0xb0, 0x1c, 0xfd, 0x9c, 0x9c, 0x0f, - 0x8a, 0x0b, 0xc2, 0xfe, 0x73, 0xeb, 0x31, 0x19, 0x74, 0x2f, 0xf6, 0x1b, 0x47, 0xfa, 0x34, 0xfb, - 0x31, 0x05, 0xa6, 0x12, 0x98, 0x52, 0x24, 0xe3, 0x17, 0xa2, 0x31, 0xd0, 0x8f, 0x24, 0x41, 0xa4, - 0x2c, 0x5f, 0xff, 0xf8, 0x00, 0x9c, 0x4e, 0x59, 0x51, 0xe6, 0x62, 0x43, 0x62, 0xd2, 0x65, 0x2f, - 0x2e, 0x36, 0x09, 0x49, 0x35, 0x70, 0xb1, 0x89, 0x43, 0x70, 0x82, 0x2e, 0x7a, 0x16, 0xfa, 0x74, - 0xc7, 0x10, 0x9b, 0xf6, 0xd1, 0x5c, 0x53, 0x80, 0x2b, 0x0b, 0xa3, 0x82, 0x62, 0x5f, 0x09, 0x57, - 0x30, 0x45, 0x48, 0xcf, 0x62, 0xf9, 0x18, 0xf1, 0x05, 0x56, 0x76, 0x16, 0xcb, 0xa7, 0x8d, 0x8b, - 0xa3, 0xf5, 0xd0, 0xf3, 0x30, 0x23, 0x34, 0x55, 0x3f, 0x92, 0x89, 0x6d, 0xb9, 0x1e, 0xe5, 0xf8, - 0x9e, 0x90, 0x29, 0x98, 0xf7, 0xeb, 0xcd, 0x8c, 0x3a, 0x38, 0xb3, 0x35, 0xfa, 0x4e, 0x98, 0x88, - 0x70, 0x0f, 0x3f, 0x2a, 0x59, 0xce, 0x27, 0x33, 0x32, 0x26, 0xce, 0x2b, 0xa3, 0x65, 0x38, 0x46, - 0x0d, 0xfd, 0xbf, 0x0a, 0x4c, 0x05, 0x1b, 0x75, 0x97, 0x6f, 0x79, 0x3f, 0x83, 0xdf, 0xd2, 0xd1, - 0x7c, 0x41, 0x52, 0x7a, 0xd5, 0x38, 0x21, 0x9c, 0xa4, 0xad, 0xfe, 0xf0, 0x20, 0xc8, 0x19, 0x9d, - 0xd1, 0x4a, 0x2f, 0x76, 0xc9, 0x70, 0x0f, 0xf8, 0xb6, 0xc9, 0x15, 0xe8, 0x6b, 0xb4, 0xda, 0x39, - 0x0d, 0x93, 0x01, 0xba, 0xeb, 0x14, 0x5d, 0xa3, 0xd5, 0x46, 0xcf, 0x06, 0xa6, 0xce, 0x7c, 0xc6, - 0xc8, 0xe0, 0xa9, 0x64, 0xcc, 0xdc, 0xe9, 0xb3, 0xf7, 0xfe, 0x4c, 0xf6, 0xde, 0x84, 0x21, 0x57, - 0xd8, 0x41, 0x07, 0xf2, 0x87, 0x91, 0x94, 0x66, 0x5a, 0xd8, 0x3d, 0xb9, 0x85, 0xc6, 0x37, 0x8b, - 0xfa, 0x34, 0xa8, 0x22, 0xd8, 0x66, 0xe7, 0x92, 0x08, 0x92, 0xc4, 0x14, 0xc1, 0x75, 0x56, 0x82, - 0x05, 0x24, 0x21, 0xcc, 0x0d, 0x75, 0x23, 0xcc, 0xa1, 0xcf, 0xc5, 0xf9, 0xf7, 0x30, 0xdb, 0x7d, - 0xcf, 0xf4, 0x38, 0x9c, 0xff, 0xcb, 0x18, 0xf7, 0xff, 0x13, 0x1e, 0x7e, 0xd2, 0x62, 0xa1, 0xfb, - 0x60, 0x80, 0x45, 0x51, 0x12, 0x72, 0x40, 0x60, 0xdc, 0x60, 0x71, 0x74, 0x30, 0x87, 0xa1, 0x9a, - 0x08, 0x1d, 0x98, 0x6f, 0xd3, 0x33, 0xdf, 0x46, 0x41, 0x4f, 0x8a, 0x33, 0x78, 0x25, 0xf2, 0x26, - 0x32, 0x4d, 0x87, 0x58, 0x87, 0xa1, 0xa6, 0x61, 0x31, 0x77, 0x8f, 0x7c, 0x46, 0x74, 0xee, 0x82, - 0xc5, 0x51, 0x60, 0x1f, 0x97, 0xfa, 0xe7, 0x05, 0xca, 0x20, 0x42, 0xa5, 0x7e, 0x97, 0x07, 0xc2, - 0x15, 0xaf, 0xaf, 0x95, 0xfc, 0x46, 0x40, 0x09, 0xe9, 0x7c, 0x80, 0x90, 0x3b, 0x2a, 0x84, 0xbf, - 0xb1, 0x44, 0x8c, 0x92, 0xf6, 0x8c, 0x26, 0x79, 0xce, 0xb0, 0xea, 0xf6, 0x1d, 0x31, 0xbd, 0xbd, - 0x92, 0x5e, 0x0b, 0x10, 0x72, 0xd2, 0xe1, 0x6f, 0x2c, 0x11, 0xa3, 0x47, 0x12, 0x4b, 0xc6, 0x61, - 0xb1, 0x4c, 0xbc, 0xa2, 0x6f, 0x3c, 0xcf, 0xb6, 0xf0, 0x3b, 0x66, 0x47, 0x52, 0x29, 0xa3, 0x0e, - 0xce, 0x6c, 0xad, 0xfe, 0x8c, 0x02, 0x67, 0x52, 0xa7, 0x02, 0x5d, 0x87, 0xa9, 0xd0, 0x1d, 0x56, - 0x16, 0x12, 0x86, 0x43, 0x1e, 0x7f, 0x33, 0x5e, 0x01, 0x27, 0xdb, 0xf0, 0xfc, 0x66, 0x09, 0x21, - 0x44, 0xf8, 0xd2, 0xca, 0xaa, 0x96, 0x0c, 0xc6, 0x69, 0x6d, 0xd4, 0x17, 0x23, 0x9d, 0x0d, 0x27, - 0x8b, 0x7e, 0x19, 0x1b, 0xa4, 0x11, 0xbc, 0x49, 0x0f, 0xbe, 0x8c, 0x05, 0x5a, 0x88, 0x39, 0x0c, - 0x5d, 0x92, 0x43, 0x65, 0x04, 0xdc, 0xdd, 0x0f, 0x97, 0xa1, 0xfe, 0xaa, 0x02, 0xe7, 0x56, 0x34, - 0xab, 0xad, 0x99, 0xfc, 0xf1, 0x7b, 0xd5, 0xb6, 0x4d, 0x31, 0x4d, 0x9d, 0xf2, 0xad, 0x2b, 0xaf, - 0x97, 0x7c, 0xeb, 0xef, 0x87, 0x73, 0x19, 0xfe, 0x37, 0xa8, 0x0c, 0x63, 0xee, 0x1d, 0xad, 0xb5, - 0x40, 0xb6, 0xb4, 0x6d, 0x43, 0x04, 0xd6, 0xe2, 0x6e, 0xda, 0x63, 0x35, 0xa9, 0xfc, 0x5e, 0xec, - 0x37, 0x8e, 0xb4, 0x52, 0x3d, 0x00, 0xe1, 0xce, 0x6f, 0x58, 0x0d, 0xb4, 0x09, 0xc3, 0x9a, 0x49, - 0x1c, 0x2f, 0x0c, 0x1e, 0xf9, 0xad, 0xb9, 0xec, 0xb4, 0x02, 0x07, 0x7f, 0xbe, 0xe6, 0xff, 0xc2, - 0x01, 0x6e, 0xf5, 0x9f, 0x28, 0x70, 0x36, 0x3d, 0x94, 0x52, 0x17, 0x6a, 0x51, 0x13, 0x46, 0x9d, - 0xb0, 0x99, 0xf8, 0x68, 0xdf, 0x25, 0x67, 0xa8, 0x90, 0x1e, 0xde, 0x50, 0x45, 0xb0, 0xe4, 0xd8, - 0xae, 0xbf, 0x73, 0xe3, 0x49, 0x2b, 0x02, 0xab, 0x98, 0xd4, 0x13, 0x2c, 0xe3, 0x67, 0x09, 0x64, - 0x28, 0x75, 0xb7, 0xa5, 0xe9, 0xa4, 0x7e, 0xc2, 0x39, 0xd5, 0x8f, 0x20, 0x6b, 0x43, 0x7a, 0xdf, - 0x8f, 0x37, 0x81, 0x4c, 0x06, 0xcd, 0x83, 0x13, 0xc8, 0xa4, 0x37, 0x7c, 0x83, 0x64, 0x36, 0x48, - 0xef, 0x7c, 0xc6, 0xc3, 0xf7, 0xff, 0x32, 0x98, 0x35, 0xda, 0x43, 0x26, 0x66, 0xdf, 0x3e, 0xc6, - 0xc4, 0xec, 0x13, 0xdf, 0x48, 0xca, 0x9e, 0x92, 0x94, 0x3d, 0x96, 0x28, 0x7c, 0xf0, 0x84, 0x12, - 0x85, 0xbf, 0x02, 0x83, 0x2d, 0xcd, 0x21, 0x96, 0x7f, 0x97, 0x5b, 0xc9, 0xe7, 0x68, 0x10, 0x6e, - 0xb4, 0x90, 0x0b, 0x06, 0x9f, 0x64, 0x95, 0x11, 0xc0, 0x82, 0x50, 0x4a, 0xf0, 0x94, 0xe1, 0xe3, - 0x8a, 0x4e, 0x18, 0xa6, 0x28, 0x1f, 0x39, 0x8e, 0x14, 0xe5, 0xea, 0xdf, 0x2a, 0x70, 0xb1, 0x13, - 0x5b, 0x62, 0x06, 0x14, 0x3d, 0xf6, 0x19, 0xf6, 0x62, 0x40, 0x49, 0x70, 0xdb, 0xc0, 0x80, 0x12, - 0x87, 0xe0, 0x04, 0x5d, 0xf4, 0x5e, 0x40, 0xf6, 0x06, 0xf7, 0xeb, 0xb9, 0x4e, 0x69, 0xf0, 0xd7, - 0xb4, 0x05, 0xf6, 0x56, 0x21, 0xc8, 0x6b, 0x79, 0x2b, 0x51, 0x03, 0xa7, 0xb4, 0x52, 0x7f, 0xb9, - 0x00, 0xb0, 0x4a, 0xbc, 0x3b, 0xb6, 0x73, 0x9b, 0x9e, 0xf1, 0x17, 0x23, 0x57, 0x07, 0xc3, 0x5f, - 0xbf, 0x78, 0x94, 0x17, 0xa1, 0xbf, 0x65, 0xd7, 0x5d, 0xa1, 0x7f, 0xb0, 0x8e, 0xb0, 0xa7, 0x1a, - 0xac, 0x14, 0x15, 0x61, 0x80, 0x39, 0x3d, 0x09, 0x05, 0x9a, 0x5d, 0x3c, 0xb0, 0xcb, 0x15, 0xcc, - 0xcb, 0x29, 0x87, 0x14, 0x31, 0x0d, 0x5c, 0x71, 0xb9, 0x33, 0xc6, 0x63, 0x88, 0xf3, 0x32, 0x1c, - 0x40, 0xd1, 0xe3, 0x00, 0x46, 0x6b, 0x49, 0x6b, 0x1a, 0xa6, 0x21, 0x3e, 0xd7, 0x11, 0x66, 0x11, - 0x87, 0x4a, 0xd5, 0x2f, 0xbd, 0xb7, 0x57, 0x1c, 0x16, 0xbf, 0x76, 0xb1, 0x54, 0x5b, 0xfd, 0xa2, - 0x02, 0x93, 0xe1, 0xe4, 0x89, 0xad, 0xe2, 0xf7, 0x9c, 0x07, 0x03, 0xce, 0xec, 0x39, 0x0f, 0xdb, - 0xdf, 0xb9, 0xe7, 0xdc, 0x80, 0x95, 0xd5, 0xf3, 0x87, 0x60, 0x94, 0xf0, 0x90, 0x47, 0x95, 0x32, - 0xe6, 0x3c, 0x6e, 0x84, 0xab, 0x73, 0x8b, 0x61, 0x31, 0x96, 0xeb, 0xa8, 0x7f, 0xdf, 0x07, 0x63, - 0xab, 0x0d, 0xc3, 0xda, 0xf1, 0x63, 0x3b, 0x05, 0x17, 0xef, 0xca, 0xf1, 0x5c, 0xbc, 0x3f, 0x0f, - 0x33, 0xa6, 0x7c, 0xcd, 0xc5, 0x05, 0x27, 0xcd, 0x6a, 0x04, 0x33, 0xc0, 0xf4, 0x98, 0xe5, 0x8c, - 0x3a, 0x38, 0xb3, 0x35, 0xf2, 0x60, 0x50, 0xf7, 0xf3, 0x2b, 0xe6, 0x8e, 0x57, 0x24, 0xcf, 0xc5, - 0x9c, 0x1c, 0xba, 0x23, 0xe0, 0x79, 0x62, 0x7b, 0x0a, 0x5a, 0xe8, 0xa3, 0x0a, 0x9c, 0x21, 0x3b, - 0x3c, 0x74, 0xcd, 0x9a, 0xa3, 0x6d, 0x6e, 0x1a, 0xba, 0x78, 0xf1, 0xc7, 0x77, 0xe2, 0xf2, 0xfe, - 0x5e, 0xf1, 0xcc, 0x62, 0x5a, 0x85, 0x7b, 0x7b, 0xc5, 0x6b, 0xa9, 0x91, 0x84, 0xd8, 0x6a, 0xa6, - 0x36, 0xc1, 0xe9, 0xa4, 0x66, 0x1f, 0x83, 0xd1, 0x43, 0xbc, 0x13, 0x8f, 0xc4, 0x0b, 0xfa, 0x95, - 0x02, 0x8c, 0xd1, 0xed, 0xb6, 0x6c, 0xeb, 0x9a, 0x59, 0x5e, 0xad, 0xa1, 0x07, 0xe2, 0x61, 0x12, - 0x83, 0x2b, 0xc6, 0x44, 0xa8, 0xc4, 0x65, 0x98, 0xde, 0xb4, 0x1d, 0x9d, 0xac, 0x95, 0xaa, 0x6b, - 0xb6, 0x70, 0x3e, 0x2b, 0xaf, 0xd6, 0x84, 0x5e, 0xc7, 0xae, 0x3c, 0x96, 0x52, 0xe0, 0x38, 0xb5, - 0x15, 0xba, 0x05, 0x67, 0xc2, 0xf2, 0xf5, 0x16, 0x7f, 0xb0, 0x40, 0xd1, 0xf5, 0x85, 0x0f, 0x2e, - 0x96, 0xd2, 0x2a, 0xe0, 0xf4, 0x76, 0x48, 0x83, 0x0b, 0xe2, 0x6e, 0x65, 0xc9, 0x76, 0xee, 0x68, - 0x4e, 0x3d, 0x8a, 0xb6, 0x3f, 0xcc, 0x55, 0x5e, 0xce, 0xae, 0x86, 0x3b, 0xe1, 0x50, 0xff, 0x4e, - 0x81, 0x68, 0x10, 0x4a, 0x74, 0x1e, 0xfa, 0x1c, 0x91, 0xbe, 0x4f, 0x04, 0x63, 0xa4, 0x2a, 0x02, - 0x2d, 0x43, 0x73, 0x00, 0x4e, 0x18, 0x09, 0xb3, 0x10, 0xa6, 0xb5, 0x90, 0x62, 0x58, 0x4a, 0x35, - 0x28, 0x2a, 0x4f, 0x6b, 0x08, 0x86, 0xc7, 0x50, 0xad, 0x69, 0x0d, 0x4c, 0xcb, 0x58, 0xfe, 0x12, - 0xa3, 0x41, 0x5c, 0xdf, 0x1c, 0xcd, 0xf3, 0x97, 0xb0, 0x12, 0x2c, 0x20, 0x48, 0x83, 0xf1, 0x56, - 0xdb, 0x14, 0xf1, 0x94, 0xa8, 0xea, 0xc3, 0xcd, 0x86, 0x57, 0xd3, 0x92, 0xf3, 0xb1, 0xd5, 0x4f, - 0xcd, 0xd0, 0x57, 0x95, 0x51, 0xe0, 0x28, 0x46, 0xf5, 0xc7, 0x06, 0x41, 0x0a, 0xaf, 0x73, 0x08, - 0x29, 0xf4, 0x27, 0x15, 0x98, 0xd6, 0x4d, 0x83, 0x58, 0x5e, 0x2c, 0x52, 0x05, 0x3f, 0x3e, 0xd6, - 0x73, 0xc5, 0xfd, 0x69, 0x11, 0xab, 0x52, 0x16, 0xcf, 0x5b, 0x4a, 0x29, 0xc8, 0xc5, 0x13, 0xa0, - 0x14, 0x08, 0x4e, 0xed, 0x0c, 0x1b, 0x0f, 0x2b, 0xaf, 0x94, 0xe5, 0xe8, 0x99, 0x25, 0x51, 0x86, - 0x03, 0x28, 0xe5, 0xbc, 0x0d, 0xc7, 0x6e, 0xb7, 0xdc, 0x12, 0x7b, 0xc5, 0xca, 0x17, 0x85, 0x71, + 0xf7, 0xc8, 0xe0, 0x27, 0x0e, 0xd1, 0xa3, 0xf7, 0x02, 0xb2, 0x37, 0xd8, 0x4d, 0xbe, 0x7e, 0x9d, + 0x5f, 0x52, 0xe9, 0x60, 0xe9, 0xf2, 0xf7, 0x2d, 0xcc, 0x8a, 0x7d, 0x89, 0x6e, 0x25, 0x6a, 0xe0, + 0x94, 0x56, 0xe8, 0x36, 0xa0, 0xe0, 0xa2, 0xcb, 0x4f, 0xfb, 0x03, 0xb6, 0x46, 0x50, 0x49, 0x6c, + 0x8d, 0xb3, 0x94, 0xd8, 0xf5, 0x04, 0x0a, 0x9c, 0x82, 0x56, 0xfd, 0xad, 0x02, 0x8c, 0xf2, 0x2d, + 0xc2, 0x2f, 0x23, 0xc7, 0x7f, 0x56, 0x90, 0xc8, 0x59, 0x51, 0xca, 0xff, 0xf9, 0xb3, 0x0e, 0x67, + 0x1e, 0x15, 0xcd, 0xd8, 0x51, 0xb1, 0xd8, 0x2b, 0xa1, 0xce, 0x27, 0xc5, 0x1f, 0x2b, 0x70, 0x4a, + 0xaa, 0x7d, 0x02, 0x07, 0x45, 0x3d, 0x7a, 0x50, 0x3c, 0xd5, 0xe3, 0xf8, 0x32, 0xce, 0x09, 0x3b, + 0x32, 0x2c, 0xc6, 0xc3, 0x1f, 0x06, 0xd8, 0x60, 0xec, 0x44, 0x12, 0x6d, 0x83, 0x25, 0x5f, 0x08, + 0x20, 0x58, 0xaa, 0x15, 0x61, 0x8a, 0x85, 0x4e, 0x4c, 0x51, 0xfd, 0xf7, 0x7d, 0x30, 0x95, 0x98, + 0xf6, 0x24, 0x1f, 0x51, 0xbe, 0x41, 0x7c, 0xa4, 0xf0, 0x8d, 0xe0, 0x23, 0x7d, 0xb9, 0xf8, 0x48, + 0xf7, 0x07, 0x91, 0x03, 0xa8, 0x69, 0x34, 0x78, 0xb3, 0x9a, 0xa7, 0x39, 0xde, 0x9a, 0xd1, 0x24, + 0x82, 0xe3, 0x7c, 0x6b, 0x77, 0x5b, 0x96, 0xb6, 0xe0, 0x8c, 0x67, 0x25, 0x81, 0x09, 0xa7, 0x60, + 0x57, 0xbf, 0xa7, 0x00, 0x43, 0x0b, 0x9a, 0xcb, 0x7a, 0xfa, 0x41, 0x18, 0x13, 0xa8, 0x2b, 0x4d, + 0xad, 0x41, 0x7a, 0x51, 0x47, 0x08, 0x94, 0x2b, 0x12, 0x3a, 0x7e, 0xa3, 0x93, 0x4b, 0x70, 0x84, + 0x1c, 0xda, 0x85, 0xd1, 0x66, 0x78, 0x7b, 0x11, 0x4b, 0xbc, 0xd4, 0x3b, 0x75, 0x8a, 0x8d, 0x5f, + 0x5b, 0xa5, 0x02, 0x2c, 0xd3, 0x52, 0x5f, 0x82, 0xd3, 0x29, 0x3d, 0xee, 0xe2, 0xe2, 0xf6, 0x16, + 0x18, 0xa2, 0x77, 0xef, 0x50, 0x0c, 0x1b, 0xdd, 0xdf, 0x2b, 0x0e, 0x3d, 0xcb, 0x8b, 0xb0, 0x0f, + 0x53, 0xdf, 0x45, 0x05, 0x80, 0x78, 0x9f, 0xba, 0x51, 0xf0, 0x0d, 0x00, 0x94, 0xe6, 0xb1, 0xed, + 0xf1, 0xad, 0xf4, 0x14, 0x0c, 0xb4, 0xb6, 0x34, 0xd7, 0x6f, 0xf1, 0x80, 0xcf, 0x2a, 0xaa, 0xb4, + 0xf0, 0xde, 0x5e, 0x71, 0x46, 0x16, 0x44, 0x44, 0x23, 0x06, 0xc3, 0xbc, 0x1d, 0xdd, 0x61, 0x74, + 0x93, 0x97, 0xec, 0x66, 0xcb, 0x24, 0x14, 0xca, 0x76, 0x58, 0x21, 0xdf, 0x0e, 0x5b, 0x4e, 0x60, + 0xc2, 0x29, 0xd8, 0x7d, 0x9a, 0x15, 0xcb, 0xf0, 0x0c, 0x2d, 0xa0, 0xd9, 0x97, 0x9f, 0x66, 0x14, + 0x13, 0x4e, 0xc1, 0x8e, 0x3e, 0xa9, 0xc0, 0x6c, 0xb4, 0x78, 0xc9, 0xb0, 0x0c, 0x77, 0x8b, 0xd4, + 0x19, 0xf1, 0xfe, 0x43, 0x13, 0xbf, 0xbc, 0xbf, 0x57, 0x9c, 0x5d, 0xce, 0xc4, 0x88, 0x3b, 0x50, + 0x43, 0x3f, 0xa0, 0xc0, 0x85, 0xd8, 0xbc, 0x38, 0x46, 0xa3, 0x41, 0x1c, 0xd1, 0x9b, 0xc3, 0x7f, + 0xe0, 0xc5, 0xfd, 0xbd, 0xe2, 0x85, 0xe5, 0x6c, 0x94, 0xb8, 0x13, 0x3d, 0xf4, 0x53, 0x0a, 0x9c, + 0x6d, 0x11, 0xab, 0x6e, 0x58, 0x8d, 0xe7, 0x6c, 0xe7, 0x36, 0x71, 0x5c, 0x6c, 0x9b, 0xa6, 0xdd, + 0xf6, 0xdc, 0x99, 0x41, 0x76, 0x86, 0x55, 0xf2, 0x7c, 0x73, 0xd5, 0x34, 0x8c, 0x0b, 0x97, 0xc5, + 0x16, 0x3d, 0x9b, 0x0a, 0x76, 0x71, 0x46, 0x47, 0xd4, 0xdf, 0x54, 0xa0, 0xaf, 0x84, 0x2b, 0xe8, + 0xc1, 0xc8, 0x27, 0x72, 0x4e, 0xfe, 0x44, 0xee, 0xed, 0x15, 0x87, 0x4a, 0xb8, 0x22, 0x7d, 0x8c, + 0x3f, 0xa0, 0xc0, 0x94, 0x6e, 0x5b, 0x9e, 0x46, 0xe7, 0x0e, 0x73, 0x59, 0xd9, 0x3f, 0x97, 0x73, + 0x5d, 0x86, 0x4b, 0x31, 0x64, 0xa1, 0xb2, 0x3b, 0x0e, 0x71, 0x71, 0x92, 0xb2, 0xfa, 0x79, 0x05, + 0xa6, 0x4b, 0x5a, 0x4b, 0xa8, 0x82, 0xca, 0x64, 0xd3, 0xa0, 0x1b, 0xa4, 0x1b, 0xcd, 0x3e, 0xda, + 0x82, 0x41, 0xa6, 0x6d, 0x76, 0x7b, 0xb9, 0xcb, 0x87, 0xb4, 0x9f, 0x65, 0xb8, 0xb8, 0x1e, 0x84, + 0xff, 0x8f, 0x05, 0x7e, 0xf5, 0x09, 0x98, 0x8c, 0xd7, 0x43, 0x45, 0x5f, 0xa6, 0xe1, 0xca, 0xc7, + 0x91, 0xb8, 0x38, 0xf2, 0xf8, 0xf0, 0xff, 0xfb, 0x5a, 0xf1, 0x4d, 0x1f, 0xfe, 0xb3, 0x2b, 0x6f, + 0x52, 0xbf, 0xaa, 0xc0, 0x58, 0xc9, 0xb4, 0xdb, 0xf5, 0xaa, 0x63, 0x6f, 0x1a, 0x26, 0x79, 0x63, + 0x68, 0x39, 0xe4, 0x1e, 0x67, 0x89, 0xae, 0x4c, 0xeb, 0x20, 0x57, 0x7c, 0x83, 0x68, 0x1d, 0xe4, + 0x2e, 0x67, 0x48, 0x93, 0x2f, 0xc2, 0x19, 0xb9, 0x56, 0xa8, 0x32, 0xbd, 0x02, 0xfd, 0xb7, 0x0d, + 0xab, 0x1e, 0xdf, 0x98, 0x37, 0x0d, 0xab, 0x8e, 0x19, 0x24, 0xd8, 0xba, 0x85, 0xcc, 0x33, 0xeb, + 0x6f, 0x46, 0xa2, 0xd3, 0xc6, 0x84, 0xd5, 0xab, 0x30, 0xac, 0x6b, 0x0b, 0x6d, 0xab, 0x6e, 0x06, + 0xbb, 0x9e, 0x4e, 0x41, 0x69, 0x9e, 0x97, 0xe1, 0x00, 0x8a, 0xee, 0x02, 0x84, 0xaf, 0x13, 0xbd, + 0x08, 0x01, 0xe1, 0xc3, 0x47, 0x8d, 0x78, 0x9e, 0x61, 0x35, 0xdc, 0x70, 0x5f, 0x85, 0x30, 0x2c, + 0x51, 0x43, 0x1f, 0x84, 0x71, 0x59, 0x22, 0xe1, 0x6a, 0xd2, 0x9c, 0xcb, 0x10, 0x11, 0x7d, 0xce, + 0x08, 0xc2, 0xe3, 0x72, 0xa9, 0x8b, 0xa3, 0xd4, 0xd0, 0x6e, 0x20, 0x7f, 0x71, 0x25, 0x6d, 0x7f, + 0xfe, 0x1b, 0x85, 0x2c, 0xfa, 0x4c, 0x0b, 0xe2, 0x63, 0x11, 0xa5, 0x71, 0x84, 0x54, 0x8a, 0x62, + 0x66, 0xe0, 0xb8, 0x14, 0x33, 0x04, 0x86, 0xb8, 0x6a, 0xca, 0x3f, 0x6e, 0x1e, 0xcf, 0x33, 0x40, + 0xae, 0xe5, 0x0a, 0x9f, 0xdb, 0xf8, 0x6f, 0x17, 0xfb, 0xb8, 0xd1, 0x36, 0x8c, 0x51, 0xc1, 0xba, + 0x46, 0x4c, 0xa2, 0x7b, 0xb6, 0x33, 0x33, 0x94, 0xff, 0x39, 0xab, 0x26, 0xe1, 0xe1, 0x52, 0xac, + 0x5c, 0x82, 0x23, 0x74, 0x02, 0xcd, 0xdd, 0x70, 0xa6, 0xe6, 0xae, 0x0d, 0xa3, 0xdb, 0x92, 0x2a, + 0x7e, 0x84, 0x4d, 0xc2, 0x93, 0x79, 0x3a, 0x16, 0xea, 0xe5, 0x17, 0x4e, 0x0b, 0x42, 0xa3, 0xb2, + 0x0e, 0x5f, 0xa6, 0x83, 0x36, 0x60, 0x68, 0x83, 0xcb, 0xa0, 0x33, 0xc0, 0xe6, 0xe2, 0x3d, 0x3d, + 0x88, 0xd6, 0x5c, 0xce, 0x15, 0x3f, 0xb0, 0x8f, 0x18, 0xbd, 0x04, 0x83, 0xa6, 0xd1, 0x34, 0x3c, + 0x77, 0x66, 0x94, 0x91, 0xc8, 0xb5, 0xb4, 0xcb, 0x0c, 0x03, 0x3f, 0xac, 0xf8, 0xff, 0x58, 0x60, + 0x45, 0x9f, 0x56, 0xe0, 0xb4, 0xd8, 0x86, 0xc1, 0xa1, 0x65, 0x10, 0x77, 0x66, 0x8c, 0xcd, 0xe1, + 0x8d, 0xde, 0x0e, 0xc9, 0xf0, 0x80, 0x5e, 0xb8, 0x20, 0x66, 0xf3, 0xf4, 0x4a, 0x92, 0x18, 0x4e, + 0xeb, 0x81, 0xfa, 0x2f, 0xc6, 0x61, 0xaa, 0x64, 0xb6, 0x5d, 0x8f, 0x38, 0xf3, 0xc2, 0x92, 0x85, + 0x38, 0xe8, 0xa3, 0x0a, 0x9c, 0x65, 0xff, 0x96, 0xed, 0x3b, 0x56, 0x99, 0x98, 0xda, 0xee, 0xfc, + 0x26, 0xad, 0x51, 0xaf, 0x1f, 0xee, 0xf0, 0x28, 0xb7, 0xc5, 0x35, 0x99, 0xbd, 0xd8, 0xd4, 0x52, + 0x31, 0xe2, 0x0c, 0x4a, 0xe8, 0xfb, 0x15, 0x38, 0x9f, 0x02, 0x2a, 0x13, 0x93, 0x78, 0xbe, 0xf0, + 0x7f, 0xd8, 0x7e, 0x5c, 0xda, 0xdf, 0x2b, 0x9e, 0xaf, 0x65, 0x21, 0xc5, 0xd9, 0xf4, 0xd0, 0x0f, + 0x2a, 0x30, 0x9b, 0x02, 0x5d, 0xd2, 0x0c, 0xb3, 0xed, 0xf8, 0xf7, 0x82, 0xc3, 0x76, 0x87, 0x89, + 0xe7, 0xb5, 0x4c, 0xac, 0xb8, 0x03, 0x45, 0xf4, 0x21, 0x38, 0x13, 0x40, 0xd7, 0x2d, 0x8b, 0x90, + 0x7a, 0xe4, 0x96, 0x70, 0xd8, 0xae, 0x9c, 0xdf, 0xdf, 0x2b, 0x9e, 0xa9, 0xa5, 0x21, 0xc4, 0xe9, + 0x74, 0x50, 0x03, 0x2e, 0x85, 0x00, 0xcf, 0x30, 0x8d, 0xbb, 0xfc, 0x22, 0xb3, 0xe5, 0x10, 0x77, + 0xcb, 0x36, 0xeb, 0x8c, 0x15, 0x2b, 0x0b, 0x6f, 0xde, 0xdf, 0x2b, 0x5e, 0xaa, 0x75, 0xaa, 0x88, + 0x3b, 0xe3, 0x41, 0x75, 0x18, 0x73, 0x75, 0xcd, 0xaa, 0x58, 0x1e, 0x71, 0xb6, 0x35, 0x73, 0x66, + 0x30, 0xd7, 0x00, 0x39, 0x03, 0x94, 0xf0, 0xe0, 0x08, 0x56, 0xf4, 0x6e, 0x18, 0x26, 0x3b, 0x2d, + 0xcd, 0xaa, 0x13, 0xce, 0x74, 0x47, 0x16, 0x2e, 0xd2, 0xa3, 0x7e, 0x51, 0x94, 0xdd, 0xdb, 0x2b, + 0x8e, 0xf9, 0xff, 0xaf, 0xd8, 0x75, 0x82, 0x83, 0xda, 0xe8, 0x03, 0x30, 0xcd, 0x4c, 0x6d, 0xea, + 0x84, 0x1d, 0x21, 0xae, 0x7f, 0x57, 0x1c, 0xce, 0xd5, 0x4f, 0xf6, 0x0c, 0xbf, 0x92, 0x82, 0x0f, + 0xa7, 0x52, 0xa1, 0xcb, 0xd0, 0xd4, 0x76, 0xae, 0x3b, 0x9a, 0x4e, 0x36, 0xdb, 0xe6, 0x1a, 0x71, + 0x9a, 0x86, 0xc5, 0x95, 0x25, 0x44, 0xb7, 0xad, 0x3a, 0x65, 0xd4, 0xca, 0xd5, 0x01, 0xbe, 0x0c, + 0x2b, 0x9d, 0x2a, 0xe2, 0xce, 0x78, 0xd0, 0x23, 0x30, 0x66, 0x34, 0x2c, 0xdb, 0x21, 0x6b, 0x9a, + 0x61, 0x79, 0xee, 0x0c, 0x30, 0x21, 0x9b, 0x4d, 0x6b, 0x45, 0x2a, 0xc7, 0x91, 0x5a, 0x68, 0x1b, + 0x90, 0x45, 0xee, 0x54, 0xed, 0x3a, 0xdb, 0x02, 0xeb, 0x2d, 0xb6, 0x91, 0x05, 0x9b, 0x3d, 0xec, + 0xd4, 0xb0, 0xab, 0xf4, 0x6a, 0x02, 0x1b, 0x4e, 0xa1, 0x80, 0x96, 0x00, 0x35, 0xb5, 0x9d, 0xc5, + 0x66, 0xcb, 0xdb, 0x5d, 0x68, 0x9b, 0xb7, 0x05, 0xd7, 0x18, 0x63, 0x73, 0xc1, 0x15, 0x4d, 0x09, + 0x28, 0x4e, 0x69, 0x81, 0x34, 0xb8, 0xc0, 0xc7, 0x53, 0xd6, 0x48, 0xd3, 0xb6, 0x5c, 0xe2, 0xb9, + 0xd2, 0x26, 0x9d, 0x19, 0x67, 0x06, 0x17, 0xec, 0x62, 0x5b, 0xc9, 0xae, 0x86, 0x3b, 0xe1, 0x88, + 0x9a, 0x9c, 0x4d, 0x1c, 0x60, 0x72, 0xf6, 0x28, 0x8c, 0xbb, 0x9e, 0xe6, 0x78, 0xed, 0x96, 0x58, + 0x86, 0x53, 0x6c, 0x19, 0x98, 0x1e, 0xb2, 0x26, 0x03, 0x70, 0xb4, 0x1e, 0x5d, 0x3e, 0xae, 0x6c, + 0x16, 0xed, 0x26, 0xc3, 0xe5, 0xab, 0x49, 0xe5, 0x38, 0x52, 0x0b, 0xad, 0xc3, 0xb9, 0xa6, 0xb6, + 0x13, 0x7c, 0xbe, 0x55, 0xcd, 0xd1, 0x4c, 0x93, 0x98, 0x86, 0xdb, 0x9c, 0x99, 0x62, 0x3d, 0x65, + 0x2f, 0xfc, 0x2b, 0xe9, 0x55, 0x70, 0x56, 0x5b, 0x61, 0x83, 0x56, 0x76, 0x34, 0x23, 0x82, 0x12, + 0x45, 0x6c, 0xd0, 0xe2, 0x60, 0x9c, 0xd6, 0x46, 0xfd, 0xef, 0xfd, 0x30, 0x93, 0x38, 0xc1, 0x7c, + 0x43, 0xb2, 0x03, 0x79, 0x94, 0x72, 0x44, 0x3c, 0xaa, 0x05, 0x57, 0x82, 0x0a, 0xd7, 0x5b, 0xed, + 0x54, 0x5a, 0x05, 0x46, 0xeb, 0xfe, 0xfd, 0xbd, 0xe2, 0x95, 0xda, 0x01, 0x75, 0xf1, 0x81, 0xd8, + 0xb2, 0xf9, 0x7f, 0xdf, 0x09, 0xf1, 0xff, 0x0f, 0xc0, 0xb4, 0x04, 0x70, 0x88, 0x56, 0xdf, 0xed, + 0xe1, 0xfc, 0x61, 0x6c, 0xaf, 0x96, 0x82, 0x0f, 0xa7, 0x52, 0xc9, 0x64, 0xba, 0x03, 0x27, 0xc1, + 0x74, 0xd5, 0xbd, 0x3e, 0x18, 0x29, 0xd9, 0x56, 0x9d, 0xeb, 0x45, 0x1e, 0x8a, 0xbc, 0x7a, 0x5f, + 0x92, 0x65, 0xe7, 0x7b, 0x7b, 0xc5, 0xf1, 0xa0, 0xa2, 0x24, 0x4c, 0x3f, 0x16, 0xbc, 0x2f, 0xf1, + 0x1b, 0xe9, 0x9b, 0xa3, 0x0f, 0x43, 0xf7, 0xf6, 0x8a, 0xa7, 0x82, 0x66, 0xd1, 0xb7, 0x22, 0xca, + 0x51, 0x4d, 0xcd, 0xf5, 0xd6, 0x1c, 0xcd, 0x72, 0x8d, 0x1e, 0x14, 0x93, 0xc1, 0x83, 0xc0, 0x72, + 0x02, 0x1b, 0x4e, 0xa1, 0x80, 0x5e, 0x86, 0x09, 0x5a, 0xba, 0xde, 0xaa, 0x6b, 0x1e, 0xc9, 0xa9, + 0x8f, 0x0c, 0x6c, 0x98, 0x96, 0x23, 0x98, 0x70, 0x0c, 0x33, 0xb7, 0x12, 0xd0, 0x5c, 0xdb, 0x62, + 0xeb, 0x19, 0xb1, 0x12, 0xa0, 0xa5, 0x58, 0x40, 0xd1, 0x03, 0x30, 0xd4, 0x24, 0xae, 0xab, 0x35, + 0x08, 0x93, 0x0a, 0x46, 0xc2, 0x8b, 0xd5, 0x0a, 0x2f, 0xc6, 0x3e, 0x1c, 0xbd, 0x0d, 0x06, 0x74, + 0xbb, 0x4e, 0xdc, 0x99, 0x21, 0xc6, 0xf8, 0xe8, 0x19, 0x30, 0x50, 0xa2, 0x05, 0xf7, 0xf6, 0x8a, + 0x23, 0xec, 0xf9, 0x84, 0xfe, 0xc2, 0xbc, 0x92, 0xfa, 0x93, 0x0a, 0x4c, 0xc6, 0x95, 0x65, 0x5d, + 0x58, 0x37, 0x9c, 0x9c, 0xa1, 0x80, 0xfa, 0x69, 0x05, 0xc6, 0x68, 0x0f, 0x1d, 0xdb, 0xac, 0x9a, + 0x9a, 0x45, 0xd0, 0xf7, 0x29, 0x30, 0xb9, 0x65, 0x34, 0xb6, 0x64, 0x3b, 0x2e, 0x21, 0xae, 0xe7, + 0x52, 0x36, 0xdd, 0x88, 0xe1, 0x5a, 0x98, 0xde, 0xdf, 0x2b, 0x4e, 0xc6, 0x4b, 0x71, 0x82, 0xa6, + 0xfa, 0x97, 0x05, 0x38, 0x27, 0xf7, 0x6c, 0x3e, 0x34, 0x91, 0x47, 0x7f, 0xac, 0x00, 0x34, 0x0d, + 0x6b, 0xde, 0x34, 0xed, 0x3b, 0xcc, 0xf8, 0x94, 0x5e, 0x80, 0x5e, 0xcc, 0xab, 0xe4, 0x4c, 0xa1, + 0x30, 0xb7, 0x12, 0x60, 0xe7, 0x0f, 0x93, 0xcf, 0xfb, 0xca, 0x93, 0x10, 0x70, 0x6f, 0xaf, 0x58, + 0x4c, 0xda, 0xe5, 0xcf, 0x61, 0x61, 0xfc, 0xbe, 0x6c, 0xb8, 0xde, 0x47, 0xff, 0xa2, 0x63, 0x15, + 0xfe, 0x42, 0x19, 0x0e, 0x64, 0xb6, 0x09, 0xa7, 0x62, 0x84, 0x53, 0x0c, 0x73, 0xcb, 0xb2, 0x61, + 0xee, 0x01, 0x4c, 0x6a, 0xce, 0x37, 0xc5, 0x9f, 0x7b, 0xa6, 0xad, 0x59, 0x1e, 0x9d, 0x69, 0xc9, + 0x90, 0xf7, 0xcf, 0x0b, 0x30, 0x2d, 0x26, 0xc0, 0xa4, 0x57, 0x94, 0x96, 0x69, 0xef, 0x36, 0x89, + 0x75, 0x12, 0xc6, 0x5a, 0xfe, 0x47, 0x50, 0xc8, 0xfc, 0x08, 0x9a, 0x89, 0x8f, 0xa0, 0x2f, 0xcf, + 0x47, 0x10, 0xf0, 0x8a, 0x03, 0x14, 0x33, 0x18, 0xce, 0x1a, 0x16, 0xed, 0xe8, 0x75, 0xb6, 0x61, + 0x42, 0x93, 0x4a, 0xc6, 0x9f, 0x86, 0xf9, 0xdd, 0xb3, 0x92, 0x5a, 0x03, 0x67, 0xb4, 0x54, 0xff, + 0x5a, 0x81, 0x99, 0xb4, 0xf9, 0x3d, 0x01, 0x5d, 0x6a, 0x33, 0xaa, 0x4b, 0xbd, 0xd1, 0xc3, 0xb7, + 0x11, 0xe9, 0x7a, 0x86, 0x4e, 0xf5, 0xaf, 0x0a, 0x70, 0x36, 0xac, 0x5e, 0xb1, 0x5c, 0x4f, 0x33, + 0x4d, 0x2e, 0x97, 0x1e, 0xff, 0x5e, 0x6a, 0x45, 0x54, 0xe2, 0xab, 0xbd, 0x0d, 0x55, 0xee, 0x7b, + 0xa6, 0x5d, 0xc7, 0x4e, 0xcc, 0xae, 0xa3, 0x7a, 0x84, 0x34, 0x3b, 0x9b, 0x78, 0xfc, 0x47, 0x05, + 0x66, 0xd3, 0x1b, 0x9e, 0xc0, 0xa6, 0xb2, 0xa3, 0x9b, 0xea, 0xbd, 0x47, 0x37, 0xea, 0x8c, 0x6d, + 0xf5, 0x0b, 0x85, 0xac, 0xd1, 0x32, 0xbd, 0xfa, 0x26, 0x9c, 0x72, 0x48, 0xc3, 0x70, 0x3d, 0x61, + 0x80, 0x70, 0x38, 0x1b, 0x6b, 0xff, 0x3d, 0xed, 0x14, 0x8e, 0xe2, 0xc0, 0x71, 0xa4, 0x68, 0x15, + 0x86, 0x5c, 0x42, 0xea, 0x14, 0x7f, 0xa1, 0x7b, 0xfc, 0x81, 0x10, 0x51, 0xe3, 0x6d, 0xb1, 0x8f, + 0x04, 0x7d, 0x07, 0x8c, 0xd7, 0x83, 0x2f, 0x8a, 0x62, 0xed, 0xeb, 0x1e, 0x2b, 0xbb, 0xa2, 0x95, + 0xe5, 0xd6, 0x38, 0x8a, 0x4c, 0xfd, 0x3b, 0x05, 0x2e, 0x76, 0xda, 0x5b, 0xe8, 0x15, 0x00, 0xdd, + 0x97, 0x0a, 0x5d, 0x71, 0x78, 0x3e, 0x91, 0x73, 0x2d, 0x39, 0x96, 0xf0, 0x03, 0x0d, 0x8a, 0x5c, + 0x2c, 0x11, 0x49, 0xb1, 0xf6, 0x2b, 0x1c, 0x93, 0xb5, 0x9f, 0xfa, 0x9f, 0x14, 0x99, 0x15, 0xc9, + 0x6b, 0xfb, 0x46, 0x63, 0x45, 0x72, 0xdf, 0x33, 0xdf, 0xe9, 0xbe, 0x52, 0x80, 0x2b, 0xe9, 0x4d, + 0xa4, 0xf3, 0xfc, 0x69, 0x18, 0x6c, 0x71, 0x3f, 0x08, 0x6e, 0x12, 0x7e, 0x95, 0x72, 0x16, 0xee, + 0xa5, 0x70, 0x6f, 0xaf, 0x38, 0x9b, 0xc6, 0xe8, 0x85, 0x7f, 0x83, 0x68, 0x87, 0x8c, 0xd8, 0x83, + 0x02, 0x17, 0xda, 0xdf, 0xd1, 0x25, 0x73, 0xd1, 0x36, 0x88, 0xd9, 0xf5, 0x1b, 0xc2, 0x47, 0x14, + 0x98, 0x88, 0xec, 0x68, 0x77, 0x66, 0x80, 0xed, 0xd1, 0x5c, 0x86, 0x56, 0x91, 0x4f, 0x25, 0x94, + 0x06, 0x22, 0xc5, 0x2e, 0x8e, 0x11, 0x8c, 0xb1, 0x59, 0x79, 0x56, 0xdf, 0x70, 0x6c, 0x56, 0xee, + 0x7c, 0x06, 0x9b, 0xfd, 0x89, 0x42, 0xd6, 0x68, 0x19, 0x9b, 0xbd, 0x03, 0x23, 0xbe, 0x18, 0xe9, + 0xb3, 0x8b, 0xa5, 0x5e, 0xfb, 0xc4, 0xd1, 0x2d, 0x4c, 0x89, 0xfe, 0x8c, 0xf8, 0x25, 0x2e, 0x0e, + 0x69, 0xa1, 0xef, 0x55, 0x00, 0xc2, 0x85, 0x11, 0x1f, 0xd5, 0xda, 0xd1, 0x4d, 0x87, 0x24, 0xd6, + 0x4c, 0xd0, 0x4f, 0x5a, 0xda, 0x14, 0x12, 0x5d, 0xf5, 0x5f, 0xf7, 0x03, 0x4a, 0xf6, 0xbd, 0xbb, + 0xe7, 0xe2, 0x03, 0x84, 0xdc, 0x16, 0x4c, 0x3a, 0x54, 0x5a, 0xd4, 0x0d, 0x93, 0x5d, 0x59, 0xed, + 0xb6, 0x97, 0x53, 0xf3, 0xc1, 0xae, 0x55, 0x38, 0x86, 0x0b, 0x27, 0xb0, 0xa3, 0xb7, 0xc0, 0x50, + 0xcb, 0x31, 0x9a, 0x9a, 0xb3, 0xcb, 0x2e, 0xc5, 0xc3, 0xfc, 0x2d, 0xab, 0xca, 0x8b, 0xb0, 0x0f, + 0x43, 0x1f, 0x80, 0x11, 0xd3, 0xd8, 0x24, 0xfa, 0xae, 0x6e, 0x12, 0xa1, 0x2a, 0xbf, 0x75, 0x34, + 0x6b, 0xbe, 0xec, 0xa3, 0x15, 0x16, 0x88, 0xfe, 0x4f, 0x1c, 0x12, 0x44, 0x15, 0x38, 0x7d, 0x87, + 0xd9, 0xc4, 0x98, 0xc4, 0x75, 0x6b, 0xed, 0x56, 0xcb, 0x76, 0x3c, 0x52, 0x67, 0x0a, 0xf5, 0x61, + 0xae, 0xd8, 0x7b, 0x2e, 0x09, 0xc6, 0x69, 0x6d, 0xd0, 0x13, 0x00, 0x5a, 0xdb, 0xb3, 0xb9, 0x43, + 0xe2, 0xcc, 0x30, 0xbb, 0xb5, 0x5f, 0xa2, 0x8b, 0x3d, 0x1f, 0x94, 0xde, 0xdb, 0x2b, 0x8e, 0x0a, + 0xdd, 0x1f, 0x5b, 0x1a, 0xa9, 0x01, 0x7a, 0x11, 0xa6, 0x75, 0x0e, 0x2a, 0xd9, 0xcd, 0x96, 0xe6, + 0x19, 0xe2, 0x46, 0x3c, 0xc2, 0x10, 0x7d, 0xcb, 0xfe, 0x5e, 0x71, 0xba, 0x94, 0x02, 0x8f, 0xa3, + 0x4c, 0x45, 0xa2, 0x7e, 0xb2, 0x00, 0x17, 0x3a, 0x4c, 0x10, 0xc2, 0xf4, 0xc3, 0x13, 0xeb, 0x27, + 0xb6, 0xd9, 0x23, 0xfc, 0x63, 0x11, 0x85, 0xf7, 0xf6, 0x8a, 0xf7, 0x75, 0x40, 0x50, 0xa3, 0xfb, + 0x9c, 0x34, 0x76, 0x71, 0x88, 0x06, 0x55, 0x60, 0xb0, 0x1e, 0xbe, 0x7d, 0x8d, 0x2c, 0x3c, 0x44, + 0x8f, 0x02, 0xae, 0xa5, 0xee, 0x16, 0x9b, 0x40, 0x80, 0x96, 0x61, 0x88, 0xdb, 0x54, 0x12, 0x71, + 0xac, 0x3c, 0xcc, 0x54, 0x26, 0xbc, 0xa8, 0x5b, 0x64, 0x3e, 0x0a, 0xf5, 0x6f, 0x15, 0x18, 0x2a, + 0xd9, 0x0e, 0x29, 0xaf, 0xd6, 0xd0, 0x2e, 0x8c, 0x4a, 0x1e, 0xf1, 0x82, 0xc5, 0xe6, 0xe4, 0x39, + 0x0c, 0xa3, 0x74, 0xb5, 0xf7, 0x7d, 0xf8, 0x82, 0x02, 0x2c, 0xd3, 0x42, 0xaf, 0xd0, 0x39, 0xbf, + 0xe3, 0x18, 0x1e, 0x25, 0xdc, 0x8b, 0x91, 0x0d, 0x27, 0x8c, 0x7d, 0x5c, 0x7c, 0xb7, 0x07, 0x3f, + 0x71, 0x48, 0x45, 0xad, 0x52, 0xf6, 0x12, 0xef, 0x26, 0x7a, 0x1c, 0xfa, 0x9b, 0x76, 0xdd, 0x5f, + 0xf7, 0xb7, 0xfa, 0xcc, 0x63, 0xc5, 0xae, 0xd3, 0xb9, 0x3d, 0x9b, 0x6c, 0xc1, 0xde, 0x93, 0x58, + 0x1b, 0x75, 0x15, 0x26, 0xe3, 0xf4, 0xd1, 0xe3, 0x30, 0xa1, 0xdb, 0xcd, 0xa6, 0x6d, 0xd5, 0xda, + 0x9b, 0x9b, 0xc6, 0x0e, 0x89, 0x38, 0x57, 0x96, 0x22, 0x10, 0x1c, 0xab, 0xa9, 0x7e, 0x4e, 0x81, + 0x3e, 0xba, 0x2e, 0x2a, 0x0c, 0xd6, 0xed, 0xa6, 0x66, 0x58, 0xa2, 0x57, 0xec, 0x95, 0xba, 0xcc, + 0x4a, 0xb0, 0x80, 0xa0, 0x16, 0x8c, 0xf8, 0x12, 0x59, 0x4f, 0x66, 0xe1, 0xe5, 0xd5, 0x5a, 0xe0, + 0x55, 0x13, 0x1c, 0x13, 0x7e, 0x89, 0x8b, 0x43, 0x22, 0xaa, 0x06, 0x53, 0xe5, 0xd5, 0x5a, 0xc5, + 0xd2, 0xcd, 0x76, 0x9d, 0x2c, 0xee, 0xb0, 0x3f, 0x94, 0xcf, 0x19, 0xbc, 0x44, 0x8c, 0x93, 0xf1, + 0x39, 0x51, 0x09, 0xfb, 0x30, 0x5a, 0x8d, 0xf0, 0x16, 0xc2, 0xd7, 0x90, 0x55, 0x13, 0x48, 0xb0, + 0x0f, 0x53, 0xbf, 0x5a, 0x80, 0x51, 0xa9, 0x43, 0xc8, 0x84, 0x21, 0x3e, 0x5c, 0xb7, 0x17, 0x7f, + 0xf2, 0x44, 0xaf, 0x39, 0x75, 0x3e, 0xa1, 0x2e, 0xf6, 0x49, 0xc8, 0x3c, 0xbb, 0xd0, 0x81, 0x67, + 0xcf, 0x45, 0x5c, 0x36, 0xf9, 0x27, 0x39, 0x91, 0xed, 0xae, 0x89, 0x2e, 0x8a, 0xe3, 0x89, 0xdb, + 0x65, 0x0f, 0xc7, 0x8e, 0xa6, 0x4d, 0x18, 0xb8, 0x6b, 0x5b, 0xc4, 0x15, 0xba, 0xf0, 0x23, 0x1a, + 0x20, 0xb3, 0xa6, 0x7b, 0x81, 0xe2, 0xc5, 0x1c, 0xbd, 0xfa, 0x53, 0x0a, 0x40, 0x59, 0xf3, 0x34, + 0x6e, 0xba, 0xd1, 0x85, 0x75, 0xe0, 0xc5, 0xc8, 0xa9, 0x3a, 0x9c, 0xf0, 0x0c, 0xeb, 0x77, 0x8d, + 0xbb, 0xfe, 0xf0, 0x03, 0x69, 0x9d, 0x63, 0xaf, 0x19, 0x77, 0x09, 0x66, 0x70, 0xf4, 0x20, 0x8c, + 0x10, 0x4b, 0x77, 0x76, 0x5b, 0xf4, 0x60, 0xe1, 0x2a, 0x1e, 0xf6, 0x85, 0x2e, 0xfa, 0x85, 0x38, + 0x84, 0xab, 0x0f, 0x41, 0xf4, 0xca, 0xd5, 0x85, 0xf1, 0xf2, 0xdf, 0x2b, 0x70, 0xae, 0xdc, 0xd6, + 0xcc, 0xf9, 0x16, 0xdd, 0xa8, 0x9a, 0xb9, 0x64, 0x73, 0x1b, 0x00, 0x7a, 0x0f, 0x79, 0x1b, 0x0c, + 0xfb, 0x42, 0x4e, 0xdc, 0x75, 0xd4, 0x67, 0x94, 0x38, 0xa8, 0x81, 0x34, 0x18, 0x76, 0x7d, 0xb1, + 0xbb, 0xd0, 0x83, 0xd8, 0xed, 0x93, 0x08, 0xc4, 0xee, 0x00, 0x2d, 0x57, 0x7e, 0xb1, 0x05, 0xaa, + 0x11, 0x67, 0xdb, 0xd0, 0xc9, 0xbc, 0xae, 0xdb, 0x6d, 0xcb, 0x73, 0x85, 0x2b, 0xb8, 0x50, 0x7e, + 0xa5, 0xd5, 0xc0, 0x19, 0x2d, 0xd5, 0x2f, 0x29, 0xd0, 0xbf, 0xb8, 0x56, 0x2a, 0xa3, 0xef, 0x80, + 0xfe, 0x80, 0x65, 0xe4, 0x34, 0xf5, 0xa1, 0x78, 0xb8, 0x9e, 0x8e, 0xaf, 0xf7, 0x0a, 0x65, 0x38, + 0x0c, 0x2b, 0xda, 0x80, 0x41, 0xb2, 0x4d, 0x68, 0x57, 0x0b, 0x47, 0x82, 0x9f, 0xb1, 0xb4, 0x45, + 0x86, 0x11, 0x0b, 0xcc, 0xea, 0xa7, 0x14, 0x80, 0xb0, 0x0a, 0xfa, 0xee, 0xb4, 0xd3, 0xe9, 0xe6, + 0x11, 0x6a, 0x9f, 0x3b, 0x1f, 0x51, 0xea, 0xaf, 0x0f, 0xc0, 0x79, 0xda, 0x1d, 0xb1, 0x55, 0x0d, + 0xdb, 0xba, 0x49, 0x76, 0xbf, 0x69, 0x26, 0xff, 0x4d, 0x33, 0xf9, 0x23, 0x34, 0x93, 0x7f, 0x11, + 0xce, 0xd3, 0x2d, 0x27, 0xc0, 0x84, 0xd9, 0x0c, 0x55, 0x1d, 0xd2, 0xd2, 0x1c, 0x52, 0x67, 0xf7, + 0x81, 0x61, 0x6e, 0x05, 0x35, 0x9f, 0x55, 0x09, 0x67, 0xb7, 0x57, 0x9f, 0x82, 0xc9, 0x70, 0xef, + 0x8a, 0x8f, 0xea, 0xc1, 0xf8, 0x25, 0x73, 0xc4, 0x97, 0x98, 0x92, 0x17, 0x43, 0xf5, 0x9e, 0x02, + 0x93, 0x8b, 0x3b, 0x2d, 0xc3, 0x61, 0xfe, 0xf9, 0xdc, 0xcd, 0x04, 0x3d, 0x10, 0x7a, 0xa3, 0x28, + 0xd1, 0x57, 0xbc, 0xb8, 0x47, 0x0a, 0xda, 0x84, 0x09, 0xc2, 0x9a, 0xb3, 0x5b, 0xa0, 0xe6, 0xe5, + 0xd9, 0xde, 0x3c, 0x28, 0x45, 0x04, 0x0b, 0x8e, 0x61, 0x45, 0x35, 0x98, 0xd0, 0x4d, 0xcd, 0x75, + 0x8d, 0x4d, 0x43, 0x0f, 0xbd, 0xa8, 0x46, 0x16, 0x1e, 0x64, 0x32, 0x57, 0x04, 0x72, 0x6f, 0xaf, + 0x78, 0x46, 0xf4, 0x33, 0x0a, 0xc0, 0x31, 0x14, 0xea, 0x67, 0x0a, 0x30, 0xbe, 0xb8, 0xd3, 0xb2, + 0xdd, 0xb6, 0x43, 0x58, 0xd5, 0x13, 0xd0, 0x6b, 0x3d, 0x00, 0x43, 0x5b, 0x9a, 0x55, 0x37, 0x89, + 0x23, 0x8e, 0xdd, 0x60, 0x6e, 0x6f, 0xf0, 0x62, 0xec, 0xc3, 0xd1, 0xab, 0x00, 0xae, 0xbe, 0x45, + 0xea, 0x6d, 0xc6, 0x1c, 0xfb, 0xf2, 0x33, 0xc7, 0xc8, 0x18, 0x6b, 0x01, 0x4a, 0x21, 0xd2, 0x04, + 0xbf, 0xb1, 0x44, 0x4e, 0xfd, 0x13, 0x05, 0xa6, 0x22, 0xed, 0x4e, 0x40, 0x5d, 0xb3, 0x19, 0x55, + 0xd7, 0xcc, 0xf7, 0x3c, 0xd6, 0x0c, 0x2d, 0xcd, 0xc7, 0x0b, 0x70, 0x2e, 0x63, 0x4e, 0x12, 0xf6, + 0xbe, 0xca, 0x09, 0xd9, 0xfb, 0xb6, 0x61, 0xd4, 0xb3, 0x4d, 0xe1, 0xec, 0xe7, 0xcf, 0x40, 0xae, + 0x23, 0x78, 0x2d, 0x40, 0x13, 0x5a, 0xf3, 0x86, 0x65, 0x2e, 0x96, 0xe9, 0xa8, 0xbf, 0xa9, 0xc0, + 0x48, 0xa0, 0x15, 0x7e, 0x5d, 0x3d, 0xa8, 0x77, 0x1f, 0x47, 0x47, 0xfd, 0xbd, 0x02, 0x9c, 0x0d, + 0x70, 0xfb, 0x6c, 0xae, 0xe6, 0x51, 0xbe, 0x71, 0xb0, 0x6a, 0xe9, 0x62, 0xc4, 0x13, 0x61, 0x38, + 0xe9, 0x98, 0xd7, 0x6a, 0x3b, 0x2d, 0xdb, 0xf5, 0xe5, 0x60, 0x7e, 0x61, 0xe0, 0x45, 0xd8, 0x87, + 0xa1, 0x55, 0x18, 0x70, 0x29, 0x3d, 0x71, 0xd6, 0x1d, 0x72, 0x36, 0x98, 0x28, 0xcf, 0xfa, 0x8b, + 0x39, 0x1a, 0xf4, 0xaa, 0xcc, 0xc3, 0x07, 0xf2, 0x2b, 0x2f, 0xe9, 0x48, 0xea, 0x81, 0x24, 0x1c, + 0x3e, 0x7d, 0x74, 0x52, 0x16, 0xaa, 0xcb, 0x30, 0x29, 0x8c, 0x5a, 0xf9, 0xb6, 0xb1, 0x74, 0x82, + 0xde, 0x1d, 0xd9, 0x19, 0xf7, 0xc7, 0x4c, 0x6a, 0xa6, 0xe3, 0xf5, 0xc3, 0x1d, 0xa3, 0xba, 0x30, + 0x7c, 0x5d, 0x74, 0x12, 0xcd, 0x42, 0xc1, 0xf0, 0xd7, 0x02, 0x04, 0x8e, 0x42, 0xa5, 0x8c, 0x0b, + 0x46, 0x17, 0x1e, 0x21, 0xf2, 0xb1, 0xd4, 0xd7, 0xf9, 0x58, 0x52, 0xbf, 0x5e, 0x80, 0x69, 0x9f, + 0xaa, 0x3f, 0xc6, 0xb2, 0x78, 0x2d, 0x3f, 0xe0, 0x52, 0x74, 0xb0, 0xaa, 0xf1, 0x16, 0xf4, 0x33, + 0x06, 0x98, 0xeb, 0x15, 0x3d, 0x40, 0x48, 0xbb, 0x83, 0x19, 0x22, 0xf4, 0x01, 0x18, 0x34, 0xe9, + 0x0d, 0xc3, 0x77, 0xd5, 0xc8, 0xa5, 0x98, 0x4d, 0x1b, 0x2e, 0xbf, 0xb8, 0x88, 0x10, 0x66, 0xc1, + 0x43, 0x28, 0x2f, 0xc4, 0x82, 0xe6, 0xec, 0x63, 0x30, 0x2a, 0x55, 0x3b, 0x54, 0xfc, 0xb2, 0xcf, + 0x15, 0x60, 0xe6, 0x06, 0x31, 0x9b, 0xa9, 0xa6, 0x0f, 0x45, 0x18, 0xd0, 0xb7, 0x34, 0x87, 0x87, + 0xc6, 0x1b, 0xe3, 0x9b, 0xbc, 0x44, 0x0b, 0x30, 0x2f, 0xa7, 0x17, 0x8e, 0x88, 0x73, 0xda, 0x93, + 0xd2, 0x4c, 0x86, 0x31, 0x13, 0xdf, 0x1f, 0x04, 0x55, 0x0c, 0x07, 0x1e, 0xa9, 0x40, 0x8f, 0x97, + 0xf7, 0xd6, 0x6e, 0xad, 0xa6, 0xb9, 0xa5, 0xa1, 0xbb, 0x30, 0x6e, 0xeb, 0x06, 0x26, 0x2d, 0xdb, + 0x35, 0x3c, 0xdb, 0xd9, 0x15, 0x8b, 0x96, 0xeb, 0x68, 0xb9, 0x55, 0xaa, 0x84, 0x88, 0xf8, 0xf3, + 0x61, 0xa4, 0x08, 0x47, 0x49, 0xa9, 0x5f, 0x54, 0x60, 0xf4, 0x86, 0xb1, 0x41, 0x1c, 0x6e, 0xb7, + 0xcb, 0x34, 0x24, 0x91, 0x20, 0x6f, 0xa3, 0x69, 0x01, 0xde, 0xd0, 0x0e, 0x8c, 0x88, 0x73, 0x38, + 0xf0, 0x3a, 0xbc, 0x9e, 0xcf, 0x5e, 0x28, 0x20, 0x2d, 0xce, 0x37, 0xe9, 0xc3, 0xf7, 0x4b, 0x5c, + 0x1c, 0x12, 0x53, 0x5f, 0x85, 0xd3, 0x29, 0x8d, 0xe8, 0x42, 0x32, 0xd3, 0x55, 0xf1, 0xd1, 0xf8, + 0xdc, 0x8a, 0x2e, 0x24, 0x2b, 0x47, 0xe7, 0xa1, 0x8f, 0x58, 0x75, 0xf1, 0xc5, 0x0c, 0xed, 0xef, + 0x15, 0xfb, 0x16, 0xad, 0x3a, 0xa6, 0x65, 0x94, 0x89, 0x9b, 0x76, 0x44, 0x62, 0x63, 0x4c, 0x7c, + 0x59, 0x94, 0xe1, 0x00, 0xca, 0x2c, 0xbc, 0xe2, 0xc6, 0x4c, 0xf4, 0x66, 0x31, 0xb9, 0x19, 0xe3, + 0x2d, 0xbd, 0xd8, 0x50, 0xc5, 0xf9, 0xd4, 0xc2, 0x8c, 0x98, 0x90, 0x04, 0xc7, 0xc3, 0x09, 0xba, + 0xea, 0xaf, 0xf4, 0xc3, 0xa5, 0x1b, 0xb6, 0x63, 0xdc, 0xb5, 0x2d, 0x4f, 0x33, 0xab, 0x76, 0x3d, + 0xb4, 0x6f, 0x15, 0x47, 0xd6, 0xc7, 0x14, 0x38, 0xa7, 0xb7, 0xda, 0xfc, 0x66, 0xe2, 0x9b, 0x88, + 0x56, 0x89, 0x63, 0xd8, 0x79, 0x1d, 0x35, 0x98, 0x39, 0x6f, 0xa9, 0xba, 0x9e, 0x86, 0x12, 0x67, + 0xd1, 0x62, 0xfe, 0x22, 0x75, 0xfb, 0x8e, 0xc5, 0x3a, 0x57, 0xf3, 0xd8, 0x6c, 0xde, 0x0d, 0x17, + 0x21, 0xa7, 0xbf, 0x48, 0x39, 0x15, 0x23, 0xce, 0xa0, 0x84, 0x3e, 0x04, 0x67, 0x0c, 0xde, 0x39, + 0x4c, 0xb4, 0xba, 0x61, 0x11, 0xd7, 0xe5, 0xc6, 0xe6, 0x3d, 0x38, 0x44, 0x54, 0xd2, 0x10, 0xe2, + 0x74, 0x3a, 0xe8, 0x25, 0x00, 0x77, 0xd7, 0xd2, 0xc5, 0xfc, 0xe7, 0x33, 0x44, 0xe5, 0x22, 0x72, + 0x80, 0x05, 0x4b, 0x18, 0xe9, 0x45, 0xcb, 0x0b, 0x36, 0xe5, 0x20, 0x33, 0x26, 0x66, 0x17, 0xad, + 0x70, 0x0f, 0x85, 0x70, 0xf5, 0x53, 0x0a, 0x4c, 0x54, 0xac, 0xaa, 0xa9, 0xe9, 0x84, 0xdb, 0x55, + 0xba, 0xe8, 0x1a, 0x8c, 0xb8, 0xc1, 0x8b, 0x0c, 0xe7, 0x08, 0xe1, 0xf7, 0x19, 0xbc, 0xc5, 0x84, + 0x75, 0x98, 0x95, 0xb6, 0x61, 0x89, 0xc3, 0x6e, 0xc9, 0x76, 0x38, 0x22, 0xf1, 0xdd, 0x71, 0x2b, + 0xed, 0x24, 0x18, 0xa7, 0xb5, 0x51, 0x7f, 0x5e, 0x81, 0xe9, 0x68, 0x77, 0x84, 0x49, 0xc3, 0x8f, + 0x2b, 0x30, 0x1d, 0x71, 0xa6, 0x16, 0xe0, 0x5e, 0xe2, 0x3b, 0x55, 0x53, 0xf0, 0x71, 0x13, 0xdf, + 0x34, 0x08, 0x4e, 0xa5, 0xaf, 0xfe, 0x13, 0x05, 0x86, 0x44, 0xac, 0x47, 0xf4, 0xd6, 0x98, 0xf6, + 0x3c, 0x38, 0xda, 0x62, 0x1a, 0xf4, 0x5d, 0x66, 0x9f, 0x21, 0x8e, 0x26, 0x71, 0xca, 0xe4, 0x52, + 0xbf, 0x0a, 0xc2, 0xe1, 0x39, 0x17, 0xb1, 0xd3, 0xf0, 0x9f, 0x66, 0x24, 0x62, 0xea, 0x6b, 0x0a, + 0x4c, 0x25, 0x5a, 0x75, 0x21, 0x8e, 0x9e, 0xa0, 0xc5, 0xea, 0x57, 0xfa, 0xe9, 0x96, 0xf4, 0x28, + 0xbf, 0x37, 0xb9, 0x62, 0xfb, 0x04, 0xee, 0xbf, 0x0f, 0xc2, 0x88, 0xd1, 0x6c, 0xb6, 0x3d, 0xf6, + 0x88, 0x38, 0x10, 0x6a, 0x8b, 0x2b, 0x7e, 0x21, 0x0e, 0xe1, 0xc8, 0x12, 0x92, 0x16, 0x3f, 0x05, + 0x97, 0xf3, 0xad, 0x9c, 0x3c, 0xc0, 0x39, 0x2a, 0x15, 0x71, 0x71, 0x28, 0x4d, 0x10, 0xfb, 0x3e, + 0x05, 0xc0, 0xf5, 0x1c, 0xc3, 0x6a, 0xd0, 0x42, 0x21, 0x8d, 0xe1, 0x23, 0x20, 0x5b, 0x0b, 0x90, + 0x72, 0xe2, 0x61, 0xfc, 0xc7, 0x00, 0x80, 0x25, 0xca, 0x68, 0x5e, 0x08, 0xa1, 0xfc, 0xc8, 0x7c, + 0x7b, 0x4c, 0xdc, 0xbe, 0x94, 0x62, 0x09, 0xcb, 0x09, 0x85, 0x52, 0xea, 0xec, 0xa3, 0x30, 0x12, + 0xd0, 0x3b, 0x48, 0xa8, 0x1b, 0x93, 0x84, 0xba, 0xd9, 0x27, 0xe0, 0x54, 0xac, 0xbb, 0x87, 0x92, + 0x09, 0xff, 0x54, 0x01, 0x14, 0x1d, 0xfd, 0x09, 0x68, 0x0e, 0x1a, 0x51, 0xcd, 0xc1, 0x42, 0xef, + 0x4b, 0x96, 0xa1, 0x3a, 0xf8, 0x59, 0x04, 0x2c, 0x14, 0x6e, 0x10, 0x1a, 0x5a, 0x9c, 0xfc, 0x54, + 0x50, 0x09, 0x3d, 0xc0, 0xc5, 0x97, 0xdb, 0x83, 0xa0, 0x72, 0x33, 0x86, 0x2b, 0x14, 0x54, 0xe2, + 0x10, 0x9c, 0xa0, 0x8b, 0x3e, 0xa1, 0xc0, 0xa4, 0x16, 0x0d, 0x85, 0xeb, 0xcf, 0x4c, 0xae, 0x00, + 0x5d, 0xb1, 0xb0, 0xba, 0x61, 0x5f, 0x62, 0x00, 0x17, 0x27, 0xc8, 0xa2, 0x47, 0x60, 0x4c, 0x6b, + 0x19, 0xf3, 0xed, 0xba, 0x41, 0x6f, 0x9e, 0x7e, 0xc4, 0x50, 0xa6, 0x0d, 0x99, 0xaf, 0x56, 0x82, + 0x72, 0x1c, 0xa9, 0x15, 0xc4, 0x9c, 0x2d, 0x85, 0x86, 0xc3, 0xbd, 0xc4, 0x9c, 0x15, 0x73, 0x18, + 0xc6, 0x9c, 0x15, 0x53, 0x27, 0x13, 0x41, 0x16, 0x80, 0x6d, 0xd4, 0x75, 0x41, 0x72, 0x30, 0xff, + 0x1b, 0xc8, 0xad, 0x4a, 0xb9, 0x24, 0x28, 0x32, 0xf1, 0x21, 0xfc, 0x8d, 0x25, 0x0a, 0xe8, 0xd3, + 0x0a, 0x8c, 0x0b, 0xde, 0x2d, 0x68, 0x0e, 0xb1, 0x25, 0x7a, 0x21, 0xef, 0x7e, 0x89, 0xed, 0xc9, + 0x39, 0x2c, 0x23, 0xe7, 0x7c, 0x27, 0x08, 0x20, 0x10, 0x81, 0xe1, 0x68, 0x3f, 0x98, 0x0c, 0xe0, + 0x46, 0x1e, 0xa1, 0x44, 0x07, 0x87, 0xf3, 0xcb, 0x00, 0xb5, 0x14, 0x7c, 0xc2, 0xc9, 0x28, 0x05, + 0x82, 0x53, 0xe9, 0x53, 0xb9, 0xf6, 0xd4, 0x1d, 0xcd, 0xd3, 0xb7, 0x4a, 0x9a, 0xbe, 0xc5, 0xde, + 0x20, 0xb9, 0x3b, 0x65, 0xce, 0x7d, 0xfd, 0x5c, 0x14, 0xd5, 0xc2, 0xe9, 0xfd, 0xbd, 0xe2, 0xa9, + 0x58, 0x21, 0x8e, 0x13, 0x44, 0x36, 0x0c, 0x3b, 0x22, 0x1e, 0xbc, 0xf0, 0x80, 0xcf, 0x17, 0x02, + 0x3d, 0x1e, 0x5c, 0x9e, 0xdf, 0x8c, 0xfc, 0x5f, 0x38, 0x20, 0x82, 0x1a, 0x70, 0x89, 0xdf, 0x0d, + 0xe7, 0x2d, 0xdb, 0xda, 0x6d, 0xda, 0x6d, 0x77, 0xbe, 0xed, 0x6d, 0x11, 0xcb, 0xf3, 0x55, 0xe1, + 0xa3, 0xec, 0x18, 0x65, 0x4e, 0x73, 0x8b, 0x9d, 0x2a, 0xe2, 0xce, 0x78, 0xd0, 0xf3, 0x30, 0xcc, + 0xde, 0xe9, 0xd6, 0xd6, 0x96, 0x99, 0x67, 0xe6, 0xe1, 0xc5, 0x65, 0x36, 0x84, 0x45, 0x81, 0x03, + 0x07, 0xd8, 0xd0, 0x6d, 0x18, 0x32, 0x79, 0x40, 0x7f, 0xe6, 0xa1, 0x99, 0x93, 0x29, 0xc6, 0x93, + 0x03, 0xf0, 0x0b, 0xb4, 0xf8, 0x81, 0x7d, 0x0a, 0xa8, 0x05, 0x57, 0xea, 0x64, 0x53, 0x6b, 0x9b, + 0xde, 0xaa, 0xed, 0x61, 0xe6, 0xa1, 0x16, 0x68, 0x3c, 0x7d, 0x27, 0xdc, 0x09, 0x16, 0x83, 0x8d, + 0xf9, 0xfe, 0x95, 0x0f, 0xa8, 0x8b, 0x0f, 0xc4, 0x86, 0x76, 0xe1, 0x3e, 0x51, 0x87, 0xb9, 0xc4, + 0xe9, 0x5b, 0x74, 0x96, 0x93, 0x44, 0x4f, 0x31, 0xa2, 0xdf, 0xb2, 0xbf, 0x57, 0xbc, 0xaf, 0x7c, + 0x70, 0x75, 0xdc, 0x0d, 0x4e, 0xe6, 0x65, 0x44, 0x62, 0x4f, 0x40, 0x33, 0x93, 0xf9, 0xe7, 0x38, + 0xfe, 0x9c, 0xc4, 0xcd, 0xe1, 0xe2, 0xa5, 0x38, 0x41, 0x13, 0x7d, 0x41, 0x81, 0x19, 0xd7, 0x73, + 0xda, 0xba, 0xd7, 0x76, 0x48, 0x3d, 0xb6, 0x43, 0xa7, 0x58, 0x87, 0x72, 0x09, 0x70, 0xb5, 0x0c, + 0x9c, 0xcc, 0x1d, 0x7c, 0x26, 0x0b, 0x8a, 0x33, 0xfb, 0x82, 0x3e, 0xaf, 0xc0, 0xb9, 0x28, 0x90, + 0xde, 0xe9, 0x79, 0x3f, 0x51, 0xfe, 0x47, 0x96, 0x5a, 0x3a, 0x4a, 0x7e, 0x83, 0xcf, 0x00, 0xe2, + 0xac, 0x8e, 0xc4, 0x5f, 0xc6, 0x4f, 0x9f, 0xf0, 0xcb, 0xf8, 0xec, 0xd3, 0x80, 0x92, 0xc7, 0xc7, + 0x41, 0x72, 0xe0, 0xb0, 0x2c, 0x07, 0x7e, 0x76, 0x00, 0x2e, 0xd0, 0x53, 0x29, 0xbc, 0xfd, 0xac, + 0x68, 0x96, 0xd6, 0x78, 0x7d, 0x4a, 0x4c, 0x5f, 0x54, 0xe0, 0xdc, 0x56, 0xba, 0x6a, 0x47, 0xdc, + 0xbf, 0x9e, 0xc9, 0xa5, 0x82, 0xeb, 0xa4, 0x2d, 0xe2, 0x0c, 0xbb, 0x63, 0x15, 0x9c, 0xd5, 0x29, + 0xf4, 0x34, 0x4c, 0x5a, 0x76, 0x9d, 0x94, 0x2a, 0x65, 0xbc, 0xa2, 0xb9, 0xb7, 0x6b, 0xbe, 0xa1, + 0xce, 0x00, 0xff, 0x5e, 0x57, 0x63, 0x30, 0x9c, 0xa8, 0x8d, 0xb6, 0x01, 0xb5, 0xec, 0xfa, 0xe2, + 0x36, 0x4f, 0x7c, 0xd1, 0x9b, 0xc9, 0x2c, 0x33, 0x16, 0xa8, 0x26, 0xb0, 0xe1, 0x14, 0x0a, 0x4c, + 0x37, 0x45, 0x3b, 0xb3, 0x62, 0x5b, 0x86, 0x67, 0x3b, 0x2c, 0xc0, 0x41, 0x4f, 0x2a, 0x1a, 0xa6, + 0x9b, 0x5a, 0x4d, 0xc5, 0x88, 0x33, 0x28, 0xa9, 0xff, 0x4d, 0x81, 0x53, 0x74, 0x5b, 0x54, 0x1d, + 0x7b, 0x67, 0xf7, 0xf5, 0xb8, 0x21, 0x1f, 0x10, 0x36, 0x8b, 0x5c, 0xb7, 0x73, 0x46, 0xb2, 0x57, + 0x1c, 0x61, 0x7d, 0x0e, 0x4d, 0x14, 0x65, 0xb5, 0x72, 0x5f, 0xb6, 0x5a, 0x59, 0xfd, 0x74, 0x81, + 0xdf, 0x5c, 0x7c, 0xb5, 0xee, 0xeb, 0xf2, 0x3b, 0x7c, 0x14, 0xc6, 0x69, 0xd9, 0x8a, 0xb6, 0x53, + 0x2d, 0x3f, 0x6b, 0x9b, 0xbe, 0x37, 0x36, 0xd3, 0xb5, 0xdf, 0x94, 0x01, 0x38, 0x5a, 0x0f, 0x3d, + 0x0e, 0x43, 0x2d, 0x1e, 0x27, 0x4c, 0xdc, 0x99, 0xaf, 0x70, 0xc3, 0x3e, 0x56, 0x74, 0x6f, 0xaf, + 0x38, 0x15, 0x3e, 0xf1, 0xfa, 0xd1, 0xca, 0xfc, 0x06, 0xea, 0xaf, 0x9c, 0x05, 0x86, 0xdc, 0x24, + 0xde, 0xeb, 0x71, 0x4e, 0x1e, 0x82, 0x51, 0xbd, 0xd5, 0x2e, 0x2d, 0xd5, 0x9e, 0x69, 0xdb, 0x4c, + 0x17, 0xc2, 0xf2, 0xc3, 0x50, 0xee, 0x5d, 0xaa, 0xae, 0xfb, 0xc5, 0x58, 0xae, 0x43, 0xb9, 0x83, + 0xde, 0x6a, 0x0b, 0x7e, 0x5b, 0x95, 0xfd, 0x55, 0x18, 0x77, 0x28, 0x55, 0xd7, 0x23, 0x30, 0x9c, + 0xa8, 0x8d, 0x3e, 0x04, 0x63, 0x44, 0x7c, 0xb8, 0x37, 0x34, 0xa7, 0x2e, 0xf8, 0x42, 0x25, 0xef, + 0xe0, 0x83, 0xa9, 0xf5, 0xb9, 0x01, 0xbf, 0x01, 0x2e, 0x4a, 0x24, 0x70, 0x84, 0x20, 0x7a, 0x11, + 0xce, 0xfb, 0xbf, 0xe9, 0x2a, 0xdb, 0xf5, 0x38, 0xa3, 0x18, 0xe0, 0x66, 0x33, 0x8b, 0x59, 0x95, + 0x70, 0x76, 0x7b, 0xf4, 0x73, 0x0a, 0x9c, 0x0d, 0xa0, 0x86, 0x65, 0x34, 0xdb, 0x4d, 0x4c, 0x74, + 0x53, 0x33, 0x9a, 0xe2, 0xde, 0xf7, 0xdc, 0x91, 0x0d, 0x34, 0x8a, 0x9e, 0x33, 0xab, 0x74, 0x18, + 0xce, 0xe8, 0x12, 0x7a, 0x4d, 0x81, 0x2b, 0x3e, 0xa8, 0xea, 0x10, 0xd7, 0x6d, 0x3b, 0x24, 0x8c, + 0x05, 0x20, 0xa6, 0x64, 0x28, 0x17, 0xef, 0x64, 0x02, 0xf0, 0xe2, 0x01, 0xb8, 0xf1, 0x81, 0xd4, + 0xe5, 0xed, 0x52, 0xb3, 0x37, 0x3d, 0x71, 0x51, 0x3c, 0xae, 0xed, 0x42, 0x49, 0xe0, 0x08, 0x41, + 0xf4, 0xf3, 0x0a, 0x9c, 0x93, 0x0b, 0xe4, 0xdd, 0xc2, 0x6f, 0x88, 0xcf, 0x1f, 0x59, 0x67, 0x62, + 0xf8, 0xb9, 0x84, 0x97, 0x01, 0xc4, 0x59, 0xbd, 0xa2, 0x6c, 0xbb, 0xc9, 0x36, 0x26, 0xbf, 0x45, + 0x0e, 0x70, 0xb6, 0xcd, 0xf7, 0xaa, 0x8b, 0x7d, 0x18, 0x7a, 0x04, 0xc6, 0x5a, 0x76, 0xbd, 0x6a, + 0xd4, 0x5d, 0x16, 0xc3, 0x8c, 0xdd, 0xf5, 0xfa, 0xf8, 0x74, 0x54, 0xed, 0x7a, 0xb5, 0x52, 0xe6, + 0xe5, 0x38, 0x52, 0x0b, 0xcd, 0x01, 0x6c, 0x6a, 0x86, 0x59, 0xbb, 0xa3, 0xb5, 0x6e, 0xf9, 0x41, + 0x71, 0x98, 0x2e, 0x62, 0x29, 0x28, 0xc5, 0x52, 0x0d, 0xba, 0x7e, 0x94, 0xef, 0x60, 0xc2, 0xc3, + 0x4e, 0xb3, 0xeb, 0xd1, 0x51, 0xac, 0x9f, 0x8f, 0x90, 0x77, 0xf8, 0xa6, 0x44, 0x02, 0x47, 0x08, + 0xa2, 0x8f, 0x29, 0x30, 0xe1, 0xee, 0xba, 0x1e, 0x69, 0x06, 0x7d, 0x38, 0x75, 0xd4, 0x7d, 0x60, + 0x3a, 0xf1, 0x5a, 0x84, 0x08, 0x8e, 0x11, 0x65, 0xe1, 0x85, 0x9a, 0x5a, 0x83, 0x5c, 0x2f, 0xdd, + 0x30, 0x1a, 0x5b, 0x41, 0x74, 0x97, 0x2a, 0x71, 0x74, 0x62, 0x79, 0xec, 0x62, 0x35, 0x20, 0xc2, + 0x0b, 0x65, 0x57, 0xc3, 0x9d, 0x70, 0xa0, 0x97, 0x60, 0x56, 0x80, 0x97, 0xed, 0x3b, 0x09, 0x0a, + 0x3c, 0x8a, 0x0f, 0x33, 0x80, 0xac, 0x64, 0xd6, 0xc2, 0x1d, 0x30, 0xa0, 0x0a, 0x9c, 0x76, 0x89, + 0xc3, 0xde, 0x04, 0x79, 0x44, 0xc8, 0x6a, 0xdb, 0x34, 0x5d, 0x76, 0xb5, 0x11, 0x2e, 0x3f, 0xb5, + 0x24, 0x18, 0xa7, 0xb5, 0x41, 0x4f, 0x04, 0x6e, 0xc1, 0xbb, 0xb4, 0xe0, 0x99, 0x6a, 0x8d, 0xdd, + 0x44, 0x06, 0xb8, 0xa6, 0x04, 0x47, 0x41, 0x38, 0x5e, 0x97, 0x9e, 0xe6, 0x7e, 0xd1, 0x42, 0xdb, + 0x71, 0xbd, 0x99, 0x69, 0xd6, 0x98, 0x9d, 0xe6, 0x58, 0x06, 0xe0, 0x68, 0x3d, 0xf4, 0x38, 0x4c, + 0xb8, 0x44, 0xd7, 0xed, 0x66, 0x4b, 0xdc, 0x93, 0x67, 0xce, 0xb0, 0xde, 0xf3, 0x15, 0x8c, 0x40, + 0x70, 0xac, 0x26, 0xda, 0x85, 0xd3, 0x41, 0x08, 0xdd, 0x65, 0xbb, 0xb1, 0xa2, 0xed, 0x30, 0xe1, + 0xf8, 0x6c, 0x9e, 0x10, 0x0f, 0x7c, 0xba, 0x4a, 0x49, 0x74, 0x38, 0x8d, 0x06, 0x5a, 0x86, 0xe9, + 0x58, 0xf1, 0x92, 0x61, 0x12, 0x77, 0xe6, 0x1c, 0x1b, 0x36, 0x53, 0x76, 0x95, 0x52, 0xe0, 0x38, + 0xb5, 0x15, 0xba, 0x05, 0x67, 0x5a, 0x8e, 0xed, 0x11, 0xdd, 0xbb, 0x49, 0x05, 0x02, 0x53, 0x0c, + 0xd0, 0x9d, 0x99, 0x61, 0x73, 0xc1, 0xde, 0x43, 0xab, 0x69, 0x15, 0x70, 0x7a, 0x3b, 0xf4, 0x59, + 0x05, 0x2e, 0xbb, 0x9e, 0x43, 0xb4, 0xa6, 0x61, 0x35, 0x4a, 0xb6, 0x65, 0x11, 0xc6, 0x98, 0x2a, + 0xf5, 0xd0, 0x63, 0xee, 0x7c, 0xae, 0x53, 0x44, 0xdd, 0xdf, 0x2b, 0x5e, 0xae, 0x75, 0xc4, 0x8c, + 0x0f, 0xa0, 0x8c, 0x5e, 0x05, 0x68, 0x92, 0xa6, 0xed, 0xec, 0x52, 0x8e, 0x34, 0x33, 0x9b, 0xff, + 0xbe, 0xbb, 0x12, 0x60, 0xe1, 0x9f, 0x7f, 0xe4, 0x25, 0x37, 0x04, 0x62, 0x89, 0x1c, 0x9d, 0x6a, + 0xca, 0x6f, 0x45, 0x14, 0x2b, 0xe9, 0xa3, 0xb9, 0xc0, 0x56, 0x8e, 0x4d, 0xf5, 0x4a, 0x5a, 0x05, + 0x9c, 0xde, 0x0e, 0xb9, 0x30, 0xc5, 0xbe, 0x50, 0x71, 0x92, 0x5f, 0x2f, 0xcd, 0x37, 0xc8, 0xcc, + 0xc5, 0x5c, 0x93, 0x4b, 0x65, 0xff, 0xa9, 0x4a, 0x1c, 0x19, 0x4e, 0xe2, 0x0f, 0x89, 0x6a, 0x3b, + 0x21, 0xd1, 0x4b, 0xbd, 0x12, 0x95, 0x90, 0xe1, 0x24, 0x7e, 0x75, 0xaf, 0x00, 0x67, 0x52, 0x4f, + 0x49, 0xca, 0x3c, 0xf8, 0x14, 0xcf, 0xfb, 0xf9, 0xbf, 0xc4, 0xb3, 0x27, 0x63, 0x1e, 0x2b, 0x51, + 0x10, 0x8e, 0xd7, 0xa5, 0x32, 0x2c, 0xa3, 0xb6, 0x54, 0x0b, 0xdb, 0x17, 0x42, 0x19, 0xb6, 0x12, + 0x83, 0xe1, 0x44, 0x6d, 0x54, 0x12, 0xf3, 0xb1, 0x54, 0xab, 0xd0, 0x6b, 0xa0, 0xbb, 0xe4, 0x10, + 0xff, 0x76, 0x10, 0x8e, 0x4f, 0x06, 0xe2, 0x64, 0x7d, 0x3a, 0x0a, 0xfa, 0x43, 0xee, 0x45, 0x7f, + 0x38, 0x8a, 0xd5, 0x28, 0x08, 0xc7, 0xeb, 0xfa, 0xf7, 0xf4, 0x48, 0x17, 0x06, 0xc2, 0x51, 0xac, + 0xc6, 0x60, 0x38, 0x51, 0x5b, 0xfd, 0xb3, 0x7e, 0xb8, 0xaf, 0x0b, 0xc9, 0x12, 0x35, 0xd3, 0xa7, + 0xfb, 0xf0, 0x3c, 0xaf, 0xbb, 0xe5, 0x69, 0x65, 0x2c, 0xcf, 0xe1, 0xe9, 0x75, 0xbb, 0x9c, 0x6e, + 0xd6, 0x72, 0x1e, 0x9e, 0x64, 0xf7, 0xcb, 0xdf, 0x4c, 0x5f, 0xfe, 0x9c, 0xb3, 0x7a, 0xe0, 0x76, + 0x69, 0x65, 0x6c, 0x97, 0x9c, 0xb3, 0xda, 0xc5, 0xf6, 0xfa, 0xf3, 0x7e, 0xb8, 0xbf, 0x1b, 0x29, + 0x37, 0xe7, 0xfe, 0x4a, 0xe1, 0x2d, 0xc7, 0xba, 0xbf, 0xb2, 0xfc, 0xb9, 0x8f, 0x71, 0x7f, 0x75, + 0x64, 0x9f, 0xc7, 0xb3, 0xbf, 0xb2, 0x66, 0xf5, 0xb8, 0xf6, 0x57, 0xd6, 0xac, 0x76, 0xb1, 0xbf, + 0xfe, 0x26, 0x7e, 0x3e, 0x04, 0xa2, 0x76, 0x05, 0xfa, 0xf4, 0x56, 0x3b, 0x27, 0x93, 0x62, 0x56, + 0x86, 0xa5, 0xea, 0x3a, 0xa6, 0x38, 0x10, 0x86, 0x41, 0xbe, 0x7f, 0x72, 0xb2, 0x20, 0x66, 0x39, + 0xca, 0xb7, 0x24, 0x16, 0x98, 0xe8, 0x54, 0x91, 0xd6, 0x16, 0x69, 0x12, 0x47, 0x33, 0x6b, 0x9e, + 0xed, 0x68, 0x8d, 0xbc, 0xdc, 0x86, 0xbf, 0xa0, 0xc4, 0x70, 0xe1, 0x04, 0x76, 0x3a, 0x21, 0x2d, + 0xa3, 0x9e, 0x93, 0xbf, 0xb0, 0x09, 0xa9, 0x56, 0xca, 0x98, 0xe2, 0x50, 0x7f, 0x7d, 0x18, 0xa4, + 0xe0, 0xf4, 0xe8, 0x93, 0x0a, 0x4c, 0xe9, 0xf1, 0x98, 0x9c, 0xbd, 0xd8, 0x43, 0x25, 0x02, 0x7c, + 0xf2, 0x2d, 0x9f, 0x28, 0xc6, 0x49, 0xb2, 0xe8, 0xc3, 0x0a, 0x57, 0xf2, 0x05, 0xaf, 0x79, 0x62, + 0x5a, 0xaf, 0x1f, 0xd1, 0xbb, 0x77, 0xa8, 0x2d, 0x0c, 0x9f, 0x58, 0xa3, 0x04, 0xd1, 0x6b, 0x0a, + 0x9c, 0xb9, 0x9d, 0xf6, 0x36, 0x21, 0x26, 0xff, 0x56, 0xde, 0xae, 0x64, 0x3c, 0x76, 0x70, 0x09, + 0x32, 0xb5, 0x02, 0x4e, 0xef, 0x48, 0x30, 0x4b, 0x81, 0xba, 0x56, 0x7c, 0xa7, 0xb9, 0x67, 0x29, + 0xa6, 0xf7, 0x0d, 0x67, 0x29, 0x00, 0xe0, 0x28, 0x41, 0xd4, 0x82, 0x91, 0xdb, 0xbe, 0x8e, 0x5c, + 0xe8, 0xc5, 0x4a, 0x79, 0xa9, 0x4b, 0x8a, 0x76, 0x6e, 0xef, 0x15, 0x14, 0xe2, 0x90, 0x08, 0xda, + 0x82, 0xa1, 0xdb, 0x9c, 0x57, 0x08, 0x7d, 0xd6, 0x7c, 0xcf, 0xb7, 0x7f, 0xae, 0x56, 0x11, 0x45, + 0xd8, 0x47, 0x2f, 0xfb, 0x12, 0x0c, 0x1f, 0xe0, 0xe2, 0xf6, 0x59, 0x05, 0xce, 0x6c, 0x13, 0xc7, + 0x33, 0xf4, 0xf8, 0xcb, 0xd0, 0x48, 0x7e, 0x0d, 0xc5, 0xb3, 0x69, 0x08, 0xf9, 0x36, 0x49, 0x05, + 0xe1, 0xf4, 0x2e, 0xa0, 0x67, 0xa1, 0x9f, 0x78, 0x7a, 0x5d, 0x04, 0x70, 0x7e, 0x77, 0x5e, 0x97, + 0x5d, 0xee, 0xf7, 0x42, 0xff, 0xc3, 0x0c, 0x9f, 0xfa, 0x57, 0x0a, 0x24, 0x74, 0xd3, 0xe8, 0x87, + 0x14, 0x18, 0xdb, 0x24, 0x9a, 0xd7, 0x76, 0xc8, 0x75, 0x61, 0x13, 0xda, 0x77, 0x75, 0xf4, 0xe1, + 0x67, 0x8f, 0x42, 0x25, 0x3e, 0xb7, 0x24, 0x21, 0xe6, 0xc6, 0x2a, 0x41, 0xc2, 0x09, 0x19, 0x84, + 0x23, 0x3d, 0x98, 0x7d, 0x0a, 0xa6, 0x12, 0x0d, 0x0f, 0xf5, 0x4c, 0xf9, 0x6b, 0x0a, 0xa4, 0xe5, + 0x36, 0x47, 0x2f, 0xc1, 0x80, 0x56, 0xaf, 0x07, 0x29, 0x2e, 0x1f, 0xcb, 0x67, 0x37, 0x55, 0x97, + 0x23, 0x05, 0xb1, 0x9f, 0x98, 0xa3, 0x45, 0x4b, 0x80, 0xb4, 0xc8, 0xbb, 0xf4, 0x4a, 0x18, 0xa2, + 0x82, 0x3d, 0xa7, 0xcd, 0x27, 0xa0, 0x38, 0xa5, 0x85, 0xfa, 0x71, 0x05, 0x50, 0x32, 0x45, 0x09, + 0x72, 0x60, 0x58, 0xec, 0x5f, 0x7f, 0x95, 0xca, 0x39, 0xbd, 0xe9, 0x22, 0xae, 0xa1, 0xa1, 0x11, + 0x9e, 0x28, 0x70, 0x71, 0x40, 0x47, 0xfd, 0x3b, 0x05, 0xc2, 0x34, 0x78, 0xe8, 0x9d, 0x30, 0x5a, + 0x27, 0xae, 0xee, 0x18, 0x2d, 0x2f, 0x74, 0x24, 0x0d, 0x1c, 0xd2, 0xca, 0x21, 0x08, 0xcb, 0xf5, + 0x90, 0x0a, 0x83, 0x9e, 0xe6, 0xde, 0xae, 0x94, 0xc5, 0x65, 0x8f, 0x1d, 0xcd, 0x6b, 0xac, 0x04, + 0x0b, 0x48, 0x18, 0x3a, 0xb6, 0xaf, 0x8b, 0xd0, 0xb1, 0x68, 0xf3, 0x08, 0xe2, 0xe4, 0xa2, 0x83, + 0x63, 0xe4, 0xaa, 0x3f, 0x53, 0x80, 0x53, 0xb4, 0xca, 0x8a, 0x66, 0x58, 0x1e, 0xb1, 0x98, 0xdb, + 0x54, 0xce, 0x49, 0x68, 0xc0, 0xb8, 0x17, 0x71, 0x5a, 0x3e, 0xbc, 0x53, 0x6d, 0x60, 0xe9, 0x15, + 0x75, 0x55, 0x8e, 0xe2, 0x45, 0x8f, 0xf9, 0x7e, 0x6b, 0xfc, 0x5a, 0x7c, 0x9f, 0xbf, 0x55, 0x99, + 0x33, 0xda, 0x3d, 0xe1, 0x01, 0x1e, 0xe4, 0x4e, 0x8c, 0xb8, 0xa8, 0x3d, 0x0a, 0xe3, 0xc2, 0x43, + 0x82, 0xc7, 0x00, 0x16, 0xd7, 0x62, 0x76, 0xac, 0x2c, 0xc9, 0x00, 0x1c, 0xad, 0xa7, 0xfe, 0x61, + 0x01, 0xa2, 0x19, 0x1a, 0xf3, 0xce, 0x52, 0x32, 0x00, 0x72, 0xe1, 0xd8, 0x02, 0x20, 0xf3, 0x1c, + 0xe5, 0xcc, 0x9c, 0x5b, 0xbc, 0xb3, 0xcb, 0xf9, 0x89, 0x59, 0x39, 0x0e, 0x6a, 0x84, 0xd3, 0xda, + 0x7f, 0xe8, 0x69, 0x7d, 0xa7, 0xb0, 0xfc, 0x1d, 0x88, 0x84, 0xa1, 0xf6, 0x2d, 0x7f, 0xa7, 0x22, + 0x0d, 0x25, 0x2f, 0xbb, 0x79, 0x10, 0x39, 0x4e, 0xe8, 0xba, 0x88, 0x10, 0xd9, 0xee, 0x9a, 0xed, + 0x69, 0x26, 0x9b, 0x4e, 0xa1, 0x74, 0x5d, 0x91, 0x01, 0x38, 0x5a, 0x4f, 0x5d, 0x85, 0x37, 0x2f, + 0xdb, 0x5a, 0x7d, 0x41, 0x33, 0xe9, 0xd6, 0x75, 0x84, 0x59, 0x9e, 0xcb, 0x4e, 0xe6, 0xaa, 0x63, + 0x7b, 0xb6, 0x6e, 0x9b, 0xf4, 0xdc, 0xd4, 0x82, 0x58, 0xc1, 0xec, 0x21, 0xd2, 0x3f, 0x37, 0x45, + 0xac, 0x5d, 0xec, 0xc3, 0xd5, 0xdf, 0x51, 0x60, 0x48, 0xe4, 0x3d, 0xe9, 0xc2, 0xb1, 0x74, 0x13, + 0x06, 0xd8, 0xed, 0xa8, 0x17, 0xa9, 0xb4, 0xb6, 0x65, 0xdb, 0x5e, 0x24, 0x61, 0x12, 0x4f, 0x39, + 0xc6, 0x92, 0x44, 0x72, 0xf4, 0xcc, 0x1e, 0xd5, 0xd1, 0xb7, 0x0c, 0x8f, 0x30, 0xb3, 0x1b, 0xb1, + 0xf1, 0xb9, 0x3d, 0xaa, 0x54, 0x8e, 0x23, 0xb5, 0xd4, 0xff, 0x3a, 0x00, 0x57, 0xfc, 0x1c, 0x2e, + 0x71, 0x51, 0x2d, 0xe0, 0xb9, 0xbb, 0x41, 0x52, 0x19, 0x16, 0x14, 0xdf, 0xd7, 0xa9, 0xe6, 0xbb, + 0x25, 0x9f, 0x93, 0xd2, 0xc6, 0xc8, 0xe8, 0x70, 0x1a, 0x0d, 0x1e, 0x7d, 0x9d, 0x15, 0xdf, 0x20, + 0x9a, 0xe9, 0x6d, 0xf9, 0xb4, 0x0b, 0xbd, 0x44, 0x5f, 0x4f, 0xe2, 0xc3, 0xa9, 0x54, 0x98, 0x49, + 0x87, 0x9f, 0xcc, 0xc6, 0x21, 0x9a, 0x6c, 0x4f, 0xd2, 0x83, 0xbb, 0xd1, 0x4a, 0x2a, 0x46, 0x9c, + 0x41, 0x89, 0xa9, 0x1b, 0xb5, 0x1d, 0xa6, 0xbd, 0xc0, 0xc4, 0x73, 0x0c, 0x96, 0xf8, 0x2a, 0x78, + 0xab, 0x58, 0x89, 0x82, 0x70, 0xbc, 0x2e, 0x7a, 0x1c, 0x26, 0x98, 0x89, 0x4c, 0x18, 0xce, 0x73, + 0x20, 0x0c, 0xea, 0xb4, 0x1a, 0x81, 0xe0, 0x58, 0x4d, 0xf4, 0x3d, 0x0a, 0x4c, 0x1b, 0xb2, 0x33, + 0x8d, 0x3f, 0xfa, 0x7c, 0x99, 0x51, 0xd8, 0xf3, 0x92, 0xbf, 0x8d, 0x53, 0xd0, 0xe2, 0x54, 0x62, + 0x68, 0x19, 0xa6, 0x85, 0x47, 0x74, 0x74, 0x0f, 0xf0, 0x58, 0x6f, 0x6c, 0x4d, 0xcb, 0x29, 0x70, + 0x9c, 0xda, 0x4a, 0xfd, 0x48, 0x01, 0xc6, 0x0e, 0x99, 0xc4, 0xb4, 0x2d, 0xc9, 0x1c, 0x3d, 0xf8, + 0x2d, 0xca, 0x54, 0xbb, 0x10, 0x3b, 0xd0, 0xf3, 0x30, 0xd1, 0x66, 0x13, 0xe1, 0x47, 0x42, 0x13, + 0xdf, 0xf4, 0xb7, 0xd1, 0x95, 0x5b, 0x8f, 0x40, 0xee, 0xed, 0x15, 0x67, 0x65, 0xf4, 0x51, 0x28, + 0x8e, 0xe1, 0x51, 0x7f, 0xa3, 0x00, 0x48, 0xae, 0xbe, 0x64, 0x6a, 0xdb, 0xb6, 0x83, 0x3e, 0xaf, + 0xc0, 0x98, 0x2e, 0xa7, 0x8d, 0xe2, 0x02, 0xd6, 0xf3, 0xbd, 0x0e, 0x96, 0xa3, 0x9f, 0x93, 0xf3, + 0x41, 0x71, 0x41, 0xd8, 0x77, 0xb7, 0x1e, 0x93, 0x41, 0xf7, 0x62, 0xbf, 0x71, 0xa4, 0x4f, 0xb3, + 0x1f, 0x53, 0x60, 0x2a, 0x81, 0x29, 0x45, 0x32, 0x7e, 0x21, 0x1a, 0x03, 0xfd, 0x48, 0x12, 0x44, + 0xca, 0xf2, 0xf5, 0x4f, 0x0e, 0xc0, 0xe9, 0x94, 0x15, 0x65, 0x26, 0x36, 0x24, 0x26, 0x5d, 0xf6, + 0x62, 0x62, 0x93, 0x90, 0x54, 0x03, 0x13, 0x9b, 0x38, 0x04, 0x27, 0xe8, 0xa2, 0x67, 0xa1, 0x4f, + 0x77, 0x0c, 0xb1, 0x69, 0x1f, 0xcd, 0x35, 0x05, 0xb8, 0xb2, 0x30, 0x2a, 0x28, 0xf6, 0x95, 0x70, + 0x05, 0x53, 0x84, 0xf4, 0x2c, 0x96, 0x8f, 0x11, 0x5f, 0x60, 0x65, 0x67, 0xb1, 0x7c, 0xda, 0xb8, + 0x38, 0x5a, 0x0f, 0x3d, 0x0f, 0x33, 0xe2, 0xa6, 0xea, 0x47, 0x32, 0xb1, 0x2d, 0xd7, 0xa3, 0x1c, + 0xdf, 0x13, 0x32, 0x05, 0xb3, 0x7e, 0xbd, 0x99, 0x51, 0x07, 0x67, 0xb6, 0x46, 0xdf, 0x0d, 0x13, + 0x11, 0xee, 0xe1, 0x47, 0x25, 0xcb, 0xe9, 0x32, 0x23, 0x63, 0xe2, 0xbc, 0x32, 0x5a, 0x86, 0x63, + 0xd4, 0xd0, 0xff, 0xad, 0xc0, 0x54, 0xb0, 0x51, 0x77, 0xf9, 0x96, 0xf7, 0x33, 0xf8, 0x2d, 0x1d, + 0xcd, 0x17, 0x24, 0xa5, 0x57, 0x8d, 0x13, 0xc2, 0x49, 0xda, 0xea, 0x8f, 0x0e, 0x82, 0x9c, 0xd1, + 0x19, 0xad, 0xf4, 0xa2, 0x97, 0x0c, 0xf7, 0x80, 0xaf, 0x9b, 0x5c, 0x81, 0xbe, 0x46, 0xab, 0x9d, + 0x53, 0x31, 0x19, 0xa0, 0xbb, 0x4e, 0xd1, 0x35, 0x5a, 0x6d, 0xf4, 0x6c, 0xa0, 0xea, 0xcc, 0xa7, + 0x8c, 0x0c, 0x5c, 0x25, 0x63, 0xea, 0x4e, 0x9f, 0xbd, 0xf7, 0x67, 0xb2, 0xf7, 0x26, 0x0c, 0xb9, + 0x42, 0x0f, 0x3a, 0x90, 0x3f, 0x8c, 0xa4, 0x34, 0xd3, 0x42, 0xef, 0xc9, 0x35, 0x34, 0xbe, 0x5a, + 0xd4, 0xa7, 0x41, 0x2f, 0x82, 0x6d, 0x76, 0x2e, 0x89, 0x20, 0x49, 0xec, 0x22, 0xb8, 0xce, 0x4a, + 0xb0, 0x80, 0x24, 0x84, 0xb9, 0xa1, 0x6e, 0x84, 0x39, 0xf4, 0xb9, 0x38, 0xff, 0x1e, 0x66, 0xbb, + 0xef, 0x99, 0x1e, 0x87, 0xf3, 0x7f, 0x18, 0xe3, 0xfe, 0xbf, 0xc2, 0xc3, 0x4f, 0x5a, 0x2c, 0x74, + 0x1f, 0x0c, 0xb0, 0x28, 0x4a, 0x42, 0x0e, 0x08, 0x94, 0x1b, 0x2c, 0x8e, 0x0e, 0xe6, 0x30, 0x54, + 0x13, 0xa1, 0x03, 0xf3, 0x6d, 0x7a, 0x66, 0xdb, 0x28, 0xe8, 0x49, 0x71, 0x06, 0xaf, 0x44, 0x7c, + 0x22, 0xd3, 0xee, 0x10, 0xeb, 0x30, 0xd4, 0x34, 0x2c, 0x66, 0xee, 0x91, 0x4f, 0x89, 0xce, 0x4d, + 0xb0, 0x38, 0x0a, 0xec, 0xe3, 0x52, 0xff, 0xbc, 0x40, 0x19, 0x44, 0x78, 0xa9, 0xdf, 0xe5, 0x81, + 0x70, 0x85, 0xf7, 0xb5, 0x92, 0x5f, 0x09, 0x28, 0x21, 0x9d, 0x0f, 0x10, 0x72, 0x43, 0x85, 0xf0, + 0x37, 0x96, 0x88, 0x51, 0xd2, 0x9e, 0xd1, 0x24, 0xcf, 0x19, 0x56, 0xdd, 0xbe, 0x23, 0xa6, 0xb7, + 0x57, 0xd2, 0x6b, 0x01, 0x42, 0x4e, 0x3a, 0xfc, 0x8d, 0x25, 0x62, 0xf4, 0x48, 0x62, 0xc9, 0x38, + 0x2c, 0x96, 0x89, 0x57, 0xf4, 0x8d, 0xe7, 0xd9, 0x16, 0x76, 0xc7, 0xec, 0x48, 0x2a, 0x65, 0xd4, + 0xc1, 0x99, 0xad, 0xd5, 0x9f, 0x53, 0xe0, 0x4c, 0xea, 0x54, 0xa0, 0xeb, 0x30, 0x15, 0x9a, 0xc3, + 0xca, 0x42, 0xc2, 0x70, 0xc8, 0xe3, 0x6f, 0xc6, 0x2b, 0xe0, 0x64, 0x1b, 0x9e, 0xdf, 0x2c, 0x21, + 0x84, 0x08, 0x5b, 0x5a, 0xf9, 0xaa, 0x25, 0x83, 0x71, 0x5a, 0x1b, 0xf5, 0xc5, 0x48, 0x67, 0xc3, + 0xc9, 0xa2, 0x5f, 0xc6, 0x06, 0x69, 0x04, 0x3e, 0xe9, 0xc1, 0x97, 0xb1, 0x40, 0x0b, 0x31, 0x87, + 0xa1, 0x4b, 0x72, 0xa8, 0x8c, 0x80, 0xbb, 0xfb, 0xe1, 0x32, 0xd4, 0x5f, 0x53, 0xe0, 0xdc, 0x8a, + 0x66, 0xb5, 0x35, 0x93, 0x3b, 0xbf, 0x57, 0x6d, 0xdb, 0x14, 0xd3, 0xd4, 0x29, 0xdf, 0xba, 0xf2, + 0x7a, 0xc9, 0xb7, 0xfe, 0x7e, 0x38, 0x97, 0x61, 0x7f, 0x83, 0xca, 0x30, 0xe6, 0xde, 0xd1, 0x5a, + 0x0b, 0x64, 0x4b, 0xdb, 0x36, 0x44, 0x60, 0x2d, 0x6e, 0xa6, 0x3d, 0x56, 0x93, 0xca, 0xef, 0xc5, + 0x7e, 0xe3, 0x48, 0x2b, 0xd5, 0x03, 0x10, 0xe6, 0xfc, 0x86, 0xd5, 0x40, 0x9b, 0x30, 0xac, 0x99, + 0xc4, 0xf1, 0xc2, 0xe0, 0x91, 0xdf, 0x9e, 0x4b, 0x4f, 0x2b, 0x70, 0x70, 0xf7, 0x35, 0xff, 0x17, + 0x0e, 0x70, 0xab, 0xff, 0x48, 0x81, 0xb3, 0xe9, 0xa1, 0x94, 0xba, 0xb8, 0x16, 0x35, 0x61, 0xd4, + 0x09, 0x9b, 0x89, 0x8f, 0xf6, 0x5d, 0x72, 0x86, 0x0a, 0xc9, 0xf1, 0x86, 0x5e, 0x04, 0x4b, 0x8e, + 0xed, 0xfa, 0x3b, 0x37, 0x9e, 0xb4, 0x22, 0xd0, 0x8a, 0x49, 0x3d, 0xc1, 0x32, 0x7e, 0x96, 0x40, + 0x86, 0x52, 0x77, 0x5b, 0x9a, 0x4e, 0xea, 0x27, 0x9c, 0x53, 0xfd, 0x08, 0xb2, 0x36, 0xa4, 0xf7, + 0xfd, 0x78, 0x13, 0xc8, 0x64, 0xd0, 0x3c, 0x38, 0x81, 0x4c, 0x7a, 0xc3, 0x37, 0x48, 0x66, 0x83, + 0xf4, 0xce, 0x67, 0x38, 0xbe, 0xff, 0xe7, 0xc1, 0xac, 0xd1, 0x1e, 0x32, 0x31, 0xfb, 0xf6, 0x31, + 0x26, 0x66, 0x9f, 0xf8, 0x66, 0x52, 0xf6, 0x94, 0xa4, 0xec, 0xb1, 0x44, 0xe1, 0x83, 0x27, 0x94, + 0x28, 0xfc, 0x15, 0x18, 0x6c, 0x69, 0x0e, 0xb1, 0xfc, 0xb7, 0xdc, 0x4a, 0x3e, 0x43, 0x83, 0x70, + 0xa3, 0x85, 0x5c, 0x30, 0xf8, 0x24, 0xab, 0x8c, 0x00, 0x16, 0x84, 0x52, 0x82, 0xa7, 0x0c, 0x1f, + 0x57, 0x74, 0xc2, 0x30, 0x45, 0xf9, 0xc8, 0x71, 0xa4, 0x28, 0x57, 0xff, 0x56, 0x81, 0x8b, 0x9d, + 0xd8, 0x12, 0x53, 0xa0, 0xe8, 0xb1, 0xcf, 0xb0, 0x17, 0x05, 0x4a, 0x82, 0xdb, 0x06, 0x0a, 0x94, + 0x38, 0x04, 0x27, 0xe8, 0xa2, 0xf7, 0x02, 0xb2, 0x37, 0xb8, 0x5d, 0xcf, 0x75, 0x4a, 0x83, 0x7b, + 0xd3, 0x16, 0x98, 0xaf, 0x42, 0x90, 0xd7, 0xf2, 0x56, 0xa2, 0x06, 0x4e, 0x69, 0xa5, 0xfe, 0x4a, + 0x01, 0x60, 0x95, 0x78, 0x77, 0x6c, 0xe7, 0x36, 0x3d, 0xe3, 0x2f, 0x46, 0x9e, 0x0e, 0x86, 0xbf, + 0x71, 0xf1, 0x28, 0x2f, 0x42, 0x7f, 0xcb, 0xae, 0xbb, 0xe2, 0xfe, 0xc1, 0x3a, 0xc2, 0x5c, 0x35, + 0x58, 0x29, 0x2a, 0xc2, 0x00, 0x33, 0x7a, 0x12, 0x17, 0x68, 0xf6, 0xf0, 0xc0, 0x1e, 0x57, 0x30, + 0x2f, 0xa7, 0x1c, 0x52, 0xc4, 0x34, 0x70, 0xc5, 0xe3, 0xce, 0x18, 0x8f, 0x21, 0xce, 0xcb, 0x70, + 0x00, 0x45, 0x8f, 0x03, 0x18, 0xad, 0x25, 0xad, 0x69, 0x98, 0x86, 0xf8, 0x5c, 0x47, 0x98, 0x46, + 0x1c, 0x2a, 0x55, 0xbf, 0xf4, 0xde, 0x5e, 0x71, 0x58, 0xfc, 0xda, 0xc5, 0x52, 0x6d, 0xf5, 0x8b, + 0x0a, 0x4c, 0x86, 0x93, 0x27, 0xb6, 0x8a, 0xdf, 0x73, 0x1e, 0x0c, 0x38, 0xb3, 0xe7, 0x3c, 0x6c, + 0x7f, 0xe7, 0x9e, 0x73, 0x05, 0x56, 0x56, 0xcf, 0x1f, 0x82, 0x51, 0xc2, 0x43, 0x1e, 0x55, 0xca, + 0x98, 0xf3, 0xb8, 0x11, 0x7e, 0x9d, 0x5b, 0x0c, 0x8b, 0xb1, 0x5c, 0x47, 0xfd, 0xfb, 0x3e, 0x18, + 0x5b, 0x6d, 0x18, 0xd6, 0x8e, 0x1f, 0xdb, 0x29, 0x78, 0x78, 0x57, 0x8e, 0xe7, 0xe1, 0xfd, 0x79, + 0x98, 0x31, 0xe5, 0x67, 0x2e, 0x2e, 0x38, 0x69, 0x56, 0x23, 0x98, 0x01, 0x76, 0x8f, 0x59, 0xce, + 0xa8, 0x83, 0x33, 0x5b, 0x23, 0x0f, 0x06, 0x75, 0x3f, 0xbf, 0x62, 0xee, 0x78, 0x45, 0xf2, 0x5c, + 0xcc, 0xc9, 0xa1, 0x3b, 0x02, 0x9e, 0x27, 0xb6, 0xa7, 0xa0, 0x85, 0x3e, 0xaa, 0xc0, 0x19, 0xb2, + 0xc3, 0x43, 0xd7, 0xac, 0x39, 0xda, 0xe6, 0xa6, 0xa1, 0x0b, 0x8f, 0x3f, 0xbe, 0x13, 0x97, 0xf7, + 0xf7, 0x8a, 0x67, 0x16, 0xd3, 0x2a, 0xdc, 0xdb, 0x2b, 0x5e, 0x4b, 0x8d, 0x24, 0xc4, 0x56, 0x33, + 0xb5, 0x09, 0x4e, 0x27, 0x35, 0xfb, 0x18, 0x8c, 0x1e, 0xc2, 0x4f, 0x3c, 0x12, 0x2f, 0xe8, 0x57, + 0x0b, 0x30, 0x46, 0xb7, 0xdb, 0xb2, 0xad, 0x6b, 0x66, 0x79, 0xb5, 0x86, 0x1e, 0x88, 0x87, 0x49, + 0x0c, 0x9e, 0x18, 0x13, 0xa1, 0x12, 0x97, 0x61, 0x7a, 0xd3, 0x76, 0x74, 0xb2, 0x56, 0xaa, 0xae, + 0xd9, 0xc2, 0xf8, 0xac, 0xbc, 0x5a, 0x13, 0xf7, 0x3a, 0xf6, 0xe4, 0xb1, 0x94, 0x02, 0xc7, 0xa9, + 0xad, 0xd0, 0x2d, 0x38, 0x13, 0x96, 0xaf, 0xb7, 0xb8, 0xc3, 0x02, 0x45, 0xd7, 0x17, 0x3a, 0x5c, + 0x2c, 0xa5, 0x55, 0xc0, 0xe9, 0xed, 0x90, 0x06, 0x17, 0xc4, 0xdb, 0xca, 0x92, 0xed, 0xdc, 0xd1, + 0x9c, 0x7a, 0x14, 0x6d, 0x7f, 0x98, 0xab, 0xbc, 0x9c, 0x5d, 0x0d, 0x77, 0xc2, 0xa1, 0xfe, 0x9d, + 0x02, 0xd1, 0x20, 0x94, 0xe8, 0x3c, 0xf4, 0x39, 0x22, 0x7d, 0x9f, 0x08, 0xc6, 0x48, 0xaf, 0x08, + 0xb4, 0x0c, 0xcd, 0x01, 0x38, 0x61, 0x24, 0xcc, 0x42, 0x98, 0xd6, 0x42, 0x8a, 0x61, 0x29, 0xd5, + 0xa0, 0xa8, 0x3c, 0xad, 0x21, 0x18, 0x1e, 0x43, 0xb5, 0xa6, 0x35, 0x30, 0x2d, 0x63, 0xf9, 0x4b, + 0x8c, 0x06, 0x71, 0x7d, 0x75, 0x34, 0xcf, 0x5f, 0xc2, 0x4a, 0xb0, 0x80, 0x20, 0x0d, 0xc6, 0x5b, + 0x6d, 0x53, 0xc4, 0x53, 0xa2, 0x57, 0x1f, 0xae, 0x36, 0xbc, 0x9a, 0x96, 0x9c, 0x8f, 0xad, 0x7e, + 0x6a, 0x86, 0xbe, 0xaa, 0x8c, 0x02, 0x47, 0x31, 0xaa, 0x3f, 0x31, 0x08, 0x52, 0x78, 0x9d, 0x43, + 0x48, 0xa1, 0x3f, 0xad, 0xc0, 0xb4, 0x6e, 0x1a, 0xc4, 0xf2, 0x62, 0x91, 0x2a, 0xf8, 0xf1, 0xb1, + 0x9e, 0x2b, 0xee, 0x4f, 0x8b, 0x58, 0x95, 0xb2, 0x70, 0x6f, 0x29, 0xa5, 0x20, 0x17, 0x2e, 0x40, + 0x29, 0x10, 0x9c, 0xda, 0x19, 0x36, 0x1e, 0x56, 0x5e, 0x29, 0xcb, 0xd1, 0x33, 0x4b, 0xa2, 0x0c, + 0x07, 0x50, 0xca, 0x79, 0x1b, 0x8e, 0xdd, 0x6e, 0xb9, 0x25, 0xe6, 0xc5, 0xca, 0x17, 0x85, 0x71, 0xde, 0xeb, 0x61, 0x31, 0x96, 0xeb, 0xa0, 0x47, 0x60, 0x8c, 0xff, 0xac, 0x3a, 0x64, 0xd3, 0xd8, - 0x11, 0x87, 0x12, 0x33, 0x9e, 0x5e, 0x97, 0xca, 0x71, 0xa4, 0x16, 0x8b, 0xdf, 0xe6, 0xba, 0x6d, - 0xe2, 0xac, 0xe3, 0x65, 0x91, 0xe3, 0x99, 0xc7, 0x6f, 0xf3, 0x0b, 0x71, 0x08, 0x47, 0x3f, 0xa4, + 0x11, 0x87, 0x12, 0x53, 0x9e, 0x5e, 0x97, 0xca, 0x71, 0xa4, 0x16, 0x8b, 0xdf, 0xe6, 0xba, 0x6d, + 0xe2, 0xac, 0xe3, 0x65, 0x91, 0xe3, 0x99, 0xc7, 0x6f, 0xf3, 0x0b, 0x71, 0x08, 0x47, 0x3f, 0xa2, 0xc0, 0x84, 0x43, 0x5e, 0x69, 0x1b, 0x0e, 0x15, 0x61, 0x34, 0xa3, 0xe9, 0x8a, 0x18, 0x47, 0xb8, - 0xb7, 0xb8, 0x4a, 0x73, 0x38, 0x82, 0x94, 0x33, 0xc8, 0xc0, 0x95, 0x23, 0x0a, 0xc4, 0xb1, 0x1e, - 0xd0, 0xa9, 0x72, 0x8d, 0x86, 0x65, 0x58, 0x8d, 0x79, 0xb3, 0xe1, 0x8a, 0x44, 0x56, 0xdc, 0xe6, + 0xb7, 0xb8, 0x4a, 0x73, 0x38, 0x82, 0x94, 0x33, 0xc8, 0xc0, 0x94, 0x23, 0x0a, 0xc4, 0xb1, 0x1e, + 0xd0, 0xa9, 0x72, 0x8d, 0x86, 0x65, 0x58, 0x8d, 0x79, 0xb3, 0xe1, 0x8a, 0x44, 0x56, 0x5c, 0xe7, 0x18, 0x16, 0x63, 0xb9, 0x0e, 0x7a, 0x14, 0xc6, 0xdb, 0x2e, 0x65, 0x7b, 0x4d, 0xc2, 0xe7, 0x77, - 0x24, 0xf4, 0x75, 0x59, 0x97, 0x01, 0x38, 0x5a, 0x0f, 0x3d, 0x0e, 0x13, 0x7e, 0x81, 0x98, 0x65, - 0xe0, 0xd9, 0x54, 0xd8, 0xdd, 0x64, 0x04, 0x82, 0x63, 0x35, 0x67, 0xe7, 0xe1, 0x74, 0xca, 0x30, - 0x0f, 0xc5, 0x5b, 0xff, 0xb7, 0x02, 0x67, 0xb8, 0xd4, 0xe5, 0x67, 0x87, 0xf6, 0x73, 0x5b, 0xa4, - 0xa7, 0x89, 0x50, 0x8e, 0x35, 0x4d, 0xc4, 0xd7, 0x21, 0x1d, 0x86, 0xfa, 0xd3, 0x05, 0x78, 0xf3, - 0x81, 0xdf, 0x25, 0xfa, 0x47, 0x0a, 0x8c, 0x92, 0x1d, 0xcf, 0xd1, 0x82, 0xa7, 0xfe, 0x74, 0x93, - 0x6e, 0x1e, 0x0b, 0x13, 0x98, 0x5b, 0x0c, 0x09, 0xf1, 0x8d, 0x1b, 0xa8, 0x52, 0x12, 0x04, 0xcb, - 0xfd, 0xa1, 0xdc, 0x96, 0x67, 0x1b, 0x92, 0x9d, 0xe2, 0x04, 0x17, 0x14, 0x90, 0xd9, 0x27, 0x61, - 0x32, 0x8e, 0xf9, 0x50, 0x7b, 0xe5, 0x0b, 0x0a, 0xa4, 0x86, 0xe3, 0x44, 0x25, 0x98, 0xd2, 0xda, - 0x9e, 0x1d, 0xb9, 0xd7, 0x13, 0xa2, 0x24, 0x73, 0xfa, 0x9e, 0x8f, 0x03, 0x71, 0xb2, 0x3e, 0x37, - 0xc0, 0x5a, 0x6d, 0xcd, 0x8c, 0xa2, 0xe1, 0x02, 0x97, 0x30, 0xc0, 0x26, 0xc0, 0x38, 0xad, 0x8d, - 0xfa, 0xcf, 0x14, 0x38, 0x93, 0x6a, 0x96, 0xec, 0xc2, 0x16, 0x97, 0xbe, 0xed, 0x0b, 0xc7, 0xb9, - 0xed, 0xd5, 0x5f, 0x2a, 0xc0, 0x50, 0xd5, 0xb1, 0xe9, 0xd1, 0x76, 0x02, 0x16, 0x38, 0x2d, 0x62, - 0x81, 0xcb, 0x65, 0x5f, 0x10, 0x9d, 0xcd, 0x34, 0xb9, 0x19, 0x31, 0x93, 0xdb, 0x7c, 0x2f, 0x44, - 0x3a, 0xdb, 0xd8, 0x7e, 0x4f, 0x81, 0x51, 0x51, 0xf3, 0x04, 0x8c, 0x6a, 0xdf, 0x11, 0x35, 0xaa, - 0xbd, 0xa7, 0x87, 0x71, 0x65, 0x58, 0xd1, 0x3e, 0xab, 0xc0, 0xb8, 0xa8, 0xb1, 0x42, 0x9a, 0x1b, + 0x24, 0xb4, 0x75, 0x59, 0x97, 0x01, 0x38, 0x5a, 0x0f, 0x3d, 0x0e, 0x13, 0x7e, 0x81, 0x98, 0x65, + 0xe0, 0xd9, 0x54, 0xd8, 0xdb, 0x64, 0x04, 0x82, 0x63, 0x35, 0x67, 0xe7, 0xe1, 0x74, 0xca, 0x30, + 0x0f, 0xc5, 0x5b, 0xff, 0x97, 0x02, 0x67, 0xb8, 0xd4, 0xe5, 0x67, 0x87, 0xf6, 0x73, 0x5b, 0xa4, + 0xa7, 0x89, 0x50, 0x8e, 0x35, 0x4d, 0xc4, 0x37, 0x20, 0x1d, 0x86, 0xfa, 0xb3, 0x05, 0x78, 0xf3, + 0x81, 0xdf, 0x25, 0xfa, 0xff, 0x14, 0x18, 0x25, 0x3b, 0x9e, 0xa3, 0x05, 0xae, 0xfe, 0x74, 0x93, + 0x6e, 0x1e, 0x0b, 0x13, 0x98, 0x5b, 0x0c, 0x09, 0xf1, 0x8d, 0x1b, 0x5c, 0xa5, 0x24, 0x08, 0x96, + 0xfb, 0x43, 0xb9, 0x2d, 0xcf, 0x36, 0x24, 0x1b, 0xc5, 0x09, 0x2e, 0x28, 0x20, 0xb3, 0x4f, 0xc2, + 0x64, 0x1c, 0xf3, 0xa1, 0xf6, 0xca, 0x17, 0x14, 0x48, 0x0d, 0xc7, 0x89, 0x4a, 0x30, 0xa5, 0xb5, + 0x3d, 0x3b, 0xf2, 0xae, 0x27, 0x44, 0x49, 0x66, 0xf4, 0x3d, 0x1f, 0x07, 0xe2, 0x64, 0x7d, 0xae, + 0x80, 0xb5, 0xda, 0x9a, 0x19, 0x45, 0xc3, 0x05, 0x2e, 0xa1, 0x80, 0x4d, 0x80, 0x71, 0x5a, 0x1b, + 0xf5, 0x9f, 0x2a, 0x70, 0x26, 0x55, 0x2d, 0xd9, 0x85, 0x2e, 0x2e, 0x7d, 0xdb, 0x17, 0x8e, 0x73, + 0xdb, 0xab, 0xbf, 0x5c, 0x80, 0xa1, 0xaa, 0x63, 0xd3, 0xa3, 0xed, 0x04, 0x34, 0x70, 0x5a, 0x44, + 0x03, 0x97, 0x4b, 0xbf, 0x20, 0x3a, 0x9b, 0xa9, 0x72, 0x33, 0x62, 0x2a, 0xb7, 0xf9, 0x5e, 0x88, + 0x74, 0xd6, 0xb1, 0x7d, 0x59, 0x81, 0x51, 0x51, 0xf3, 0x04, 0x94, 0x6a, 0xdf, 0x15, 0x55, 0xaa, + 0xbd, 0xa7, 0x87, 0x71, 0x65, 0x68, 0xd1, 0x3e, 0xab, 0xc0, 0xb8, 0xa8, 0xb1, 0x42, 0x9a, 0x1b, 0xc4, 0x41, 0x4b, 0x30, 0xe4, 0xb6, 0xd9, 0x42, 0x8a, 0x01, 0x5d, 0x90, 0xc5, 0x23, 0x67, 0x43, 0xd3, 0x69, 0xf7, 0x6b, 0xbc, 0x8a, 0x94, 0x09, 0x99, 0x17, 0x60, 0xbf, 0x31, 0xdd, 0xfb, 0x8e, - 0x6d, 0x26, 0xc2, 0xd6, 0x63, 0xdb, 0x24, 0x98, 0x41, 0xa8, 0x9e, 0x47, 0xff, 0xfa, 0x3a, 0x1c, - 0xd3, 0xf3, 0x28, 0xd8, 0xc5, 0xbc, 0x5c, 0xfd, 0xe2, 0x40, 0x30, 0xd9, 0x4c, 0xa9, 0xbf, 0x01, - 0x23, 0xba, 0x43, 0x34, 0x8f, 0xd4, 0x17, 0x76, 0xbb, 0xe9, 0x1c, 0x93, 0x03, 0x4a, 0x7e, 0x0b, - 0x1c, 0x36, 0xa6, 0x47, 0xae, 0xec, 0xe0, 0x59, 0x08, 0xa5, 0x93, 0x4c, 0xe7, 0xce, 0x6f, 0x85, - 0x01, 0xfb, 0x8e, 0x15, 0x3c, 0x0e, 0xe9, 0x48, 0x98, 0x0d, 0xe5, 0x16, 0xad, 0x8d, 0x79, 0x23, - 0x39, 0x6d, 0x43, 0x7f, 0x87, 0xb4, 0x0d, 0x26, 0x0c, 0x35, 0xd9, 0x32, 0xf4, 0x94, 0x18, 0x37, - 0xb2, 0xa0, 0xe1, 0x12, 0xf1, 0xdf, 0x2e, 0xf6, 0x49, 0x50, 0xd1, 0xc9, 0xf2, 0x2d, 0x3a, 0xb2, - 0xe8, 0x14, 0x98, 0x79, 0x70, 0x08, 0x47, 0xbb, 0xd1, 0x7c, 0x20, 0x43, 0xf9, 0xed, 0xa4, 0xa2, - 0x7b, 0x52, 0x0a, 0x10, 0x3e, 0xf5, 0x59, 0x39, 0x41, 0xd0, 0x17, 0x14, 0x38, 0x57, 0x4f, 0x4f, - 0xb8, 0x26, 0xae, 0xca, 0x73, 0x3d, 0xcc, 0xce, 0xc8, 0xe1, 0xb6, 0x50, 0x14, 0x13, 0x96, 0x95, - 0xe4, 0x0d, 0x67, 0x75, 0x46, 0xfd, 0xde, 0xfe, 0xe0, 0x6b, 0x12, 0x96, 0x8e, 0x74, 0x3b, 0x94, - 0x92, 0xc7, 0x0e, 0x85, 0xde, 0xe1, 0xa7, 0xff, 0xe2, 0xdb, 0xf5, 0x52, 0x3c, 0xfd, 0xd7, 0x98, - 0x20, 0x1d, 0x49, 0xf9, 0xd5, 0x86, 0xd3, 0xae, 0xa7, 0x99, 0xa4, 0x66, 0x88, 0xcb, 0x41, 0xd7, - 0xd3, 0x9a, 0xad, 0x1c, 0xf9, 0xb7, 0x78, 0xa0, 0x86, 0x24, 0x2a, 0x9c, 0x86, 0x1f, 0x7d, 0x37, - 0x0b, 0xbe, 0xa7, 0x99, 0xec, 0xf2, 0x94, 0xe7, 0x20, 0x0d, 0x89, 0x1f, 0xde, 0xdd, 0x5d, 0x84, - 0xd6, 0x4b, 0xc7, 0x87, 0x33, 0x29, 0xa1, 0x57, 0xe1, 0x0c, 0x3d, 0x00, 0xe7, 0x75, 0xcf, 0xd8, - 0x36, 0xbc, 0xdd, 0xb0, 0x0b, 0x87, 0x4f, 0xba, 0xc5, 0xb4, 0xed, 0xe5, 0x34, 0x64, 0x38, 0x9d, - 0x86, 0xfa, 0x37, 0x0a, 0xa0, 0xe4, 0x5e, 0x47, 0x26, 0x0c, 0xd7, 0xfd, 0xc8, 0x09, 0xca, 0x91, - 0x64, 0xd5, 0x09, 0x8e, 0x90, 0x20, 0xe0, 0x42, 0x40, 0x01, 0xd9, 0x30, 0x72, 0x67, 0xcb, 0xf0, - 0x88, 0x69, 0xb8, 0xde, 0x11, 0x25, 0xf1, 0x09, 0x62, 0xc2, 0x3f, 0xe7, 0x23, 0xc6, 0x21, 0x0d, - 0xf5, 0x53, 0xfd, 0x30, 0x1c, 0x24, 0xd3, 0x3c, 0xd8, 0xcd, 0xba, 0x0d, 0x48, 0x97, 0x02, 0x02, - 0xf6, 0x62, 0x33, 0x65, 0x32, 0x50, 0x29, 0x81, 0x0c, 0xa7, 0x10, 0x40, 0xaf, 0xc2, 0xb4, 0x61, - 0x6d, 0x3a, 0x5a, 0x10, 0xee, 0xb0, 0xe4, 0x1b, 0xca, 0x72, 0x10, 0x66, 0x5a, 0x74, 0x25, 0x05, - 0x1d, 0x4e, 0x25, 0x82, 0x08, 0x0c, 0xf1, 0x7c, 0xc6, 0xfe, 0xad, 0x4b, 0x2e, 0x5b, 0x3d, 0x97, - 0x35, 0x43, 0xf6, 0xee, 0xcb, 0x9e, 0x3e, 0x6e, 0x1e, 0x9c, 0x96, 0xff, 0xef, 0x5f, 0x48, 0x89, - 0x7d, 0x5f, 0xca, 0x4f, 0x2f, 0xbc, 0xdb, 0xe2, 0xc1, 0x69, 0xa3, 0x85, 0x38, 0x4e, 0x50, 0xfd, - 0x1d, 0x05, 0x06, 0x78, 0x0c, 0xb0, 0xe3, 0x17, 0x35, 0xdf, 0x1f, 0x11, 0x35, 0x9f, 0xc8, 0x33, - 0x48, 0xd6, 0xd5, 0xcc, 0x8c, 0xfc, 0xbf, 0xad, 0xc0, 0x08, 0xab, 0x71, 0x02, 0xb2, 0xdf, 0x4b, - 0x51, 0xd9, 0xef, 0xb1, 0xdc, 0xa3, 0xc9, 0x90, 0xfc, 0x7e, 0xa7, 0x4f, 0x8c, 0x85, 0x89, 0x56, - 0x15, 0x38, 0x2d, 0x1e, 0xc6, 0x2e, 0x1b, 0x9b, 0x84, 0x6e, 0xf1, 0xb2, 0xb6, 0xeb, 0x8a, 0xa7, - 0x1a, 0x3c, 0xe8, 0x4c, 0x12, 0x8c, 0xd3, 0xda, 0xa0, 0x5f, 0x51, 0xa8, 0x10, 0xe3, 0x39, 0x86, - 0xde, 0xd3, 0x65, 0x70, 0xd0, 0xb7, 0xb9, 0x15, 0x8e, 0x8c, 0xeb, 0xa6, 0xeb, 0xa1, 0x34, 0xc3, - 0x4a, 0xef, 0xed, 0x15, 0x8b, 0x29, 0x36, 0xe3, 0x30, 0x2b, 0xb5, 0xeb, 0x7d, 0xf4, 0x2f, 0x3a, - 0x56, 0x61, 0xda, 0x98, 0xdf, 0x63, 0x74, 0x03, 0x06, 0x5c, 0xdd, 0x6e, 0xf9, 0x4f, 0xab, 0xef, - 0x4b, 0xb3, 0x0d, 0xc6, 0xcd, 0x82, 0xc1, 0x04, 0xd7, 0x68, 0x4b, 0xcc, 0x11, 0xcc, 0xbe, 0x0c, - 0x63, 0x72, 0xcf, 0x53, 0x74, 0xdf, 0x72, 0xd4, 0x63, 0xee, 0x90, 0xce, 0x61, 0xb2, 0xae, 0xfc, - 0x47, 0x7d, 0x30, 0x88, 0x49, 0x43, 0xa4, 0x4c, 0x3b, 0x40, 0xe7, 0x34, 0xfc, 0xf4, 0xbf, 0x85, - 0xfc, 0xef, 0xf0, 0xe4, 0xa4, 0x38, 0x2f, 0xd8, 0x96, 0x34, 0x07, 0x72, 0x06, 0x60, 0x64, 0x05, - 0x89, 0xa4, 0xfa, 0xf2, 0x3b, 0xd4, 0xf2, 0x81, 0x75, 0x93, 0x3a, 0x0a, 0xfd, 0xa0, 0x02, 0x48, - 0xd3, 0x75, 0xe2, 0xba, 0x98, 0xb8, 0x74, 0xee, 0xb9, 0xb0, 0xca, 0xb9, 0x6c, 0xbe, 0xa8, 0xd8, - 0x71, 0x6c, 0xa1, 0xd8, 0x96, 0x00, 0xb9, 0x38, 0x85, 0x78, 0x2f, 0xe9, 0xac, 0x7e, 0x5f, 0x81, - 0xb1, 0x48, 0xb6, 0xb0, 0x66, 0x68, 0x4b, 0xcf, 0xef, 0xb2, 0xe3, 0xbf, 0xfe, 0xba, 0xd0, 0xa1, - 0x12, 0xb7, 0xcf, 0xdf, 0x0a, 0xd2, 0x5d, 0x1c, 0x4d, 0x62, 0x31, 0xf5, 0xd3, 0x0a, 0x9c, 0xf5, - 0x07, 0x14, 0x8d, 0x6b, 0x8e, 0xae, 0xc2, 0xb0, 0xd6, 0x32, 0x98, 0xa1, 0x57, 0x36, 0x95, 0xcf, - 0x57, 0x2b, 0xac, 0x0c, 0x07, 0xd0, 0x48, 0x8e, 0xe5, 0xc2, 0x81, 0x39, 0x96, 0xdf, 0x22, 0x65, - 0x8d, 0x1e, 0x08, 0x65, 0x97, 0x80, 0x30, 0x77, 0xe6, 0x54, 0xdf, 0x05, 0x23, 0xb5, 0xda, 0x0d, - 0xbe, 0xa4, 0x87, 0xb8, 0xf1, 0x51, 0x3f, 0xd1, 0x07, 0xe3, 0x22, 0x41, 0x83, 0xc1, 0x4c, 0x40, - 0x27, 0x70, 0xce, 0xad, 0xc1, 0x88, 0x1b, 0xdc, 0x61, 0x14, 0xb2, 0xf9, 0x54, 0x70, 0x2d, 0x11, - 0xcf, 0xb2, 0x17, 0x5e, 0x5f, 0x84, 0x88, 0xd0, 0x4d, 0x18, 0x7c, 0x85, 0xf2, 0x5c, 0xff, 0x5b, - 0xed, 0x8a, 0xf5, 0x05, 0x1f, 0x22, 0x63, 0xd7, 0x2e, 0x16, 0x28, 0x90, 0xcb, 0x9e, 0x27, 0x32, - 0x21, 0xb0, 0x97, 0x50, 0x9d, 0x91, 0x99, 0x0d, 0x72, 0xc6, 0x8f, 0x89, 0x57, 0x8e, 0xec, 0x17, - 0x0e, 0x08, 0xb1, 0x14, 0xa1, 0x91, 0x16, 0x6f, 0x90, 0x14, 0xa1, 0x91, 0x3e, 0x67, 0x1c, 0xd7, - 0x8f, 0xc1, 0x99, 0xd4, 0xc9, 0x38, 0x58, 0xc4, 0x56, 0xff, 0x79, 0x01, 0xfa, 0x6b, 0x84, 0xd4, - 0x4f, 0x60, 0x67, 0xbe, 0x14, 0x91, 0xc0, 0xbe, 0x35, 0x77, 0x92, 0xd2, 0x2c, 0x4b, 0xdf, 0x66, - 0xcc, 0xd2, 0xf7, 0x64, 0x6e, 0x0a, 0x9d, 0xcd, 0x7c, 0x7f, 0x55, 0x80, 0x21, 0x5a, 0xad, 0xbc, - 0x5a, 0x43, 0x4d, 0x69, 0x2b, 0x17, 0xf2, 0x8b, 0xcf, 0x02, 0xdd, 0x41, 0x9b, 0x98, 0x7e, 0x39, - 0x86, 0x48, 0xfc, 0x22, 0x06, 0x59, 0x39, 0x02, 0x72, 0x22, 0xf6, 0x04, 0x23, 0xea, 0xe7, 0x95, - 0xc1, 0x01, 0x21, 0x74, 0x47, 0xd2, 0x49, 0xfb, 0xf3, 0xfb, 0xee, 0xa6, 0x13, 0xed, 0xa0, 0x9e, - 0xaa, 0x9f, 0x50, 0xe0, 0x54, 0xac, 0x55, 0x17, 0x4a, 0xe3, 0xb1, 0x30, 0x40, 0xf5, 0x6b, 0x0a, - 0xfd, 0xc6, 0x52, 0x46, 0xd0, 0x45, 0x8f, 0xc2, 0x04, 0x60, 0x85, 0x8e, 0x09, 0xc0, 0x2e, 0x42, - 0x3f, 0x95, 0x8c, 0x64, 0x9f, 0x1d, 0x2a, 0x30, 0x61, 0x56, 0x8a, 0x74, 0x98, 0xd0, 0xa5, 0x14, - 0xee, 0x64, 0x53, 0xf0, 0xce, 0xae, 0x58, 0x71, 0x70, 0xdd, 0x58, 0x8a, 0xa0, 0xc0, 0x31, 0x94, - 0xea, 0x6f, 0x29, 0x30, 0x4c, 0x87, 0x79, 0x02, 0xcc, 0xf1, 0xdb, 0xa3, 0xcc, 0xf1, 0xdd, 0x79, - 0xf7, 0x54, 0x06, 0x4f, 0xfc, 0xeb, 0x02, 0xb0, 0x0c, 0xc6, 0xc2, 0xbf, 0x48, 0xf2, 0x1c, 0x52, - 0x32, 0x7c, 0x9e, 0xae, 0x08, 0xc7, 0xa3, 0x98, 0x51, 0x5a, 0x72, 0x3e, 0x7a, 0x5b, 0xc4, 0xb7, - 0x28, 0x22, 0x5c, 0xa4, 0xf8, 0x17, 0xdd, 0x85, 0x71, 0x77, 0xcb, 0xb6, 0xbd, 0x72, 0xf8, 0xf1, - 0xe4, 0xbe, 0x80, 0x60, 0x8f, 0xc5, 0xfd, 0xa1, 0xf0, 0xab, 0xdc, 0x9a, 0x8c, 0x1b, 0x47, 0x49, - 0xa1, 0x39, 0x80, 0x0d, 0xd3, 0xd6, 0x6f, 0x73, 0xd7, 0x26, 0xfe, 0x38, 0x98, 0x39, 0x4f, 0x2c, - 0x04, 0xa5, 0x58, 0xaa, 0xd1, 0x93, 0x17, 0xd7, 0xd7, 0x14, 0x3e, 0xd3, 0x87, 0xf8, 0x46, 0x4f, - 0xd0, 0x11, 0xee, 0xad, 0x30, 0xe8, 0x30, 0x1d, 0x41, 0xac, 0x58, 0xf0, 0xf1, 0x71, 0xcd, 0x01, - 0x0b, 0x28, 0xdd, 0x1e, 0x5c, 0xf1, 0xe9, 0x0f, 0x2f, 0x1c, 0x64, 0x75, 0x45, 0xfd, 0x25, 0x31, - 0xcc, 0x20, 0x09, 0x76, 0x0b, 0xc6, 0x99, 0x66, 0x11, 0xcb, 0xbe, 0xfd, 0x8e, 0x2e, 0xbf, 0x11, - 0xb9, 0x69, 0xe8, 0x96, 0x1b, 0x29, 0xc6, 0x51, 0x02, 0xe8, 0x51, 0x18, 0xf7, 0x47, 0xc7, 0xbd, - 0x63, 0x0b, 0xe1, 0x0b, 0xcd, 0xaa, 0x0c, 0xc0, 0xd1, 0x7a, 0xea, 0x67, 0x0a, 0x70, 0x89, 0xf7, - 0x9d, 0x59, 0x5e, 0xca, 0xa4, 0x45, 0xac, 0x3a, 0xb1, 0xf4, 0x5d, 0x26, 0x67, 0xd7, 0xed, 0x06, - 0x7a, 0x15, 0x06, 0xef, 0x10, 0x52, 0x0f, 0xae, 0x30, 0x9e, 0xcb, 0x9f, 0x43, 0x3c, 0x83, 0xc4, - 0x73, 0x0c, 0x3d, 0xbf, 0x0d, 0xe6, 0xff, 0x63, 0x41, 0x92, 0x12, 0x6f, 0x39, 0xf6, 0x46, 0x20, - 0x0e, 0x1e, 0x3d, 0xf1, 0x2a, 0x43, 0xcf, 0x89, 0xf3, 0xff, 0xb1, 0x20, 0xa9, 0x56, 0xe1, 0xbe, - 0x2e, 0x9a, 0x1e, 0x46, 0xec, 0x3f, 0x08, 0x23, 0x1f, 0xfd, 0x61, 0x30, 0xfe, 0x89, 0x02, 0xf7, - 0x4b, 0x28, 0x17, 0x77, 0xa8, 0x26, 0x52, 0xd2, 0x5a, 0x9a, 0x4e, 0x75, 0x7d, 0x16, 0xa3, 0xee, - 0x50, 0x59, 0x7b, 0x3f, 0xa1, 0xc0, 0x10, 0xf7, 0xc8, 0xf3, 0xd9, 0xef, 0x4b, 0x3d, 0x4e, 0x79, - 0x66, 0x97, 0xfc, 0x6c, 0x66, 0xfe, 0xd8, 0xf8, 0x6f, 0x17, 0xfb, 0xf4, 0xd5, 0x7f, 0x3d, 0x00, - 0xdf, 0xdc, 0x3d, 0x22, 0xf4, 0x35, 0x45, 0xce, 0x36, 0xce, 0x6d, 0xe4, 0xcd, 0xe3, 0xed, 0x7c, - 0x60, 0x0d, 0x12, 0x06, 0x86, 0xe7, 0x12, 0x09, 0xc9, 0x8f, 0xc8, 0xd0, 0x14, 0x0e, 0x0c, 0xfd, - 0x53, 0x05, 0xc6, 0xe8, 0xb1, 0x14, 0x30, 0x17, 0xbe, 0x4c, 0xad, 0x63, 0x1e, 0xe9, 0xaa, 0x44, - 0x32, 0x16, 0xd7, 0x4a, 0x06, 0xe1, 0x48, 0xdf, 0xd0, 0x7a, 0xf4, 0xfa, 0x8f, 0xab, 0x88, 0x97, - 0xd3, 0xe4, 0x92, 0xc3, 0xa4, 0xfb, 0x9f, 0x35, 0x61, 0x22, 0x3a, 0xf3, 0xc7, 0x69, 0x26, 0x9b, - 0x7d, 0x0a, 0xa6, 0x12, 0xa3, 0x3f, 0x94, 0x41, 0xe6, 0x67, 0x07, 0xa1, 0x28, 0x4d, 0x75, 0x5a, - 0x78, 0x1a, 0xf4, 0x39, 0x05, 0x46, 0x35, 0xcb, 0x12, 0x8e, 0x4d, 0xfe, 0xfe, 0xad, 0xf7, 0xb8, - 0xaa, 0x69, 0xa4, 0xe6, 0xe6, 0x43, 0x32, 0x31, 0xcf, 0x1d, 0x09, 0x82, 0xe5, 0xde, 0x74, 0xf0, - 0xce, 0x2d, 0x9c, 0x98, 0x77, 0x2e, 0xfa, 0xa0, 0x7f, 0x10, 0xf7, 0xe5, 0x0f, 0x54, 0x71, 0xc0, - 0xdc, 0xb0, 0x73, 0x3d, 0xc3, 0x2a, 0xf9, 0xfd, 0x0a, 0x3b, 0x64, 0xc3, 0x28, 0x42, 0xe2, 0x4c, - 0xca, 0xe5, 0x64, 0x79, 0x60, 0x88, 0xa2, 0xe0, 0xec, 0x0e, 0x8b, 0x70, 0x94, 0x3c, 0xfa, 0x11, - 0x05, 0xc6, 0xee, 0xda, 0x96, 0x66, 0x0a, 0x3f, 0x6b, 0x71, 0x4d, 0xf3, 0xfe, 0xe3, 0x99, 0x97, - 0x80, 0x0c, 0xf7, 0xad, 0x94, 0x4b, 0x70, 0xa4, 0x1b, 0xb3, 0x4f, 0xc2, 0x64, 0x7c, 0x8b, 0x1d, - 0xea, 0x73, 0x59, 0x83, 0x07, 0x0f, 0xd1, 0x9d, 0x2e, 0x4f, 0x36, 0xf5, 0xd7, 0xfb, 0x23, 0x27, - 0x65, 0xe6, 0xea, 0x77, 0x61, 0x0a, 0x7f, 0x2d, 0xf6, 0xad, 0x72, 0x0e, 0x6c, 0x1c, 0xd7, 0x7e, - 0x3c, 0xda, 0x0f, 0xb6, 0xef, 0xe4, 0x3e, 0xd8, 0xd7, 0xdb, 0x17, 0xd3, 0xf3, 0xce, 0x5c, 0xe0, - 0xaa, 0xbe, 0x58, 0xb0, 0x30, 0x17, 0x13, 0x8b, 0xc3, 0x69, 0xb8, 0x86, 0x1f, 0x4d, 0x5a, 0x92, - 0xd8, 0x9e, 0xe5, 0xc5, 0xd8, 0x87, 0xab, 0xcb, 0x91, 0xb3, 0x60, 0xcd, 0x6e, 0xd9, 0xa6, 0xdd, - 0xd8, 0x9d, 0xbf, 0xa3, 0x39, 0x04, 0xdb, 0x6d, 0x4f, 0x60, 0xeb, 0x56, 0xfe, 0xfb, 0xc8, 0x00, - 0x5c, 0x91, 0xd0, 0xa5, 0x06, 0xdd, 0x3c, 0xcc, 0x53, 0x84, 0xd7, 0xe2, 0xb1, 0x31, 0x0b, 0xf9, - 0x7d, 0x48, 0x0f, 0xea, 0x57, 0xbe, 0x58, 0x99, 0x68, 0x4f, 0x01, 0x68, 0x6a, 0x3b, 0x22, 0x50, - 0x9b, 0x38, 0x0c, 0xea, 0xc7, 0xd2, 0xc1, 0x95, 0x80, 0x0c, 0xef, 0xde, 0xf3, 0xbe, 0x19, 0x33, - 0x04, 0x1c, 0x91, 0x80, 0x27, 0x8d, 0xa8, 0xe7, 0x60, 0xa0, 0xb3, 0x4d, 0x38, 0x15, 0xeb, 0xf9, - 0xb1, 0x5e, 0x23, 0xfe, 0xde, 0x90, 0xaf, 0xff, 0x8a, 0x08, 0x72, 0xbf, 0xa8, 0xc0, 0x79, 0x92, - 0x25, 0x4f, 0x0a, 0x65, 0xf8, 0xf9, 0xe3, 0x92, 0x57, 0x45, 0x3e, 0xad, 0x2c, 0x30, 0xce, 0xee, - 0x19, 0xda, 0x05, 0x70, 0x83, 0x6f, 0xba, 0x97, 0xb0, 0x14, 0xa9, 0x4c, 0x82, 0x9b, 0x59, 0xc2, - 0xdf, 0x58, 0x22, 0x86, 0x7e, 0x42, 0x81, 0x69, 0x33, 0x85, 0xc3, 0x09, 0x8e, 0x59, 0x3b, 0x86, - 0xb3, 0x85, 0x3b, 0xa0, 0xa4, 0x41, 0x70, 0x6a, 0x57, 0xd0, 0x4f, 0x65, 0x46, 0x10, 0xe6, 0x82, - 0xc7, 0xda, 0x71, 0x7c, 0x83, 0x39, 0x82, 0x09, 0x7f, 0x46, 0x01, 0x54, 0x4f, 0xe8, 0xd6, 0xc2, - 0xf7, 0xf0, 0x99, 0x23, 0xb7, 0x20, 0x70, 0x0f, 0xa2, 0x64, 0x39, 0x4e, 0xe9, 0x04, 0x5b, 0x67, - 0x2f, 0x85, 0xe7, 0x8b, 0x17, 0xbe, 0xbd, 0xae, 0x73, 0xda, 0x71, 0xc2, 0xd7, 0x39, 0x0d, 0x82, - 0x53, 0xbb, 0xa2, 0xfe, 0xe6, 0x08, 0x37, 0xf5, 0x32, 0x17, 0x8f, 0x97, 0x60, 0x70, 0x43, 0xd3, - 0x6f, 0x8b, 0xeb, 0xd5, 0x9c, 0x4e, 0x47, 0x0b, 0x0c, 0x03, 0x37, 0xb2, 0xf0, 0xff, 0xb1, 0xc0, - 0x8a, 0x5e, 0x80, 0xbe, 0xba, 0xe5, 0x3f, 0xa0, 0x7f, 0x4f, 0x0f, 0xd7, 0x07, 0x61, 0x18, 0x92, - 0xf2, 0x6a, 0x0d, 0x53, 0xa4, 0xc8, 0x82, 0x61, 0x4b, 0x58, 0x46, 0x85, 0xf1, 0xea, 0xe9, 0xbc, - 0x04, 0x02, 0x0b, 0x6b, 0x60, 0xd7, 0xf5, 0x4b, 0x70, 0x40, 0x83, 0xd2, 0x8b, 0x5d, 0x5f, 0xe6, - 0xa6, 0x17, 0x5c, 0xf8, 0xc8, 0xf1, 0x59, 0xe3, 0x97, 0x3e, 0x04, 0x06, 0x3d, 0xcd, 0xb0, 0x3c, - 0xff, 0x31, 0xfc, 0x13, 0x79, 0xa9, 0xad, 0x51, 0x2c, 0xa1, 0x01, 0x94, 0xfd, 0x74, 0xb1, 0x40, - 0x8e, 0x36, 0x60, 0x90, 0x3f, 0x88, 0x17, 0x9f, 0x50, 0xee, 0xeb, 0x33, 0xfe, 0xc6, 0x9e, 0x6f, - 0x03, 0xfe, 0x3f, 0x16, 0x98, 0xd1, 0xcb, 0x30, 0xec, 0xfa, 0xde, 0x66, 0xc3, 0xbd, 0x4d, 0x5d, - 0xe0, 0x6a, 0x26, 0x9e, 0xf7, 0x0a, 0x1f, 0xb3, 0x00, 0x3f, 0xda, 0x80, 0x21, 0x43, 0x68, 0x4c, - 0x23, 0xf9, 0xb7, 0x9d, 0xaf, 0x0d, 0x31, 0x6d, 0xc3, 0x57, 0x84, 0x7c, 0xc4, 0x59, 0x2e, 0x25, - 0xf0, 0x75, 0x74, 0x29, 0x41, 0xaf, 0x00, 0x10, 0xdf, 0x42, 0xee, 0xce, 0x8c, 0xe6, 0xdf, 0x32, - 0x92, 0x9d, 0xdd, 0x17, 0x88, 0x82, 0x22, 0x17, 0x4b, 0x44, 0xd0, 0xab, 0xb2, 0x49, 0x6e, 0xac, - 0xb7, 0x18, 0x1f, 0xc9, 0xa8, 0x35, 0xe1, 0xcd, 0x5c, 0x60, 0x0d, 0x92, 0x2c, 0x65, 0xea, 0xef, - 0x01, 0x40, 0x78, 0x6b, 0x8b, 0x36, 0x61, 0xd8, 0xa7, 0xd2, 0x4b, 0x94, 0x9d, 0xeb, 0x02, 0xcc, - 0xb7, 0x97, 0xff, 0x0b, 0x07, 0xb8, 0x51, 0x29, 0x2d, 0xda, 0x53, 0x98, 0x00, 0xb7, 0xbb, 0x48, - 0x4f, 0xaf, 0x00, 0xe8, 0x61, 0x0c, 0xd7, 0xbe, 0xfc, 0x6b, 0x15, 0xc4, 0x77, 0x0d, 0xd7, 0x4a, - 0x0a, 0x01, 0x2b, 0x11, 0xc9, 0x70, 0x3a, 0xef, 0xcf, 0xe5, 0x74, 0xfe, 0x04, 0x9c, 0x12, 0x4e, - 0x7e, 0x15, 0x76, 0x89, 0xe8, 0xed, 0x8a, 0x77, 0x99, 0xcc, 0xfd, 0xb3, 0x14, 0x05, 0xe1, 0x78, - 0x5d, 0xf4, 0xaf, 0x14, 0x18, 0xd6, 0x85, 0x80, 0x26, 0x78, 0xdb, 0x72, 0x6f, 0x77, 0xf6, 0x73, - 0xbe, 0xbc, 0xc7, 0x05, 0xf9, 0x67, 0x7d, 0xae, 0xea, 0x17, 0x1f, 0x91, 0x18, 0x1f, 0xf4, 0x1a, - 0xfd, 0xae, 0x02, 0xa3, 0x9a, 0x69, 0xda, 0xba, 0xe6, 0xb1, 0xe8, 0x7f, 0xfc, 0xc1, 0xe8, 0xad, - 0x1e, 0x47, 0x31, 0x1f, 0x62, 0xe4, 0x03, 0x79, 0x5f, 0x60, 0x09, 0x08, 0x21, 0x47, 0x34, 0x16, - 0xb9, 0xfb, 0xe8, 0x1f, 0x2b, 0x70, 0x3f, 0x7f, 0xa5, 0x5b, 0xa2, 0x32, 0xd7, 0xa6, 0xa1, 0x6b, - 0x1e, 0xe1, 0x21, 0x49, 0xfd, 0x17, 0x62, 0xdc, 0x45, 0x7e, 0xf8, 0xd0, 0x2e, 0xf2, 0x57, 0xf7, - 0xf7, 0x8a, 0xf7, 0x97, 0xba, 0xc0, 0x8d, 0xbb, 0xea, 0x01, 0xba, 0x0b, 0xe3, 0xa6, 0x1c, 0x5e, - 0x5c, 0x30, 0xf9, 0x5c, 0xb7, 0xab, 0x91, 0x38, 0xe5, 0xdc, 0xc0, 0x10, 0x29, 0xc2, 0x51, 0x52, - 0xb3, 0xb7, 0x61, 0x3c, 0xb2, 0xd1, 0x8e, 0xd5, 0x2e, 0x6d, 0xc1, 0x64, 0x7c, 0x3f, 0x1c, 0xab, - 0x9e, 0x77, 0x13, 0x46, 0x02, 0x61, 0x01, 0x5d, 0x92, 0x08, 0x85, 0xa2, 0xd7, 0x4d, 0xb2, 0xcb, - 0xa9, 0x16, 0x23, 0x36, 0x14, 0x7e, 0x69, 0xca, 0x82, 0x33, 0x0a, 0x84, 0xea, 0x1f, 0x88, 0x4b, - 0xd3, 0x35, 0xd2, 0x6c, 0x99, 0x9a, 0x47, 0xde, 0xf8, 0x6e, 0x46, 0xea, 0x7f, 0x54, 0xf8, 0x79, - 0xc3, 0x45, 0x1b, 0xa4, 0xc1, 0x68, 0x93, 0xe7, 0xb6, 0x63, 0xa1, 0x1c, 0x95, 0xfc, 0x41, 0x24, - 0x57, 0x42, 0x34, 0x58, 0xc6, 0x89, 0xee, 0xc0, 0x88, 0x2f, 0x0c, 0xfa, 0x96, 0x9a, 0xa5, 0xde, - 0x84, 0xb3, 0x40, 0xee, 0x0c, 0x8e, 0x56, 0xbf, 0xc4, 0xc5, 0x21, 0x2d, 0x55, 0x03, 0x94, 0x6c, - 0x83, 0x1e, 0x08, 0x9f, 0xab, 0x29, 0xd1, 0x6c, 0x34, 0x89, 0x27, 0x6b, 0xbe, 0x91, 0xb5, 0x90, - 0x65, 0x64, 0x55, 0x7f, 0xa3, 0x00, 0xd3, 0x42, 0xf5, 0x9c, 0xd7, 0x75, 0xbb, 0x6d, 0x79, 0xe1, - 0x2b, 0x62, 0xfe, 0x34, 0x5f, 0x10, 0x61, 0xe2, 0x24, 0x7f, 0xb7, 0x8f, 0x05, 0x04, 0xdd, 0xe2, - 0xd6, 0x4f, 0xab, 0xbe, 0x66, 0xdf, 0x26, 0x56, 0xc8, 0x25, 0xe4, 0x18, 0x18, 0x8b, 0x69, 0x15, - 0x70, 0x7a, 0x3b, 0xb4, 0x0d, 0xa8, 0xa9, 0xed, 0xc4, 0xb1, 0xe5, 0xcb, 0x71, 0xc6, 0xf4, 0xc5, - 0x95, 0x04, 0x36, 0x9c, 0x42, 0x81, 0x1e, 0xa4, 0x54, 0x92, 0x6b, 0x79, 0xa4, 0xce, 0x87, 0xe8, - 0xfb, 0x6c, 0xb0, 0x83, 0x74, 0x3e, 0x0a, 0xc2, 0xf1, 0xba, 0xea, 0xc7, 0x06, 0xe1, 0x7c, 0x74, - 0x12, 0xe9, 0x17, 0xea, 0xbf, 0x9e, 0x7f, 0xca, 0x7f, 0x1a, 0xc6, 0x27, 0xf2, 0x81, 0xf8, 0xd3, - 0xb0, 0x19, 0xd9, 0x27, 0x48, 0x34, 0x8a, 0x3c, 0x13, 0xfb, 0x3a, 0x3c, 0x85, 0xcf, 0x78, 0xfb, - 0xdc, 0x77, 0xac, 0x4f, 0xfe, 0x3f, 0xa9, 0xc0, 0x6c, 0xb4, 0x78, 0xc9, 0xb0, 0x0c, 0x77, 0x4b, - 0xa4, 0x35, 0x39, 0xfc, 0xcb, 0x34, 0x96, 0x18, 0x79, 0x39, 0x13, 0x23, 0xee, 0x40, 0x0d, 0x7d, - 0x9f, 0x02, 0x17, 0x62, 0xf3, 0x12, 0x49, 0xb2, 0x72, 0xf8, 0x47, 0x6a, 0x2c, 0x76, 0xcb, 0x72, - 0x36, 0x4a, 0xdc, 0x89, 0x5e, 0xa7, 0x80, 0x9e, 0x83, 0xaf, 0x97, 0x80, 0x9e, 0xbf, 0x50, 0x80, - 0x01, 0xe6, 0x16, 0xf5, 0xc6, 0x78, 0x4f, 0xc4, 0xba, 0x9a, 0xe9, 0xce, 0xda, 0x88, 0xb9, 0xb3, - 0x3e, 0x95, 0x9f, 0x44, 0x67, 0x7f, 0xd6, 0xf7, 0xc1, 0x59, 0x56, 0x6d, 0xbe, 0xce, 0x0c, 0x6d, - 0x2e, 0xa9, 0xcf, 0xd7, 0xeb, 0x4c, 0xbd, 0x3d, 0xf8, 0x8e, 0xec, 0x12, 0xf4, 0xb5, 0x1d, 0x33, - 0x1e, 0x21, 0x76, 0x1d, 0x2f, 0x63, 0x5a, 0xae, 0x7e, 0x52, 0x81, 0x49, 0x86, 0x5b, 0x62, 0x31, - 0x68, 0x1b, 0x86, 0x1d, 0xc1, 0x66, 0xc4, 0xda, 0x2c, 0xe7, 0x1e, 0x5a, 0x0a, 0xeb, 0xe2, 0x1a, - 0x9b, 0xff, 0x0b, 0x07, 0xb4, 0xd4, 0x3f, 0x1d, 0x80, 0x99, 0xac, 0x46, 0xe8, 0x87, 0x14, 0x38, - 0xab, 0x87, 0x12, 0xe7, 0x7c, 0xdb, 0xdb, 0xb2, 0x1d, 0x3f, 0x61, 0x42, 0x6e, 0x73, 0x48, 0x69, - 0x3e, 0xe8, 0x15, 0xcb, 0x12, 0x52, 0x4a, 0xa5, 0x80, 0x33, 0x28, 0xa3, 0x57, 0x01, 0x5c, 0x77, - 0xeb, 0x26, 0xd9, 0x6d, 0x69, 0x86, 0xef, 0x98, 0x75, 0x33, 0xff, 0x36, 0xa8, 0xdd, 0x10, 0xa8, - 0x82, 0x4e, 0x71, 0x5b, 0x75, 0x58, 0x2e, 0x91, 0x43, 0x1f, 0x55, 0x60, 0xdc, 0x96, 0xc3, 0xb1, - 0xf4, 0xe2, 0xab, 0x9f, 0x1a, 0xd7, 0x85, 0x4b, 0xda, 0x51, 0x50, 0x94, 0x24, 0x5d, 0x96, 0x29, - 0x37, 0x7e, 0xb2, 0x09, 0xde, 0xb7, 0x92, 0x4f, 0x06, 0xca, 0x38, 0x26, 0xb9, 0xd6, 0x9e, 0x04, - 0x27, 0xc9, 0xb3, 0x4e, 0x11, 0x4f, 0xaf, 0x2f, 0x5a, 0xba, 0xb3, 0xcb, 0x02, 0x00, 0xd0, 0x4e, - 0x0d, 0xe6, 0xef, 0xd4, 0xe2, 0x5a, 0xa9, 0x1c, 0x41, 0x16, 0xed, 0x54, 0x12, 0x9c, 0x24, 0xaf, - 0x7e, 0xa4, 0x00, 0xe7, 0xd8, 0x32, 0xdf, 0x6c, 0x6f, 0x10, 0xee, 0x5b, 0xf5, 0x0f, 0x2e, 0x7e, - 0xce, 0x6f, 0x2b, 0x30, 0xc2, 0xe6, 0xe0, 0x0d, 0xf2, 0x04, 0x93, 0xf5, 0x35, 0xc3, 0x7f, 0xf9, - 0xb7, 0x14, 0x98, 0x4a, 0xa4, 0x88, 0xea, 0xea, 0x01, 0xdf, 0x89, 0xb9, 0xd6, 0xbe, 0x25, 0x4c, - 0x4b, 0xd9, 0x17, 0xc6, 0xad, 0x88, 0xa7, 0xa4, 0x54, 0x9f, 0x83, 0xf1, 0x88, 0xfb, 0xb2, 0x14, - 0xe1, 0x31, 0x2d, 0x36, 0xa5, 0x1c, 0xc0, 0xb1, 0xd0, 0x29, 0xf4, 0x64, 0xb8, 0xe5, 0x93, 0x9c, - 0xed, 0x1f, 0xce, 0x96, 0x47, 0x62, 0xcb, 0xfb, 0xd7, 0x38, 0x2c, 0x6e, 0xa4, 0xdb, 0xcb, 0x35, - 0x0e, 0x0b, 0x40, 0x29, 0xe2, 0xbc, 0xf2, 0xff, 0xb1, 0xc0, 0x8a, 0x9e, 0x8e, 0x46, 0x71, 0x5d, - 0x0d, 0x75, 0xbb, 0xe9, 0x78, 0xec, 0x55, 0xb6, 0x25, 0x13, 0xb5, 0x11, 0xe6, 0x17, 0x41, 0xfc, - 0x2c, 0xcb, 0x95, 0xbc, 0xa6, 0xbc, 0x5a, 0xe3, 0xe1, 0xfd, 0x82, 0x0b, 0xa0, 0xa8, 0xc5, 0xbb, - 0xff, 0x24, 0x2c, 0xde, 0x0e, 0x8c, 0x6e, 0x19, 0x1b, 0xc4, 0xb1, 0xb8, 0x18, 0x33, 0x90, 0x5f, - 0x42, 0xbb, 0x11, 0xa2, 0xe1, 0x66, 0x00, 0xa9, 0x00, 0xcb, 0x44, 0x90, 0x13, 0x89, 0x45, 0x3d, - 0x98, 0x5f, 0x2a, 0x09, 0x4d, 0xd3, 0xe1, 0x38, 0x33, 0xe2, 0x50, 0x5b, 0x00, 0x56, 0x10, 0xa0, - 0xb5, 0x97, 0x8b, 0xa1, 0x30, 0xcc, 0x2b, 0x17, 0x3a, 0xc2, 0xdf, 0x58, 0xa2, 0x40, 0xe7, 0xb5, - 0x19, 0xe6, 0x2b, 0x10, 0x66, 0xc6, 0xa7, 0x7a, 0xcc, 0x19, 0x21, 0xcc, 0x2b, 0x61, 0x01, 0x96, - 0x89, 0xd0, 0x31, 0x36, 0x83, 0x28, 0xfd, 0xc2, 0x8c, 0x98, 0x6b, 0x8c, 0x61, 0xac, 0x7f, 0x3e, - 0xc6, 0xf0, 0x37, 0x96, 0x28, 0xa0, 0x97, 0xa5, 0xfb, 0x43, 0xc8, 0x6f, 0xa4, 0xea, 0xea, 0xee, - 0xf0, 0x9d, 0xa1, 0xad, 0x66, 0x94, 0x7d, 0xa7, 0x17, 0x24, 0x3b, 0x0d, 0xcb, 0x5e, 0x40, 0x79, - 0x47, 0xc2, 0x6e, 0x13, 0x3e, 0x9a, 0x18, 0xeb, 0xf8, 0x68, 0xa2, 0x44, 0xa5, 0x33, 0xe9, 0xe1, - 0x21, 0x63, 0x08, 0xe3, 0xe1, 0x25, 0x48, 0x2d, 0x0e, 0xc4, 0xc9, 0xfa, 0x9c, 0xe1, 0x93, 0x3a, - 0x6b, 0x3b, 0x21, 0x33, 0x7c, 0x5e, 0x86, 0x03, 0x28, 0xda, 0x86, 0x31, 0x57, 0x7a, 0x81, 0x31, - 0x73, 0xaa, 0xd7, 0x2b, 0x44, 0xf1, 0xfa, 0x82, 0xb9, 0x3a, 0xca, 0x25, 0x38, 0x42, 0x27, 0x7a, - 0xbf, 0x35, 0x79, 0xb2, 0xf7, 0x5b, 0xa8, 0x1d, 0x75, 0xae, 0x9e, 0x3a, 0x92, 0x30, 0x2d, 0x07, - 0x3a, 0x5f, 0xd3, 0xa5, 0x25, 0x3b, 0x2d, 0xdb, 0x6d, 0x3b, 0x84, 0x65, 0xcb, 0x61, 0xcb, 0x83, - 0xc2, 0xa5, 0x5d, 0x8c, 0x03, 0x71, 0xb2, 0x3e, 0xfa, 0x1e, 0x05, 0x26, 0xdd, 0x5d, 0xd7, 0x23, - 0x4d, 0x7a, 0x6c, 0xd9, 0x16, 0xb1, 0x3c, 0x77, 0xe6, 0x74, 0xfe, 0xb0, 0xdf, 0xb5, 0x18, 0x2e, - 0x7e, 0xec, 0xc4, 0x4b, 0x71, 0x82, 0x26, 0xdd, 0x39, 0x72, 0xa0, 0x97, 0x99, 0xe9, 0xfc, 0x3b, - 0x47, 0x0e, 0x22, 0xc3, 0x77, 0x8e, 0x5c, 0x82, 0x23, 0x74, 0xd0, 0xa3, 0x30, 0xee, 0xfa, 0x29, - 0xca, 0xd9, 0x0c, 0x9e, 0x09, 0x63, 0x71, 0xd6, 0x64, 0x00, 0x8e, 0xd6, 0x43, 0x1f, 0x82, 0x31, - 0xf9, 0xec, 0x9c, 0x39, 0x7b, 0xd4, 0x51, 0xe9, 0x79, 0xcf, 0x65, 0x50, 0x84, 0x20, 0xc2, 0x70, - 0x56, 0x7a, 0x1b, 0x28, 0x7f, 0xdf, 0xe7, 0xd8, 0x10, 0xb8, 0x3e, 0x9b, 0x5a, 0x03, 0x67, 0xb4, - 0x44, 0x3f, 0x96, 0x7e, 0x5d, 0x3e, 0xc3, 0xb6, 0x74, 0xf5, 0x48, 0xae, 0xcb, 0x9f, 0x33, 0xbc, - 0xad, 0x5b, 0x2d, 0x1e, 0x38, 0xec, 0x90, 0x37, 0xe7, 0xea, 0x1f, 0x29, 0x00, 0x81, 0xc1, 0xe4, - 0x24, 0xae, 0x2a, 0xea, 0x11, 0x1b, 0xd2, 0x42, 0x4f, 0x06, 0x9e, 0xcc, 0xa4, 0x23, 0xea, 0x97, - 0x15, 0x98, 0x08, 0xab, 0x9d, 0x80, 0x6a, 0xa4, 0x47, 0x55, 0xa3, 0x27, 0x7b, 0x1b, 0x57, 0x86, - 0x7e, 0xf4, 0xbf, 0x0a, 0xf2, 0xa8, 0x98, 0xf4, 0xbb, 0x1d, 0xb9, 0xfa, 0xa7, 0xa4, 0x6f, 0xf4, - 0x72, 0xf5, 0x2f, 0x07, 0xd7, 0x08, 0xc7, 0x9b, 0xe2, 0x0a, 0xf0, 0x9d, 0x11, 0xf9, 0xb3, 0x87, - 0xb0, 0x36, 0x81, 0xb0, 0xe9, 0x93, 0xe6, 0x13, 0x70, 0x90, 0x30, 0xfa, 0x8a, 0x7c, 0x3c, 0xf5, - 0x90, 0x28, 0x24, 0x32, 0xe0, 0xce, 0x4e, 0x17, 0xbf, 0x30, 0x05, 0xa3, 0x92, 0x6d, 0x31, 0xe6, - 0xc8, 0xa0, 0x9c, 0x84, 0x23, 0x83, 0x07, 0xa3, 0x7a, 0x90, 0x29, 0xd2, 0x9f, 0xf6, 0x1e, 0x69, - 0x06, 0xc7, 0x62, 0x98, 0x83, 0xd2, 0xc5, 0x32, 0x19, 0x2a, 0xbc, 0x05, 0x7b, 0xac, 0xef, 0x08, - 0xdc, 0x4b, 0x3a, 0xed, 0xab, 0x47, 0x00, 0x7c, 0xf9, 0x9f, 0xd4, 0x45, 0x00, 0xf6, 0xc0, 0x75, - 0xba, 0xe2, 0xde, 0x08, 0x60, 0x58, 0xaa, 0x97, 0xbc, 0x18, 0x1f, 0x38, 0xb1, 0x8b, 0x71, 0xba, - 0x0d, 0x4c, 0x3f, 0xa7, 0x7e, 0x4f, 0xee, 0x6a, 0x41, 0x66, 0xfe, 0x70, 0x1b, 0x04, 0x45, 0x2e, - 0x96, 0x88, 0x64, 0xf8, 0xb3, 0x0c, 0xe5, 0xf2, 0x67, 0x69, 0xc3, 0x69, 0x87, 0x78, 0xce, 0x6e, - 0x69, 0x57, 0x67, 0x99, 0x4b, 0x1c, 0x8f, 0x69, 0xf0, 0xc3, 0xf9, 0xe2, 0x21, 0xe2, 0x24, 0x2a, - 0x9c, 0x86, 0x3f, 0x22, 0x00, 0x8f, 0x74, 0x14, 0x80, 0xdf, 0x09, 0xa3, 0x1e, 0xd1, 0xb7, 0x2c, - 0x43, 0xd7, 0xcc, 0x4a, 0x59, 0x84, 0xe7, 0x0e, 0x65, 0xb9, 0x10, 0x84, 0xe5, 0x7a, 0x68, 0x01, - 0xfa, 0xda, 0x46, 0x5d, 0x68, 0x00, 0xdf, 0x12, 0x58, 0xe9, 0x2b, 0xe5, 0x7b, 0x7b, 0xc5, 0x37, - 0x87, 0x0e, 0x22, 0xc1, 0xa8, 0xae, 0xb5, 0x6e, 0x37, 0xae, 0x79, 0xbb, 0x2d, 0xe2, 0xce, 0xad, - 0x57, 0xca, 0x98, 0x36, 0x4e, 0xf3, 0xf5, 0x19, 0x3b, 0x84, 0xaf, 0xcf, 0x67, 0x14, 0x38, 0xad, - 0xc5, 0x2f, 0x18, 0x88, 0x3b, 0x33, 0x9e, 0x9f, 0x5b, 0xa6, 0x5f, 0x5a, 0x2c, 0x5c, 0x10, 0xe3, - 0x3b, 0x3d, 0x9f, 0x24, 0x87, 0xd3, 0xfa, 0x80, 0x1c, 0x40, 0x4d, 0xa3, 0x11, 0xa4, 0xb7, 0x17, - 0xab, 0x3e, 0x91, 0xcf, 0x6e, 0xb3, 0x92, 0xc0, 0x84, 0x53, 0xb0, 0xa3, 0x3b, 0x30, 0x2a, 0x09, - 0x49, 0x42, 0x93, 0x29, 0x1f, 0xc5, 0x3d, 0x08, 0xd7, 0x76, 0xe5, 0x3b, 0x0e, 0x99, 0x52, 0x70, - 0xc9, 0x29, 0x99, 0x19, 0xc4, 0x45, 0x1f, 0x1b, 0xf5, 0x64, 0xfe, 0x4b, 0xce, 0x74, 0x8c, 0xb8, - 0x03, 0x35, 0x16, 0x85, 0x90, 0x82, 0x25, 0xdd, 0x7c, 0x66, 0x2a, 0x7f, 0x18, 0x95, 0xe5, 0x28, - 0x2a, 0xbe, 0x35, 0x63, 0x85, 0x38, 0x4e, 0x10, 0x2d, 0x01, 0x22, 0xdc, 0x94, 0x1e, 0x2a, 0x67, - 0xee, 0x0c, 0x62, 0xf7, 0xef, 0x6c, 0x49, 0x17, 0x13, 0x50, 0x9c, 0xd2, 0x02, 0x79, 0x11, 0x5b, - 0x49, 0x0f, 0x5a, 0x4e, 0x3c, 0x25, 0x4e, 0x47, 0x8b, 0xc9, 0x77, 0x2b, 0x89, 0x1c, 0xc9, 0x5c, - 0xb9, 0xb9, 0xd1, 0x7b, 0x8e, 0x64, 0x41, 0xbe, 0x9b, 0x4c, 0xc9, 0x9f, 0x57, 0xe0, 0x5c, 0x33, - 0x3d, 0x17, 0x24, 0xd3, 0x79, 0x72, 0x5e, 0x5c, 0x65, 0xa4, 0x97, 0x64, 0x66, 0x8c, 0xac, 0xdc, - 0x93, 0x38, 0xab, 0x23, 0xea, 0x1f, 0x2a, 0xc2, 0x14, 0x7d, 0x82, 0xee, 0x48, 0xc7, 0x7d, 0x4f, - 0xac, 0x3e, 0x07, 0x33, 0x35, 0x3f, 0x86, 0x68, 0x3d, 0x96, 0x2a, 0xe0, 0x3d, 0x30, 0xce, 0xaf, - 0x82, 0x56, 0xb4, 0xd6, 0x6a, 0x78, 0x6f, 0x10, 0x44, 0x9b, 0x28, 0xc9, 0x40, 0x1c, 0xad, 0xab, - 0x7e, 0x55, 0x81, 0x73, 0x51, 0xcc, 0xb6, 0x63, 0xdc, 0xed, 0x1d, 0x31, 0xfa, 0xb8, 0x02, 0xa3, - 0xb7, 0x83, 0x5b, 0x2a, 0x5f, 0x74, 0xcb, 0xf5, 0x64, 0xc3, 0xef, 0x15, 0x71, 0xa4, 0x6b, 0xaf, - 0x64, 0xfe, 0xc9, 0x10, 0xe8, 0x62, 0x99, 0xb4, 0xfa, 0x5f, 0x15, 0x48, 0x98, 0x0f, 0xd0, 0x06, - 0x0c, 0x51, 0x22, 0xe5, 0xd5, 0x9a, 0xd8, 0x13, 0xef, 0xc9, 0x27, 0x55, 0x32, 0x14, 0xfc, 0x52, - 0x44, 0xfc, 0xc0, 0x3e, 0x62, 0xb4, 0xcd, 0x9f, 0xf7, 0xfb, 0x79, 0x84, 0xc4, 0xf6, 0xc8, 0x25, - 0xb6, 0xcb, 0xf9, 0x88, 0xb8, 0x5a, 0x2f, 0x97, 0xe0, 0x08, 0x1d, 0x75, 0x19, 0x20, 0x34, 0xf9, - 0xf4, 0xec, 0xde, 0xf7, 0xef, 0x4e, 0xc3, 0x99, 0x9e, 0x1f, 0x23, 0x7e, 0x54, 0x81, 0xb3, 0x64, - 0xdb, 0xd0, 0xbd, 0xf9, 0x4d, 0x8f, 0x38, 0xb7, 0x6e, 0xad, 0xac, 0x6d, 0x39, 0xc4, 0xdd, 0xb2, - 0xcd, 0x7a, 0x37, 0xce, 0x8c, 0x29, 0x9e, 0x57, 0xcc, 0x34, 0xb1, 0x98, 0x8a, 0x11, 0x67, 0x50, - 0x62, 0xe6, 0xae, 0x6d, 0x6e, 0x08, 0xc0, 0x54, 0xe7, 0x6a, 0x3b, 0xae, 0x27, 0xc2, 0x02, 0x72, - 0x73, 0x57, 0x1c, 0x88, 0x93, 0xf5, 0xe3, 0x48, 0x58, 0x92, 0x3c, 0x26, 0xb7, 0x2b, 0x49, 0x24, - 0x0c, 0x88, 0x93, 0xf5, 0x65, 0x24, 0x7c, 0xa5, 0xe8, 0xa1, 0x38, 0x90, 0x44, 0x12, 0x00, 0x71, - 0xb2, 0x3e, 0xaa, 0xc3, 0x45, 0x87, 0xe8, 0x76, 0xb3, 0x49, 0xac, 0x3a, 0x9b, 0x94, 0x15, 0xcd, - 0x69, 0x18, 0xd6, 0x92, 0xa3, 0xb1, 0x8a, 0xec, 0xf6, 0x40, 0x61, 0xf9, 0x6c, 0x2f, 0xe2, 0x0e, - 0xf5, 0x70, 0x47, 0x2c, 0xa8, 0x09, 0xa7, 0xda, 0x8c, 0xff, 0x3b, 0x2c, 0x6c, 0xd8, 0xb6, 0x66, - 0x8a, 0x2b, 0x82, 0xc3, 0xae, 0x18, 0x3b, 0xa8, 0xd7, 0xa3, 0xa8, 0x70, 0x1c, 0x37, 0xda, 0xa5, - 0xe2, 0xb9, 0xe8, 0x8e, 0x44, 0x72, 0x38, 0x17, 0x49, 0x21, 0xa2, 0x27, 0xd0, 0xe1, 0x34, 0x1a, - 0xa8, 0x02, 0xa7, 0x3d, 0xcd, 0x69, 0x10, 0xaf, 0x54, 0x5d, 0xaf, 0x12, 0x47, 0xa7, 0x3c, 0xd6, - 0xe4, 0xd2, 0xba, 0xc2, 0x51, 0xad, 0x25, 0xc1, 0x38, 0xad, 0x0d, 0xfa, 0x10, 0xbc, 0x25, 0x3a, - 0xa9, 0xcb, 0xf6, 0x1d, 0xe2, 0x2c, 0xd8, 0x6d, 0xab, 0x1e, 0x45, 0x0e, 0x0c, 0xf9, 0x03, 0xfb, - 0x7b, 0xc5, 0xb7, 0xe0, 0x6e, 0x1a, 0xe0, 0xee, 0xf0, 0x26, 0x3b, 0xb0, 0xde, 0x6a, 0xa5, 0x76, - 0x60, 0x34, 0xab, 0x03, 0x19, 0x0d, 0x70, 0x77, 0x78, 0x11, 0x86, 0xb3, 0x7c, 0x62, 0x78, 0xf6, - 0x65, 0x89, 0xe2, 0x18, 0xa3, 0xc8, 0xbe, 0xdf, 0xb5, 0xd4, 0x1a, 0x38, 0xa3, 0x25, 0x3d, 0x53, - 0xae, 0x66, 0x0d, 0x3f, 0x41, 0x66, 0x9c, 0x91, 0x79, 0xdb, 0xfe, 0x5e, 0xf1, 0x2a, 0xee, 0xb2, - 0x0d, 0xee, 0x1a, 0x7b, 0x4a, 0x57, 0xc2, 0x89, 0x48, 0x74, 0x65, 0x22, 0xab, 0x2b, 0xd9, 0x6d, - 0x70, 0xd7, 0xd8, 0xd1, 0xf7, 0x2a, 0x70, 0x5e, 0x6f, 0xb5, 0x6f, 0x18, 0xae, 0x67, 0x37, 0x1c, - 0xad, 0x59, 0x26, 0xba, 0xb6, 0x7b, 0x43, 0x33, 0x37, 0x97, 0x8d, 0x4d, 0x22, 0x94, 0x8e, 0xc3, - 0x7e, 0x38, 0xec, 0xe1, 0x6d, 0xa9, 0xba, 0x9e, 0x8e, 0x14, 0x67, 0xd3, 0x43, 0x3f, 0xac, 0xc0, - 0xc5, 0x26, 0xeb, 0x62, 0x46, 0x87, 0x26, 0x73, 0x75, 0x88, 0x71, 0xb1, 0x95, 0x0e, 0x78, 0x71, - 0x47, 0xaa, 0x6c, 0x92, 0x78, 0x85, 0xf9, 0x46, 0xc3, 0x21, 0x0d, 0x86, 0x35, 0xe0, 0x2e, 0x53, - 0xf9, 0x27, 0x69, 0x25, 0x0b, 0x29, 0xce, 0xa6, 0x87, 0x5e, 0x86, 0xcb, 0x99, 0xc0, 0x92, 0xdd, - 0xb6, 0x3c, 0x76, 0x09, 0xd3, 0xb7, 0xa0, 0xee, 0xef, 0x15, 0x2f, 0xaf, 0x74, 0xac, 0x89, 0x0f, - 0xc0, 0xc4, 0x42, 0x8b, 0x44, 0xc2, 0x00, 0x9c, 0x66, 0x92, 0xd8, 0x8b, 0xb9, 0xb2, 0xed, 0x1e, - 0xe1, 0xdb, 0xff, 0x2f, 0x47, 0xdf, 0xfe, 0x4f, 0xb3, 0x5e, 0xbd, 0xef, 0xe8, 0x7a, 0xf5, 0x8d, - 0x07, 0xff, 0xdd, 0x3c, 0x04, 0xf9, 0x8c, 0x02, 0xe2, 0xfd, 0x26, 0xba, 0x18, 0x71, 0x3b, 0x1a, - 0x8e, 0xb9, 0x1c, 0xf9, 0x49, 0x6f, 0x0b, 0xa9, 0x49, 0x6f, 0xdf, 0x2a, 0xc5, 0x58, 0x1e, 0x09, - 0x95, 0x25, 0x8e, 0x39, 0x0c, 0xb2, 0x8c, 0x1e, 0x84, 0x91, 0x40, 0xa9, 0x16, 0xc6, 0x4e, 0x96, - 0x70, 0x26, 0xd4, 0xbe, 0x43, 0xb8, 0xfa, 0xfb, 0x0a, 0x40, 0x98, 0xcb, 0x19, 0xdd, 0x07, 0x03, - 0xba, 0xa9, 0xb9, 0xbe, 0x4b, 0x51, 0x70, 0x63, 0xc0, 0x6e, 0x22, 0x31, 0x87, 0x1d, 0xfc, 0x18, - 0x01, 0xa9, 0x30, 0xd8, 0x66, 0xd9, 0x2b, 0xc5, 0x03, 0x02, 0xe6, 0x02, 0xb3, 0xce, 0x4a, 0xb0, - 0x80, 0xa0, 0x75, 0x18, 0x6a, 0x1a, 0x16, 0x7b, 0xeb, 0xd1, 0x9f, 0xeb, 0xad, 0x07, 0xd3, 0x07, - 0x56, 0x38, 0x0a, 0xec, 0xe3, 0x52, 0x7f, 0x51, 0x81, 0x53, 0xd1, 0xa0, 0xd7, 0x2c, 0xe4, 0x8d, - 0x88, 0x85, 0x2a, 0x62, 0xed, 0xb3, 0xa6, 0x22, 0xc6, 0x23, 0xf6, 0x61, 0xd1, 0xdb, 0xe9, 0x1e, - 0x6e, 0x1f, 0xd2, 0x63, 0x6f, 0x1f, 0x70, 0x11, 0xf0, 0xa3, 0xa7, 0x61, 0x90, 0x2b, 0xda, 0x54, - 0x8e, 0x4f, 0x89, 0x6d, 0x75, 0x33, 0x7f, 0x3a, 0x89, 0x3c, 0x11, 0x71, 0xe4, 0xa4, 0x9a, 0x85, - 0x8e, 0x49, 0x35, 0x31, 0xf4, 0xe9, 0x8e, 0xd1, 0x8b, 0x27, 0x52, 0x09, 0x57, 0xb8, 0x27, 0x52, - 0x09, 0x57, 0x30, 0x45, 0x86, 0xbc, 0x88, 0x8b, 0x4e, 0x7f, 0x7e, 0x13, 0x10, 0x9f, 0x00, 0xc9, - 0x51, 0x67, 0xa2, 0xa3, 0x93, 0x8e, 0x1f, 0x48, 0x7f, 0x20, 0xff, 0xe3, 0x20, 0x31, 0xe5, 0xdd, - 0x04, 0xd2, 0xf7, 0x3f, 0xa4, 0xc1, 0xcc, 0x0f, 0x69, 0x13, 0x86, 0xc4, 0xa7, 0x20, 0x14, 0x82, - 0xf7, 0xf4, 0x90, 0x3a, 0x5e, 0x4a, 0x52, 0xc5, 0x0b, 0xb0, 0x8f, 0x9c, 0x6a, 0x99, 0x4d, 0x6d, - 0xc7, 0x68, 0xb6, 0x9b, 0x4c, 0x0b, 0x18, 0x90, 0xab, 0xb2, 0x62, 0xec, 0xc3, 0x59, 0x55, 0xfe, - 0xa6, 0x8a, 0x49, 0xed, 0x72, 0x55, 0x5e, 0x8c, 0x7d, 0x38, 0x7a, 0x01, 0x86, 0x9b, 0xda, 0x4e, - 0xad, 0xed, 0x34, 0x88, 0x70, 0xd0, 0xc9, 0x36, 0x0a, 0xb5, 0x3d, 0xc3, 0x9c, 0x33, 0x2c, 0xcf, - 0xf5, 0x9c, 0xb9, 0x8a, 0xe5, 0xdd, 0x72, 0x6a, 0x1e, 0x73, 0x00, 0x62, 0xbb, 0x6e, 0x45, 0x60, - 0xc1, 0x01, 0x3e, 0x64, 0xc2, 0x44, 0x53, 0xdb, 0x59, 0xb7, 0x34, 0x9e, 0x23, 0x41, 0x48, 0xd9, - 0x79, 0x28, 0x30, 0xab, 0xdc, 0x4a, 0x04, 0x17, 0x8e, 0xe1, 0x4e, 0x71, 0x06, 0x1d, 0x3b, 0x2e, - 0x67, 0xd0, 0xf9, 0x20, 0x4a, 0x01, 0x37, 0xe9, 0x9f, 0x4f, 0x0d, 0xff, 0xd7, 0x31, 0x02, 0xc1, - 0x4b, 0x41, 0x04, 0x82, 0x89, 0xfc, 0xde, 0x8b, 0x1d, 0xa2, 0x0f, 0xb4, 0x61, 0xb4, 0xae, 0x79, - 0x1a, 0x2f, 0x75, 0x67, 0x4e, 0xe5, 0xbf, 0x9d, 0x2e, 0x07, 0x68, 0x42, 0x96, 0x14, 0x96, 0xb9, - 0x58, 0xa6, 0x83, 0x6e, 0xc1, 0x19, 0xfa, 0xb1, 0x9a, 0xc4, 0x0b, 0xab, 0x30, 0x9b, 0xd9, 0x24, - 0xfb, 0x7e, 0xd8, 0x2b, 0xb5, 0x9b, 0x69, 0x15, 0x70, 0x7a, 0xbb, 0x30, 0x54, 0xed, 0x54, 0x7a, - 0xa8, 0x5a, 0xf4, 0xa9, 0x34, 0xb7, 0x1b, 0xc4, 0xe6, 0xf4, 0xbd, 0xf9, 0x79, 0x43, 0x6e, 0xe7, - 0x9b, 0x7f, 0xa1, 0xc0, 0x8c, 0xd8, 0x65, 0xc2, 0x55, 0xc6, 0x24, 0xce, 0x8a, 0x66, 0x69, 0x0d, - 0xe2, 0x08, 0x3b, 0xf9, 0x5a, 0x0f, 0xfc, 0x21, 0x81, 0x33, 0x08, 0x0d, 0x71, 0xff, 0xfe, 0x5e, - 0xf1, 0xca, 0x41, 0xb5, 0x70, 0x66, 0xdf, 0x90, 0x03, 0x43, 0xee, 0xae, 0xab, 0x7b, 0xa6, 0x2b, - 0x64, 0xd0, 0xeb, 0x3d, 0x70, 0xd6, 0x1a, 0xc7, 0xc4, 0x59, 0x6b, 0x98, 0x1a, 0x91, 0x97, 0x62, - 0x9f, 0x10, 0xfa, 0x41, 0x05, 0xa6, 0xc4, 0xe5, 0x99, 0x14, 0x7a, 0xe7, 0x4c, 0xfe, 0x77, 0x32, - 0xa5, 0x38, 0x32, 0xdf, 0x3d, 0x86, 0x59, 0x93, 0x12, 0x50, 0x9c, 0xa4, 0x4e, 0x0f, 0xd5, 0x96, - 0x63, 0xd8, 0x8e, 0xe1, 0xed, 0x32, 0x47, 0xa4, 0x01, 0x3f, 0x40, 0x3d, 0x2f, 0xc3, 0x01, 0x14, - 0xd5, 0x60, 0x82, 0x5b, 0x6d, 0x6a, 0x9e, 0xa3, 0x79, 0xa4, 0xb1, 0x2b, 0xbc, 0x85, 0x1e, 0x64, - 0x29, 0x84, 0x23, 0x90, 0x7b, 0x7b, 0xc5, 0x33, 0x62, 0x6d, 0xa2, 0x00, 0x1c, 0x43, 0x81, 0x3e, - 0x10, 0x73, 0xde, 0x9a, 0xc9, 0x9f, 0x5e, 0x90, 0xaf, 0xc5, 0x61, 0x5c, 0xb8, 0x7a, 0x8d, 0x26, - 0xd7, 0x43, 0x8a, 0x97, 0xd9, 0xc7, 0x61, 0x4c, 0xde, 0x35, 0x87, 0x0a, 0x62, 0xe7, 0x01, 0x4a, - 0x0e, 0xf6, 0xb8, 0xe3, 0xfc, 0xa8, 0x3f, 0xa9, 0xc0, 0x64, 0x5c, 0x76, 0x41, 0x5b, 0x30, 0x24, - 0x18, 0x99, 0xa0, 0x3a, 0x9f, 0xd7, 0x6b, 0xd9, 0x24, 0xe2, 0x79, 0x30, 0x17, 0x85, 0x45, 0x11, - 0xf6, 0xd1, 0xcb, 0x2f, 0x12, 0x0a, 0x1d, 0x5e, 0x24, 0x3c, 0x01, 0x67, 0xd3, 0x59, 0x1a, 0x55, - 0x24, 0x34, 0xaa, 0x5f, 0x09, 0xa3, 0x75, 0xa0, 0x48, 0x30, 0xa5, 0x0b, 0x73, 0x98, 0xfa, 0x41, - 0x88, 0xa7, 0x36, 0x43, 0x2f, 0xc3, 0x88, 0xeb, 0x6e, 0x71, 0x3f, 0x33, 0x31, 0xc8, 0x7c, 0x57, - 0x3d, 0x7e, 0x9a, 0x19, 0xae, 0xfb, 0x04, 0x3f, 0x71, 0x88, 0x7e, 0xe1, 0xf9, 0x2f, 0x7d, 0xf5, - 0xf2, 0x9b, 0xfe, 0xe0, 0xab, 0x97, 0xdf, 0xf4, 0x95, 0xaf, 0x5e, 0x7e, 0xd3, 0x87, 0xf7, 0x2f, - 0x2b, 0x5f, 0xda, 0xbf, 0xac, 0xfc, 0xc1, 0xfe, 0x65, 0xe5, 0x2b, 0xfb, 0x97, 0x95, 0xbf, 0xdc, - 0xbf, 0xac, 0xfc, 0xc0, 0xbf, 0xbf, 0xfc, 0xa6, 0x17, 0x1e, 0x0e, 0xa9, 0x5f, 0xf3, 0x89, 0x86, - 0xff, 0xb4, 0x6e, 0x37, 0xa8, 0xd2, 0xea, 0xfa, 0x5a, 0x2b, 0xa3, 0xfe, 0x7f, 0x02, 0x00, 0x00, - 0xff, 0xff, 0x03, 0x88, 0x36, 0xbe, 0x42, 0x25, 0x01, 0x00, + 0x6d, 0x26, 0xc2, 0xd6, 0x63, 0xdb, 0x24, 0x98, 0x41, 0xe8, 0x3d, 0x8f, 0xfe, 0xf5, 0xef, 0x70, + 0xec, 0x9e, 0x47, 0xc1, 0x2e, 0xe6, 0xe5, 0xea, 0x17, 0x07, 0x82, 0xc9, 0x66, 0x97, 0xfa, 0x1b, + 0x30, 0xa2, 0x3b, 0x44, 0xf3, 0x48, 0x7d, 0x61, 0xb7, 0x9b, 0xce, 0x31, 0x39, 0xa0, 0xe4, 0xb7, + 0xc0, 0x61, 0x63, 0x7a, 0xe4, 0xca, 0x06, 0x9e, 0x85, 0x50, 0x3a, 0xc9, 0x34, 0xee, 0xfc, 0x76, + 0x18, 0xb0, 0xef, 0x58, 0x81, 0x73, 0x48, 0x47, 0xc2, 0x6c, 0x28, 0xb7, 0x68, 0x6d, 0xcc, 0x1b, + 0xc9, 0x69, 0x1b, 0xfa, 0x3b, 0xa4, 0x6d, 0x30, 0x61, 0xa8, 0xc9, 0x96, 0xa1, 0xa7, 0xc4, 0xb8, + 0x91, 0x05, 0x0d, 0x97, 0x88, 0xff, 0x76, 0xb1, 0x4f, 0x82, 0x8a, 0x4e, 0x96, 0xaf, 0xd1, 0x91, + 0x45, 0xa7, 0x40, 0xcd, 0x83, 0x43, 0x38, 0xda, 0x8d, 0xe6, 0x03, 0x19, 0xca, 0xaf, 0x27, 0x15, + 0xdd, 0x93, 0x52, 0x80, 0xf0, 0xa9, 0xcf, 0xca, 0x09, 0x82, 0xbe, 0xa0, 0xc0, 0xb9, 0x7a, 0x7a, + 0xc2, 0x35, 0xf1, 0x54, 0x9e, 0xcb, 0x31, 0x3b, 0x23, 0x87, 0xdb, 0x42, 0x51, 0x4c, 0x58, 0x56, + 0x92, 0x37, 0x9c, 0xd5, 0x19, 0xf5, 0xfb, 0xfb, 0x83, 0xaf, 0x49, 0x68, 0x3a, 0xd2, 0xf5, 0x50, + 0x4a, 0x1e, 0x3d, 0x14, 0x7a, 0x87, 0x9f, 0xfe, 0x8b, 0x6f, 0xd7, 0x4b, 0xf1, 0xf4, 0x5f, 0x63, + 0x82, 0x74, 0x24, 0xe5, 0x57, 0x1b, 0x4e, 0xbb, 0x9e, 0x66, 0x92, 0x9a, 0x21, 0x1e, 0x07, 0x5d, + 0x4f, 0x6b, 0xb6, 0x72, 0xe4, 0xdf, 0xe2, 0x81, 0x1a, 0x92, 0xa8, 0x70, 0x1a, 0x7e, 0xf4, 0xbd, + 0x2c, 0xf8, 0x9e, 0x66, 0xb2, 0xc7, 0x53, 0x9e, 0x83, 0x34, 0x24, 0x7e, 0x78, 0x73, 0x77, 0x11, + 0x5a, 0x2f, 0x1d, 0x1f, 0xce, 0xa4, 0x84, 0x5e, 0x85, 0x33, 0xf4, 0x00, 0x9c, 0xd7, 0x3d, 0x63, + 0xdb, 0xf0, 0x76, 0xc3, 0x2e, 0x1c, 0x3e, 0xe9, 0x16, 0xbb, 0x6d, 0x2f, 0xa7, 0x21, 0xc3, 0xe9, + 0x34, 0xd4, 0xbf, 0x51, 0x00, 0x25, 0xf7, 0x3a, 0x32, 0x61, 0xb8, 0xee, 0x47, 0x4e, 0x50, 0x8e, + 0x24, 0xab, 0x4e, 0x70, 0x84, 0x04, 0x01, 0x17, 0x02, 0x0a, 0xc8, 0x86, 0x91, 0x3b, 0x5b, 0x86, + 0x47, 0x4c, 0xc3, 0xf5, 0x8e, 0x28, 0x89, 0x4f, 0x10, 0x13, 0xfe, 0x39, 0x1f, 0x31, 0x0e, 0x69, + 0xa8, 0x9f, 0xea, 0x87, 0xe1, 0x20, 0x99, 0xe6, 0xc1, 0x66, 0xd6, 0x6d, 0x40, 0xba, 0x14, 0x10, + 0xb0, 0x17, 0x9d, 0x29, 0x93, 0x81, 0x4a, 0x09, 0x64, 0x38, 0x85, 0x00, 0x7a, 0x15, 0xa6, 0x0d, + 0x6b, 0xd3, 0xd1, 0x82, 0x70, 0x87, 0x25, 0x5f, 0x51, 0x96, 0x83, 0x30, 0xbb, 0x45, 0x57, 0x52, + 0xd0, 0xe1, 0x54, 0x22, 0x88, 0xc0, 0x10, 0xcf, 0x67, 0xec, 0xbf, 0xba, 0xe4, 0xd2, 0xd5, 0x73, + 0x59, 0x33, 0x64, 0xef, 0xbe, 0xec, 0xe9, 0xe3, 0xe6, 0xc1, 0x69, 0xf9, 0xff, 0xfe, 0x83, 0x94, + 0xd8, 0xf7, 0xa5, 0xfc, 0xf4, 0xc2, 0xb7, 0x2d, 0x1e, 0x9c, 0x36, 0x5a, 0x88, 0xe3, 0x04, 0xd5, + 0xdf, 0x55, 0x60, 0x80, 0xc7, 0x00, 0x3b, 0x7e, 0x51, 0xf3, 0xfd, 0x11, 0x51, 0xf3, 0x89, 0x3c, + 0x83, 0x64, 0x5d, 0xcd, 0xcc, 0xc8, 0xff, 0x3b, 0x0a, 0x8c, 0xb0, 0x1a, 0x27, 0x20, 0xfb, 0xbd, + 0x14, 0x95, 0xfd, 0x1e, 0xcb, 0x3d, 0x9a, 0x0c, 0xc9, 0xef, 0x77, 0xfb, 0xc4, 0x58, 0x98, 0x68, + 0x55, 0x81, 0xd3, 0xc2, 0x31, 0x76, 0xd9, 0xd8, 0x24, 0x74, 0x8b, 0x97, 0xb5, 0x5d, 0x57, 0xb8, + 0x6a, 0xf0, 0xa0, 0x33, 0x49, 0x30, 0x4e, 0x6b, 0x83, 0x7e, 0x55, 0xa1, 0x42, 0x8c, 0xe7, 0x18, + 0x7a, 0x4f, 0x8f, 0xc1, 0x41, 0xdf, 0xe6, 0x56, 0x38, 0x32, 0x7e, 0x37, 0x5d, 0x0f, 0xa5, 0x19, + 0x56, 0x7a, 0x6f, 0xaf, 0x58, 0x4c, 0xd1, 0x19, 0x87, 0x59, 0xa9, 0x5d, 0xef, 0xa3, 0x7f, 0xd1, + 0xb1, 0x0a, 0xbb, 0x8d, 0xf9, 0x3d, 0x46, 0x37, 0x60, 0xc0, 0xd5, 0xed, 0x96, 0xef, 0x5a, 0x7d, + 0x5f, 0x9a, 0x6e, 0x30, 0xae, 0x16, 0x0c, 0x26, 0xb8, 0x46, 0x5b, 0x62, 0x8e, 0x60, 0xf6, 0x65, + 0x18, 0x93, 0x7b, 0x9e, 0x72, 0xf7, 0x2d, 0x47, 0x2d, 0xe6, 0x0e, 0x69, 0x1c, 0x26, 0xdf, 0x95, + 0xff, 0xa8, 0x0f, 0x06, 0x31, 0x69, 0x88, 0x94, 0x69, 0x07, 0xdc, 0x39, 0x0d, 0x3f, 0xfd, 0x6f, + 0x21, 0xbf, 0x1f, 0x9e, 0x9c, 0x14, 0xe7, 0x05, 0xdb, 0x92, 0xe6, 0x40, 0xce, 0x00, 0x8c, 0xac, + 0x20, 0x91, 0x54, 0x5f, 0x7e, 0x83, 0x5a, 0x3e, 0xb0, 0x6e, 0x52, 0x47, 0xa1, 0x1f, 0x56, 0x00, + 0x69, 0xba, 0x4e, 0x5c, 0x17, 0x13, 0x97, 0xce, 0x3d, 0x17, 0x56, 0x39, 0x97, 0xcd, 0x17, 0x15, + 0x3b, 0x8e, 0x2d, 0x14, 0xdb, 0x12, 0x20, 0x17, 0xa7, 0x10, 0xef, 0x25, 0x9d, 0xd5, 0xef, 0x2b, + 0x30, 0x16, 0xc9, 0x16, 0xd6, 0x0c, 0x75, 0xe9, 0xf9, 0x4d, 0x76, 0x7c, 0xef, 0xaf, 0x0b, 0x1d, + 0x2a, 0x71, 0xfd, 0xfc, 0xad, 0x20, 0xdd, 0xc5, 0xd1, 0x24, 0x16, 0x53, 0x3f, 0xad, 0xc0, 0x59, + 0x7f, 0x40, 0xd1, 0xb8, 0xe6, 0xe8, 0x2a, 0x0c, 0x6b, 0x2d, 0x83, 0x29, 0x7a, 0x65, 0x55, 0xf9, + 0x7c, 0xb5, 0xc2, 0xca, 0x70, 0x00, 0x8d, 0xe4, 0x58, 0x2e, 0x1c, 0x98, 0x63, 0xf9, 0x2d, 0x52, + 0xd6, 0xe8, 0x81, 0x50, 0x76, 0x09, 0x08, 0x73, 0x63, 0x4e, 0xf5, 0x5d, 0x30, 0x52, 0xab, 0xdd, + 0xe0, 0x4b, 0x7a, 0x88, 0x17, 0x1f, 0xf5, 0x13, 0x7d, 0x30, 0x2e, 0x12, 0x34, 0x18, 0x4c, 0x05, + 0x74, 0x02, 0xe7, 0xdc, 0x1a, 0x8c, 0xb8, 0xc1, 0x1b, 0x46, 0x21, 0x9b, 0x4f, 0x05, 0xcf, 0x12, + 0xf1, 0x2c, 0x7b, 0xe1, 0xf3, 0x45, 0x88, 0x08, 0xdd, 0x84, 0xc1, 0x57, 0x28, 0xcf, 0xf5, 0xbf, + 0xd5, 0xae, 0x58, 0x5f, 0xf0, 0x21, 0x32, 0x76, 0xed, 0x62, 0x81, 0x02, 0xb9, 0xcc, 0x3d, 0x91, + 0x09, 0x81, 0xbd, 0x84, 0xea, 0x8c, 0xcc, 0x6c, 0x90, 0x33, 0x7e, 0x4c, 0x78, 0x39, 0xb2, 0x5f, + 0x38, 0x20, 0xc4, 0x52, 0x84, 0x46, 0x5a, 0xbc, 0x41, 0x52, 0x84, 0x46, 0xfa, 0x9c, 0x71, 0x5c, + 0x3f, 0x06, 0x67, 0x52, 0x27, 0xe3, 0x60, 0x11, 0x5b, 0xfd, 0x67, 0x05, 0xe8, 0xaf, 0x11, 0x52, + 0x3f, 0x81, 0x9d, 0xf9, 0x52, 0x44, 0x02, 0xfb, 0xf6, 0xdc, 0x49, 0x4a, 0xb3, 0x34, 0x7d, 0x9b, + 0x31, 0x4d, 0xdf, 0x93, 0xb9, 0x29, 0x74, 0x56, 0xf3, 0xfd, 0x55, 0x01, 0x86, 0x68, 0xb5, 0xf2, + 0x6a, 0x0d, 0x35, 0xa5, 0xad, 0x5c, 0xc8, 0x2f, 0x3e, 0x0b, 0x74, 0x07, 0x6d, 0x62, 0xfa, 0xe5, + 0x18, 0x22, 0xf1, 0x8b, 0x18, 0x64, 0xe5, 0x08, 0xc8, 0x89, 0xd8, 0x13, 0x8c, 0xa8, 0x9f, 0x57, + 0x06, 0x07, 0x84, 0xd0, 0x1d, 0xe9, 0x4e, 0xda, 0x9f, 0xdf, 0x76, 0x37, 0x9d, 0x68, 0x87, 0xeb, + 0xa9, 0xfa, 0x09, 0x05, 0x4e, 0xc5, 0x5a, 0x75, 0x71, 0x69, 0x3c, 0x16, 0x06, 0xa8, 0x7e, 0x5d, + 0xa1, 0xdf, 0x58, 0xca, 0x08, 0xba, 0xe8, 0x51, 0x98, 0x00, 0xac, 0xd0, 0x31, 0x01, 0xd8, 0x45, + 0xe8, 0xa7, 0x92, 0x91, 0x6c, 0xb3, 0x43, 0x05, 0x26, 0xcc, 0x4a, 0x91, 0x0e, 0x13, 0xba, 0x94, + 0xc2, 0x9d, 0x6c, 0x0a, 0xde, 0xd9, 0x15, 0x2b, 0x0e, 0x9e, 0x1b, 0x4b, 0x11, 0x14, 0x38, 0x86, + 0x52, 0xfd, 0x6d, 0x05, 0x86, 0xe9, 0x30, 0x4f, 0x80, 0x39, 0x7e, 0x67, 0x94, 0x39, 0xbe, 0x3b, + 0xef, 0x9e, 0xca, 0xe0, 0x89, 0x7f, 0x5d, 0x00, 0x96, 0xc1, 0x58, 0xd8, 0x17, 0x49, 0x96, 0x43, + 0x4a, 0x86, 0xcd, 0xd3, 0x15, 0x61, 0x78, 0x14, 0x53, 0x4a, 0x4b, 0xc6, 0x47, 0x6f, 0x8b, 0xd8, + 0x16, 0x45, 0x84, 0x8b, 0x14, 0xfb, 0xa2, 0xbb, 0x30, 0xee, 0x6e, 0xd9, 0xb6, 0x57, 0x0e, 0x3f, + 0x9e, 0xdc, 0x0f, 0x10, 0xcc, 0x59, 0xdc, 0x1f, 0x0a, 0x7f, 0xca, 0xad, 0xc9, 0xb8, 0x71, 0x94, + 0x14, 0x9a, 0x03, 0xd8, 0x30, 0x6d, 0xfd, 0x36, 0x37, 0x6d, 0xe2, 0xce, 0xc1, 0xcc, 0x78, 0x62, + 0x21, 0x28, 0xc5, 0x52, 0x8d, 0x9e, 0xac, 0xb8, 0xbe, 0xae, 0xf0, 0x99, 0x3e, 0xc4, 0x37, 0x7a, + 0x82, 0x86, 0x70, 0x6f, 0x85, 0x41, 0x87, 0xdd, 0x11, 0xc4, 0x8a, 0x05, 0x1f, 0x1f, 0xbf, 0x39, + 0x60, 0x01, 0xa5, 0xdb, 0x83, 0x5f, 0x7c, 0xfa, 0xc3, 0x07, 0x07, 0xf9, 0xba, 0xa2, 0xfe, 0xb2, + 0x18, 0x66, 0x90, 0x04, 0xbb, 0x05, 0xe3, 0xec, 0x66, 0x11, 0xcb, 0xbe, 0xfd, 0x8e, 0x2e, 0xbf, + 0x11, 0xb9, 0x69, 0x68, 0x96, 0x1b, 0x29, 0xc6, 0x51, 0x02, 0xe8, 0x51, 0x18, 0xf7, 0x47, 0xc7, + 0xad, 0x63, 0x0b, 0xa1, 0x87, 0x66, 0x55, 0x06, 0xe0, 0x68, 0x3d, 0xf5, 0x33, 0x05, 0xb8, 0xc4, + 0xfb, 0xce, 0x34, 0x2f, 0x65, 0xd2, 0x22, 0x56, 0x9d, 0x58, 0xfa, 0x2e, 0x93, 0xb3, 0xeb, 0x76, + 0x03, 0xbd, 0x0a, 0x83, 0x77, 0x08, 0xa9, 0x07, 0x4f, 0x18, 0xcf, 0xe5, 0xcf, 0x21, 0x9e, 0x41, + 0xe2, 0x39, 0x86, 0x9e, 0xbf, 0x06, 0xf3, 0xff, 0xb1, 0x20, 0x49, 0x89, 0xb7, 0x1c, 0x7b, 0x23, + 0x10, 0x07, 0x8f, 0x9e, 0x78, 0x95, 0xa1, 0xe7, 0xc4, 0xf9, 0xff, 0x58, 0x90, 0x54, 0xab, 0x70, + 0x5f, 0x17, 0x4d, 0x0f, 0x23, 0xf6, 0x1f, 0x84, 0x91, 0x8f, 0xfe, 0x30, 0x18, 0xff, 0x44, 0x81, + 0xfb, 0x25, 0x94, 0x8b, 0x3b, 0xf4, 0x26, 0x52, 0xd2, 0x5a, 0x9a, 0x4e, 0xef, 0xfa, 0x2c, 0x46, + 0xdd, 0xa1, 0xb2, 0xf6, 0x7e, 0x42, 0x81, 0x21, 0x6e, 0x91, 0xe7, 0xb3, 0xdf, 0x97, 0x7a, 0x9c, + 0xf2, 0xcc, 0x2e, 0xf9, 0xd9, 0xcc, 0xfc, 0xb1, 0xf1, 0xdf, 0x2e, 0xf6, 0xe9, 0xab, 0xff, 0x6a, + 0x00, 0xbe, 0xb5, 0x7b, 0x44, 0xe8, 0xeb, 0x8a, 0x9c, 0x6d, 0x9c, 0xeb, 0xc8, 0x9b, 0xc7, 0xdb, + 0xf9, 0x40, 0x1b, 0x24, 0x14, 0x0c, 0xcf, 0x25, 0x12, 0x92, 0x1f, 0x91, 0xa2, 0x29, 0x1c, 0x18, + 0xfa, 0xc7, 0x0a, 0x8c, 0xd1, 0x63, 0x29, 0x60, 0x2e, 0x7c, 0x99, 0x5a, 0xc7, 0x3c, 0xd2, 0x55, + 0x89, 0x64, 0x2c, 0xae, 0x95, 0x0c, 0xc2, 0x91, 0xbe, 0xa1, 0xf5, 0xe8, 0xf3, 0x1f, 0xbf, 0x22, + 0x5e, 0x4e, 0x93, 0x4b, 0x0e, 0x93, 0xee, 0x7f, 0xd6, 0x84, 0x89, 0xe8, 0xcc, 0x1f, 0xa7, 0x9a, + 0x6c, 0xf6, 0x29, 0x98, 0x4a, 0x8c, 0xfe, 0x50, 0x0a, 0x99, 0x2f, 0x0f, 0x42, 0x51, 0x9a, 0xea, + 0xb4, 0xf0, 0x34, 0xe8, 0x73, 0x0a, 0x8c, 0x6a, 0x96, 0x25, 0x0c, 0x9b, 0xfc, 0xfd, 0x5b, 0xef, + 0x71, 0x55, 0xd3, 0x48, 0xcd, 0xcd, 0x87, 0x64, 0x62, 0x96, 0x3b, 0x12, 0x04, 0xcb, 0xbd, 0xe9, + 0x60, 0x9d, 0x5b, 0x38, 0x31, 0xeb, 0x5c, 0xf4, 0x41, 0xff, 0x20, 0xee, 0xcb, 0x1f, 0xa8, 0xe2, + 0x80, 0xb9, 0x61, 0xe7, 0x7a, 0x86, 0x56, 0xf2, 0x07, 0x15, 0x76, 0xc8, 0x86, 0x51, 0x84, 0xc4, + 0x99, 0x94, 0xcb, 0xc8, 0xf2, 0xc0, 0x10, 0x45, 0xc1, 0xd9, 0x1d, 0x16, 0xe1, 0x28, 0x79, 0xf4, + 0x63, 0x0a, 0x8c, 0xdd, 0xb5, 0x2d, 0xcd, 0x14, 0x76, 0xd6, 0xe2, 0x99, 0xe6, 0xfd, 0xc7, 0x33, + 0x2f, 0x01, 0x19, 0x6e, 0x5b, 0x29, 0x97, 0xe0, 0x48, 0x37, 0x58, 0xc6, 0x7c, 0xe6, 0x5b, 0x3d, + 0x18, 0xca, 0xd3, 0xb2, 0x5f, 0xf5, 0xec, 0x93, 0x30, 0x19, 0xdf, 0x83, 0x87, 0xfa, 0x9e, 0xd6, + 0xe0, 0xc1, 0x43, 0xf4, 0xb7, 0xcb, 0xa3, 0x4f, 0xfd, 0x8d, 0xfe, 0xc8, 0x51, 0x9a, 0xb9, 0x3d, + 0xba, 0xd0, 0x95, 0xbf, 0x16, 0xfb, 0x98, 0x39, 0x8b, 0x36, 0x8e, 0x6b, 0xc3, 0x1e, 0xed, 0x17, + 0xdd, 0x77, 0x72, 0x5f, 0xf4, 0xeb, 0xed, 0x93, 0xea, 0x79, 0x67, 0x2e, 0x70, 0x5d, 0x80, 0x58, + 0xb0, 0x30, 0x59, 0x13, 0x0b, 0xd4, 0x69, 0xb8, 0x86, 0x1f, 0x6e, 0x5a, 0x12, 0xe9, 0x9e, 0xe5, + 0xc5, 0xd8, 0x87, 0xab, 0xcb, 0x91, 0xc3, 0x62, 0xcd, 0x6e, 0xd9, 0xa6, 0xdd, 0xd8, 0x9d, 0xbf, + 0xa3, 0x39, 0x04, 0xdb, 0x6d, 0x4f, 0x60, 0xeb, 0x56, 0x40, 0xfc, 0xc8, 0x00, 0x5c, 0x91, 0xd0, + 0xa5, 0x46, 0xe5, 0x3c, 0x8c, 0xaf, 0xc2, 0x6b, 0xf1, 0xe0, 0x99, 0x85, 0xfc, 0x46, 0xa6, 0x07, + 0xf5, 0x2b, 0x5f, 0x30, 0x4d, 0xb4, 0xa7, 0x00, 0x34, 0xb5, 0x1d, 0x11, 0xc9, 0x4d, 0x9c, 0x16, + 0xf5, 0x63, 0xe9, 0xe0, 0x4a, 0x40, 0x86, 0x77, 0xef, 0x79, 0x5f, 0xcf, 0x19, 0x02, 0x8e, 0x48, + 0x02, 0x94, 0x46, 0xd4, 0x73, 0xb4, 0xd0, 0xd9, 0x26, 0x9c, 0x8a, 0xf5, 0xfc, 0x58, 0xdf, 0x19, + 0xbf, 0x3c, 0xe4, 0x5f, 0x90, 0x45, 0x88, 0xb9, 0x5f, 0x52, 0xe0, 0x3c, 0xc9, 0x12, 0x38, 0xc5, + 0x6d, 0xf9, 0xf9, 0xe3, 0x12, 0x68, 0x45, 0xc2, 0xad, 0x2c, 0x30, 0xce, 0xee, 0x19, 0xda, 0x05, + 0x70, 0x83, 0x6f, 0xba, 0x97, 0xb8, 0x15, 0xa9, 0x4c, 0x82, 0xeb, 0x61, 0xc2, 0xdf, 0x58, 0x22, + 0x86, 0x7e, 0x4a, 0x81, 0x69, 0x33, 0x85, 0xc3, 0x09, 0x8e, 0x59, 0x3b, 0x86, 0xb3, 0x85, 0x5b, + 0xa8, 0xa4, 0x41, 0x70, 0x6a, 0x57, 0xd0, 0xcf, 0x64, 0x86, 0x18, 0xe6, 0x92, 0xc9, 0xda, 0x71, + 0x7c, 0x83, 0x39, 0xa2, 0x0d, 0x7f, 0x46, 0x01, 0x54, 0x4f, 0x5c, 0xbe, 0x85, 0x71, 0xe2, 0x33, + 0x47, 0xae, 0x62, 0xe0, 0x26, 0x46, 0xc9, 0x72, 0x9c, 0xd2, 0x09, 0xb6, 0xce, 0x5e, 0x0a, 0xcf, + 0x17, 0x2e, 0xc0, 0xbd, 0xae, 0x73, 0xda, 0x71, 0xc2, 0xd7, 0x39, 0x0d, 0x82, 0x53, 0xbb, 0xa2, + 0xfe, 0xd6, 0x08, 0xd7, 0x05, 0x33, 0x1b, 0x90, 0x97, 0x60, 0x70, 0x43, 0xd3, 0x6f, 0x8b, 0xf7, + 0xd7, 0x9c, 0x56, 0x49, 0x0b, 0x0c, 0x03, 0xd7, 0xc2, 0xf0, 0xff, 0xb1, 0xc0, 0x8a, 0x5e, 0x80, + 0xbe, 0xba, 0xe5, 0x7b, 0xd8, 0xbf, 0xa7, 0x87, 0xf7, 0x85, 0x30, 0x4e, 0x49, 0x79, 0xb5, 0x86, + 0x29, 0x52, 0x64, 0xc1, 0xb0, 0x25, 0x54, 0xa7, 0x42, 0xbb, 0xf5, 0x74, 0x5e, 0x02, 0x81, 0x0a, + 0x36, 0x50, 0xfc, 0xfa, 0x25, 0x38, 0xa0, 0x41, 0xe9, 0xc5, 0xde, 0x37, 0x73, 0xd3, 0x0b, 0x5e, + 0x84, 0xe4, 0x00, 0xae, 0xf1, 0x57, 0x21, 0x02, 0x83, 0x9e, 0x66, 0x58, 0x9e, 0xef, 0x2d, 0xff, + 0x44, 0x5e, 0x6a, 0x6b, 0x14, 0x4b, 0xa8, 0x21, 0x65, 0x3f, 0x5d, 0x2c, 0x90, 0xa3, 0x0d, 0x18, + 0xe4, 0x1e, 0xf3, 0xe2, 0x13, 0xca, 0xfd, 0xbe, 0xc6, 0x9d, 0xf0, 0xf9, 0x36, 0xe0, 0xff, 0x63, + 0x81, 0x19, 0xbd, 0x0c, 0xc3, 0xae, 0x6f, 0x8e, 0x36, 0xdc, 0xdb, 0xd4, 0x05, 0xb6, 0x68, 0xc2, + 0xff, 0x57, 0x18, 0xa1, 0x05, 0xf8, 0xd1, 0x06, 0x0c, 0x19, 0xe2, 0x4a, 0x35, 0x92, 0x7f, 0xdb, + 0xf9, 0xd7, 0x25, 0x76, 0xdb, 0xf0, 0x6f, 0x4a, 0x3e, 0xe2, 0x2c, 0x9b, 0x13, 0xf8, 0x06, 0xda, + 0x9c, 0xa0, 0x57, 0x00, 0x88, 0xaf, 0x42, 0x77, 0x67, 0x46, 0xf3, 0x6f, 0x19, 0x49, 0x11, 0xef, + 0x0b, 0x44, 0x41, 0x91, 0x8b, 0x25, 0x22, 0xe8, 0x55, 0x59, 0x67, 0x37, 0xd6, 0x5b, 0x10, 0x90, + 0x64, 0x58, 0x9b, 0xf0, 0xe9, 0x2e, 0x50, 0x17, 0x49, 0xaa, 0x34, 0xf5, 0xcb, 0x00, 0x10, 0x3e, + 0xeb, 0xa2, 0x4d, 0x18, 0xf6, 0xa9, 0xf4, 0x12, 0x86, 0xe7, 0xba, 0x00, 0xf3, 0xed, 0xe5, 0xff, + 0xc2, 0x01, 0x6e, 0x54, 0x4a, 0x0b, 0x07, 0x15, 0x66, 0xc8, 0xed, 0x2e, 0x14, 0xd4, 0x2b, 0x00, + 0x7a, 0x18, 0xe4, 0xb5, 0x2f, 0xff, 0x5a, 0x05, 0x01, 0x60, 0xc3, 0xb5, 0x92, 0x62, 0xc4, 0x4a, + 0x44, 0x32, 0xac, 0xd2, 0xfb, 0x73, 0x59, 0xa5, 0x3f, 0x01, 0xa7, 0x84, 0x15, 0x60, 0x85, 0xbd, + 0x32, 0x7a, 0xbb, 0xc2, 0x71, 0x93, 0xd9, 0x87, 0x96, 0xa2, 0x20, 0x1c, 0xaf, 0x8b, 0xfe, 0xa5, + 0x02, 0xc3, 0xba, 0x10, 0xd0, 0x04, 0x6f, 0x5b, 0xee, 0xed, 0x51, 0x7f, 0xce, 0x97, 0xf7, 0xb8, + 0x20, 0xff, 0xac, 0xcf, 0x55, 0xfd, 0xe2, 0x23, 0x12, 0xe3, 0x83, 0x5e, 0xa3, 0xdf, 0x53, 0x60, + 0x54, 0x33, 0x4d, 0x5b, 0xd7, 0x3c, 0x16, 0x1e, 0x90, 0x7b, 0x94, 0xde, 0xea, 0x71, 0x14, 0xf3, + 0x21, 0x46, 0x3e, 0x90, 0xf7, 0x05, 0x9a, 0x80, 0x10, 0x72, 0x44, 0x63, 0x91, 0xbb, 0x8f, 0xfe, + 0x7f, 0x05, 0xee, 0xe7, 0x6e, 0xbc, 0x25, 0x2a, 0x73, 0x6d, 0x1a, 0xba, 0xe6, 0x11, 0x1e, 0xb3, + 0xd4, 0x77, 0x21, 0xe3, 0x36, 0xf4, 0xc3, 0x87, 0xb6, 0xa1, 0xbf, 0xba, 0xbf, 0x57, 0xbc, 0xbf, + 0xd4, 0x05, 0x6e, 0xdc, 0x55, 0x0f, 0xd0, 0x5d, 0x18, 0x37, 0xe5, 0xf8, 0xe3, 0x82, 0xc9, 0xe7, + 0x7a, 0x7e, 0x8d, 0x04, 0x32, 0xe7, 0x0a, 0x86, 0x48, 0x11, 0x8e, 0x92, 0x9a, 0xbd, 0x0d, 0xe3, + 0x91, 0x8d, 0x76, 0xac, 0x8a, 0x6b, 0x0b, 0x26, 0xe3, 0xfb, 0xe1, 0x58, 0xef, 0x79, 0x37, 0x61, + 0x24, 0x10, 0x16, 0xd0, 0x25, 0x89, 0x50, 0x28, 0x7a, 0xdd, 0x24, 0xbb, 0x9c, 0x6a, 0x31, 0xa2, + 0x43, 0xe1, 0x4a, 0x42, 0x16, 0xbd, 0x51, 0x20, 0x54, 0xff, 0x40, 0xbc, 0xaa, 0xae, 0x91, 0x66, + 0xcb, 0xd4, 0x3c, 0xf2, 0xc6, 0xb7, 0x43, 0x52, 0xff, 0x83, 0xc2, 0xcf, 0x1b, 0x2e, 0xda, 0x20, + 0x0d, 0x46, 0x9b, 0x3c, 0xf9, 0x1d, 0x8b, 0xf5, 0xa8, 0xe4, 0x8f, 0x32, 0xb9, 0x12, 0xa2, 0xc1, + 0x32, 0x4e, 0x74, 0x07, 0x46, 0x7c, 0x61, 0xd0, 0xd7, 0xd4, 0x2c, 0xf5, 0x26, 0x9c, 0x05, 0x72, + 0x67, 0x70, 0xb4, 0xfa, 0x25, 0x2e, 0x0e, 0x69, 0xa9, 0x1a, 0xa0, 0x64, 0x1b, 0xf4, 0x40, 0xe8, + 0xcf, 0xa6, 0x44, 0xd3, 0xd5, 0x24, 0x7c, 0xda, 0x7c, 0x25, 0x6b, 0x21, 0x4b, 0xc9, 0xaa, 0xfe, + 0x66, 0x01, 0xa6, 0xc5, 0xd5, 0x73, 0x5e, 0xd7, 0xed, 0xb6, 0xe5, 0x85, 0x6e, 0xc6, 0xdc, 0x77, + 0x5f, 0x10, 0x61, 0xe2, 0x24, 0x77, 0xec, 0xc7, 0x02, 0x82, 0x6e, 0x71, 0xed, 0xa7, 0x55, 0x5f, + 0xb3, 0x6f, 0x13, 0x2b, 0xe4, 0x12, 0x72, 0x90, 0x8c, 0xc5, 0xb4, 0x0a, 0x38, 0xbd, 0x1d, 0xda, + 0x06, 0xd4, 0xd4, 0x76, 0xe2, 0xd8, 0xf2, 0x25, 0x41, 0x63, 0xf7, 0xc5, 0x95, 0x04, 0x36, 0x9c, + 0x42, 0x81, 0x1e, 0xa4, 0x54, 0x92, 0x6b, 0x79, 0xa4, 0xce, 0x87, 0xe8, 0x1b, 0x75, 0xb0, 0x83, + 0x74, 0x3e, 0x0a, 0xc2, 0xf1, 0xba, 0xea, 0xc7, 0x06, 0xe1, 0x7c, 0x74, 0x12, 0xe9, 0x17, 0xea, + 0xbb, 0xd7, 0x3f, 0xe5, 0xfb, 0x8e, 0xf1, 0x89, 0x7c, 0x20, 0xee, 0x3b, 0x36, 0x23, 0x1b, 0x0d, + 0x89, 0x46, 0x11, 0x3f, 0xb2, 0x6f, 0x80, 0xaf, 0x7c, 0x86, 0x73, 0x74, 0xdf, 0xb1, 0xc6, 0x04, + 0xf8, 0xa4, 0x02, 0xb3, 0xd1, 0xe2, 0x25, 0xc3, 0x32, 0xdc, 0x2d, 0x91, 0xf7, 0xe4, 0xf0, 0xae, + 0x6b, 0x2c, 0x73, 0xf2, 0x72, 0x26, 0x46, 0xdc, 0x81, 0x1a, 0xfa, 0x01, 0x05, 0x2e, 0xc4, 0xe6, + 0x25, 0x92, 0x85, 0xe5, 0xf0, 0x5e, 0x6c, 0x2c, 0xb8, 0xcb, 0x72, 0x36, 0x4a, 0xdc, 0x89, 0x5e, + 0xa7, 0x88, 0x9f, 0x83, 0xaf, 0x97, 0x88, 0x9f, 0xbf, 0x58, 0x80, 0x01, 0x66, 0x37, 0xf5, 0xc6, + 0x70, 0x38, 0x62, 0x5d, 0xcd, 0xb4, 0x77, 0x6d, 0xc4, 0xec, 0x5d, 0x9f, 0xca, 0x4f, 0xa2, 0xb3, + 0xc1, 0xeb, 0xfb, 0xe0, 0x2c, 0xab, 0x36, 0x5f, 0x67, 0x8a, 0x36, 0x97, 0xd4, 0xe7, 0xeb, 0x75, + 0x76, 0xbd, 0x3d, 0xf8, 0x8d, 0xec, 0x12, 0xf4, 0xb5, 0x1d, 0x33, 0x1e, 0x42, 0x76, 0x1d, 0x2f, + 0x63, 0x5a, 0xae, 0x7e, 0x52, 0x81, 0x49, 0x86, 0x5b, 0x62, 0x31, 0x68, 0x1b, 0x86, 0x1d, 0xc1, + 0x66, 0xc4, 0xda, 0x2c, 0xe7, 0x1e, 0x5a, 0x0a, 0xeb, 0xe2, 0x37, 0x36, 0xff, 0x17, 0x0e, 0x68, + 0xa9, 0x7f, 0x3a, 0x00, 0x33, 0x59, 0x8d, 0xd0, 0x8f, 0x28, 0x70, 0x56, 0x0f, 0x25, 0xce, 0xf9, + 0xb6, 0xb7, 0x65, 0x3b, 0x7e, 0x46, 0x85, 0xdc, 0xea, 0x90, 0xd2, 0x7c, 0xd0, 0x2b, 0x96, 0x46, + 0xa4, 0x94, 0x4a, 0x01, 0x67, 0x50, 0x46, 0xaf, 0x02, 0xb8, 0xee, 0xd6, 0x4d, 0xb2, 0xdb, 0xd2, + 0x0c, 0xdf, 0x72, 0xeb, 0x66, 0xfe, 0x6d, 0x50, 0xbb, 0x21, 0x50, 0x05, 0x9d, 0xe2, 0xba, 0xea, + 0xb0, 0x5c, 0x22, 0x87, 0x3e, 0xaa, 0xc0, 0xb8, 0x2d, 0xc7, 0x6b, 0xe9, 0xc5, 0x98, 0x3f, 0x35, + 0xf0, 0x0b, 0x97, 0xb4, 0xa3, 0xa0, 0x28, 0x49, 0xba, 0x2c, 0x53, 0x6e, 0xfc, 0x64, 0x13, 0xbc, + 0x6f, 0x25, 0x9f, 0x0c, 0x94, 0x71, 0x4c, 0xf2, 0x5b, 0x7b, 0x12, 0x9c, 0x24, 0xcf, 0x3a, 0x45, + 0x3c, 0xbd, 0xbe, 0x68, 0xe9, 0xce, 0x2e, 0x8b, 0x10, 0x40, 0x3b, 0x35, 0x98, 0xbf, 0x53, 0x8b, + 0x6b, 0xa5, 0x72, 0x04, 0x59, 0xb4, 0x53, 0x49, 0x70, 0x92, 0xbc, 0xfa, 0x91, 0x02, 0x9c, 0x63, + 0xcb, 0x7c, 0xb3, 0xbd, 0x41, 0xb8, 0xf1, 0xd5, 0x3f, 0xb8, 0x00, 0x3b, 0xbf, 0xa3, 0xc0, 0x08, + 0x9b, 0x83, 0x37, 0x88, 0x8f, 0x26, 0xeb, 0x6b, 0x86, 0x81, 0xf3, 0x6f, 0x2b, 0x30, 0x95, 0xc8, + 0x21, 0xd5, 0x95, 0x87, 0xdf, 0x89, 0xd9, 0xde, 0xbe, 0x25, 0xcc, 0x5b, 0xd9, 0x17, 0x06, 0xb6, + 0x88, 0xe7, 0xac, 0x54, 0x9f, 0x83, 0xf1, 0x88, 0x7d, 0xb3, 0x14, 0x02, 0x32, 0x2d, 0x78, 0xa5, + 0x1c, 0xe1, 0xb1, 0xd0, 0x29, 0x36, 0x65, 0xb8, 0xe5, 0x93, 0x9c, 0xed, 0x1f, 0xce, 0x96, 0x47, + 0x62, 0xcb, 0xfb, 0xcf, 0x38, 0x2c, 0xb0, 0xa4, 0xdb, 0xcb, 0x33, 0x0e, 0x8b, 0x50, 0x29, 0x02, + 0xc1, 0xf2, 0xff, 0xb1, 0xc0, 0x8a, 0x9e, 0x8e, 0x86, 0x79, 0x5d, 0x0d, 0xef, 0x76, 0xd3, 0xf1, + 0xe0, 0xac, 0x6c, 0x4b, 0x26, 0x6a, 0x23, 0xcc, 0x1f, 0x82, 0xf8, 0x59, 0x96, 0x2b, 0xbb, 0x4d, + 0x79, 0xb5, 0xc6, 0xe3, 0xff, 0x05, 0x0f, 0x40, 0x51, 0x8d, 0x77, 0xff, 0x49, 0x68, 0xbc, 0x1d, + 0x18, 0xdd, 0x32, 0x36, 0x88, 0x63, 0x71, 0x31, 0x66, 0x20, 0xbf, 0x84, 0x76, 0x23, 0x44, 0xc3, + 0xd5, 0x00, 0x52, 0x01, 0x96, 0x89, 0x20, 0x27, 0x12, 0xac, 0x7a, 0x30, 0xbf, 0x54, 0x12, 0xaa, + 0xa6, 0xc3, 0x71, 0x66, 0x04, 0xaa, 0xb6, 0x00, 0xac, 0x20, 0x82, 0x6b, 0x2f, 0x0f, 0x43, 0x61, + 0x1c, 0x58, 0x2e, 0x74, 0x84, 0xbf, 0xb1, 0x44, 0x81, 0xce, 0x6b, 0x33, 0x4c, 0x68, 0x20, 0xd4, + 0x8c, 0x4f, 0xf5, 0x98, 0x54, 0x42, 0xa8, 0x57, 0xc2, 0x02, 0x2c, 0x13, 0xa1, 0x63, 0x6c, 0x06, + 0x61, 0xfc, 0x85, 0x1a, 0x31, 0xd7, 0x18, 0xc3, 0x64, 0x00, 0x7c, 0x8c, 0xe1, 0x6f, 0x2c, 0x51, + 0x40, 0x2f, 0x4b, 0xef, 0x87, 0x90, 0x5f, 0x49, 0xd5, 0xd5, 0xdb, 0xe1, 0x3b, 0x43, 0x5d, 0xcd, + 0x28, 0xfb, 0x4e, 0x2f, 0x48, 0x7a, 0x1a, 0x96, 0xde, 0x80, 0xf2, 0x8e, 0x84, 0xde, 0x26, 0xf4, + 0xaa, 0x18, 0xeb, 0xe8, 0x55, 0x51, 0xa2, 0xd2, 0x99, 0xe4, 0x99, 0xc8, 0x18, 0xc2, 0x78, 0xf8, + 0x08, 0x52, 0x8b, 0x03, 0x71, 0xb2, 0x3e, 0x67, 0xf8, 0xa4, 0xce, 0xda, 0x4e, 0xc8, 0x0c, 0x9f, + 0x97, 0xe1, 0x00, 0x8a, 0xb6, 0x61, 0xcc, 0x95, 0x5c, 0x34, 0x66, 0x4e, 0xf5, 0xfa, 0x84, 0x28, + 0xdc, 0x33, 0x98, 0x2d, 0xa4, 0x5c, 0x82, 0x23, 0x74, 0xa2, 0xef, 0x5b, 0x93, 0x27, 0xfb, 0xbe, + 0x85, 0xda, 0x51, 0xeb, 0xeb, 0xa9, 0x23, 0x89, 0xe3, 0x72, 0xa0, 0x75, 0x36, 0x5d, 0x5a, 0xb2, + 0xd3, 0xb2, 0xdd, 0xb6, 0x43, 0x98, 0xd9, 0x27, 0x5b, 0x1e, 0x14, 0x2e, 0xed, 0x62, 0x1c, 0x88, + 0x93, 0xf5, 0xd1, 0xf7, 0x29, 0x30, 0xe9, 0xee, 0xba, 0x1e, 0x69, 0xd2, 0x63, 0xcb, 0xb6, 0x88, + 0xe5, 0xb9, 0x33, 0xa7, 0xf3, 0xc7, 0x05, 0xaf, 0xc5, 0x70, 0xf1, 0x63, 0x27, 0x5e, 0x8a, 0x13, + 0x34, 0xe9, 0xce, 0x91, 0x23, 0xc1, 0xcc, 0x4c, 0xe7, 0xdf, 0x39, 0x72, 0x94, 0x19, 0xbe, 0x73, + 0xe4, 0x12, 0x1c, 0xa1, 0x83, 0x1e, 0x85, 0x71, 0xd7, 0xcf, 0x61, 0xce, 0x66, 0xf0, 0x4c, 0x18, + 0xac, 0xb3, 0x26, 0x03, 0x70, 0xb4, 0x1e, 0xfa, 0x10, 0x8c, 0xc9, 0x67, 0xe7, 0xcc, 0xd9, 0xa3, + 0x0e, 0x5b, 0xcf, 0x7b, 0x2e, 0x83, 0x22, 0x04, 0x11, 0x86, 0xb3, 0x92, 0xf3, 0xa0, 0xfc, 0x7d, + 0x9f, 0x63, 0x43, 0xe0, 0xf7, 0xd9, 0xd4, 0x1a, 0x38, 0xa3, 0x25, 0xfa, 0x89, 0xf4, 0xe7, 0xf2, + 0x19, 0xb6, 0xa5, 0xab, 0x47, 0xf2, 0x5c, 0xfe, 0x9c, 0xe1, 0x6d, 0xdd, 0x6a, 0xf1, 0xc8, 0x62, + 0x87, 0x7c, 0x39, 0x57, 0xff, 0x48, 0x01, 0x08, 0x14, 0x26, 0x27, 0xf1, 0x54, 0x51, 0x8f, 0xe8, + 0x90, 0x16, 0x7a, 0x52, 0xf0, 0x64, 0x66, 0x25, 0x51, 0xbf, 0xa2, 0xc0, 0x44, 0x58, 0xed, 0x04, + 0xae, 0x46, 0x7a, 0xf4, 0x6a, 0xf4, 0x64, 0x6f, 0xe3, 0xca, 0xb8, 0x1f, 0xfd, 0xcf, 0x82, 0x3c, + 0x2a, 0x26, 0xfd, 0x6e, 0x47, 0x9e, 0xfe, 0x29, 0xe9, 0x1b, 0xbd, 0x3c, 0xfd, 0xcb, 0xd1, 0x37, + 0xc2, 0xf1, 0xa6, 0x98, 0x02, 0x7c, 0x77, 0x44, 0xfe, 0xec, 0x21, 0xee, 0x4d, 0x20, 0x6c, 0xfa, + 0xa4, 0xf9, 0x04, 0x1c, 0x24, 0x8c, 0xbe, 0x22, 0x1f, 0x4f, 0x3d, 0x64, 0x12, 0x89, 0x0c, 0xb8, + 0xb3, 0xd1, 0xc5, 0x2f, 0x4e, 0xc1, 0xa8, 0xa4, 0x5b, 0x8c, 0x19, 0x32, 0x28, 0x27, 0x61, 0xc8, + 0xe0, 0xc1, 0xa8, 0x1e, 0xa4, 0x92, 0xf4, 0xa7, 0xbd, 0x47, 0x9a, 0xc1, 0xb1, 0x18, 0x26, 0xa9, + 0x74, 0xb1, 0x4c, 0x86, 0x0a, 0x6f, 0xc1, 0x1e, 0xeb, 0x3b, 0x02, 0xf3, 0x92, 0x4e, 0xfb, 0xea, + 0x11, 0x00, 0x5f, 0xfe, 0x27, 0x75, 0x11, 0xa1, 0x3d, 0x30, 0x9d, 0xae, 0xb8, 0x37, 0x02, 0x18, + 0x96, 0xea, 0x25, 0x1f, 0xc6, 0x07, 0x4e, 0xec, 0x61, 0x9c, 0x6e, 0x03, 0xd3, 0x4f, 0xba, 0xdf, + 0x93, 0xb9, 0x5a, 0x90, 0xba, 0x3f, 0xdc, 0x06, 0x41, 0x91, 0x8b, 0x25, 0x22, 0x19, 0xf6, 0x2c, + 0x43, 0xb9, 0xec, 0x59, 0xda, 0x70, 0xda, 0x21, 0x9e, 0xb3, 0x5b, 0xda, 0xd5, 0x59, 0x6a, 0x13, + 0xc7, 0x63, 0x37, 0xf8, 0xe1, 0x7c, 0x01, 0x13, 0x71, 0x12, 0x15, 0x4e, 0xc3, 0x1f, 0x11, 0x80, + 0x47, 0x3a, 0x0a, 0xc0, 0xef, 0x84, 0x51, 0x8f, 0xe8, 0x5b, 0x96, 0xa1, 0x6b, 0x66, 0xa5, 0x2c, + 0xe2, 0x77, 0x87, 0xb2, 0x5c, 0x08, 0xc2, 0x72, 0x3d, 0xb4, 0x00, 0x7d, 0x6d, 0xa3, 0x2e, 0x6e, + 0x00, 0xdf, 0x16, 0x68, 0xe9, 0x2b, 0xe5, 0x7b, 0x7b, 0xc5, 0x37, 0x87, 0x06, 0x22, 0xc1, 0xa8, + 0xae, 0xb5, 0x6e, 0x37, 0xae, 0x79, 0xbb, 0x2d, 0xe2, 0xce, 0xad, 0x57, 0xca, 0x98, 0x36, 0x4e, + 0xb3, 0xf5, 0x19, 0x3b, 0x84, 0xad, 0xcf, 0x67, 0x14, 0x38, 0xad, 0xc5, 0x1f, 0x18, 0x88, 0x3b, + 0x33, 0x9e, 0x9f, 0x5b, 0xa6, 0x3f, 0x5a, 0x2c, 0x5c, 0x10, 0xe3, 0x3b, 0x3d, 0x9f, 0x24, 0x87, + 0xd3, 0xfa, 0x80, 0x1c, 0x40, 0x4d, 0xa3, 0x11, 0xe4, 0xbf, 0x17, 0xab, 0x3e, 0x91, 0x4f, 0x6f, + 0xb3, 0x92, 0xc0, 0x84, 0x53, 0xb0, 0xa3, 0x3b, 0x30, 0x2a, 0x09, 0x49, 0xe2, 0x26, 0x53, 0x3e, + 0x8a, 0x77, 0x10, 0x7e, 0xdb, 0x95, 0xdf, 0x38, 0x64, 0x4a, 0xc1, 0x23, 0xa7, 0xa4, 0x66, 0x10, + 0x0f, 0x7d, 0x6c, 0xd4, 0x93, 0xf9, 0x1f, 0x39, 0xd3, 0x31, 0xe2, 0x0e, 0xd4, 0x58, 0x98, 0x42, + 0x0a, 0x96, 0xee, 0xe6, 0x33, 0x53, 0xf9, 0xe3, 0xac, 0x2c, 0x47, 0x51, 0xf1, 0xad, 0x19, 0x2b, + 0xc4, 0x71, 0x82, 0x68, 0x09, 0x10, 0xe1, 0xaa, 0xf4, 0xf0, 0x72, 0xe6, 0xce, 0x20, 0xf6, 0xfe, + 0xce, 0x96, 0x74, 0x31, 0x01, 0xc5, 0x29, 0x2d, 0x90, 0x17, 0xd1, 0x95, 0xf4, 0x70, 0xcb, 0x89, + 0xe7, 0xcc, 0xe9, 0xa8, 0x31, 0xf9, 0x5e, 0x25, 0x91, 0x44, 0x99, 0x5f, 0x6e, 0x6e, 0xf4, 0x9e, + 0x44, 0x59, 0x90, 0xef, 0x26, 0x95, 0xf2, 0xe7, 0x15, 0x38, 0xd7, 0x4c, 0x4f, 0x16, 0xc9, 0xee, + 0x3c, 0x39, 0x1f, 0xae, 0x32, 0xf2, 0x4f, 0x32, 0x35, 0x46, 0x56, 0x72, 0x4a, 0x9c, 0xd5, 0x11, + 0xf5, 0x0f, 0x15, 0xa1, 0x8a, 0x3e, 0x41, 0x73, 0xa4, 0xe3, 0x7e, 0x27, 0x56, 0x9f, 0x83, 0x99, + 0x9a, 0x1f, 0x64, 0xb4, 0x1e, 0xcb, 0x25, 0xf0, 0x1e, 0x18, 0xe7, 0x4f, 0x41, 0x2b, 0x5a, 0x6b, + 0x35, 0x7c, 0x37, 0x08, 0xc2, 0x51, 0x94, 0x64, 0x20, 0x8e, 0xd6, 0x55, 0xbf, 0xa6, 0xc0, 0xb9, + 0x28, 0x66, 0xdb, 0x31, 0xee, 0xf6, 0x8e, 0x18, 0x7d, 0x5c, 0x81, 0xd1, 0xdb, 0xc1, 0x2b, 0x95, + 0x2f, 0xba, 0xe5, 0x72, 0xd9, 0xf0, 0x7b, 0x45, 0x1c, 0xe9, 0xd9, 0x2b, 0x99, 0xa0, 0x32, 0x04, + 0xba, 0x58, 0x26, 0xad, 0xfe, 0x17, 0x05, 0x12, 0xea, 0x03, 0xb4, 0x01, 0x43, 0x94, 0x48, 0x79, + 0xb5, 0x26, 0xf6, 0xc4, 0x7b, 0xf2, 0x49, 0x95, 0x0c, 0x05, 0x7f, 0x14, 0x11, 0x3f, 0xb0, 0x8f, + 0x18, 0x6d, 0x73, 0xff, 0x7f, 0x3f, 0xd1, 0x90, 0xd8, 0x1e, 0xb9, 0xc4, 0x76, 0x39, 0x61, 0x11, + 0xbf, 0xd6, 0xcb, 0x25, 0x38, 0x42, 0x47, 0x5d, 0x06, 0x08, 0x55, 0x3e, 0x3d, 0x9b, 0xf7, 0xfd, + 0xdb, 0xd3, 0x70, 0xa6, 0x67, 0x67, 0xc4, 0x8f, 0x2a, 0x70, 0x96, 0x6c, 0x1b, 0xba, 0x37, 0xbf, + 0xe9, 0x11, 0xe7, 0xd6, 0xad, 0x95, 0xb5, 0x2d, 0x87, 0xb8, 0x5b, 0xb6, 0x59, 0xef, 0xc6, 0x98, + 0x31, 0xc5, 0xf2, 0x8a, 0xa9, 0x26, 0x16, 0x53, 0x31, 0xe2, 0x0c, 0x4a, 0x4c, 0xdd, 0xb5, 0xcd, + 0x15, 0x01, 0x98, 0xde, 0xb9, 0xda, 0x8e, 0xeb, 0x89, 0xb8, 0x81, 0x5c, 0xdd, 0x15, 0x07, 0xe2, + 0x64, 0xfd, 0x38, 0x12, 0x96, 0x45, 0x8f, 0xc9, 0xed, 0x4a, 0x12, 0x09, 0x03, 0xe2, 0x64, 0x7d, + 0x19, 0x09, 0x5f, 0x29, 0x7a, 0x28, 0x0e, 0x24, 0x91, 0x04, 0x40, 0x9c, 0xac, 0x8f, 0xea, 0x70, + 0xd1, 0x21, 0xba, 0xdd, 0x6c, 0x12, 0xab, 0xce, 0x26, 0x65, 0x45, 0x73, 0x1a, 0x86, 0xb5, 0xe4, + 0x68, 0xac, 0x22, 0x7b, 0x3d, 0x50, 0x58, 0xc2, 0xdb, 0x8b, 0xb8, 0x43, 0x3d, 0xdc, 0x11, 0x0b, + 0x6a, 0xc2, 0xa9, 0x36, 0xe3, 0xff, 0x0e, 0x8b, 0x2b, 0xb6, 0xad, 0x99, 0xe2, 0x89, 0xe0, 0xb0, + 0x2b, 0xc6, 0x0e, 0xea, 0xf5, 0x28, 0x2a, 0x1c, 0xc7, 0x8d, 0x76, 0xa9, 0x78, 0x2e, 0xba, 0x23, + 0x91, 0x1c, 0xce, 0x45, 0x52, 0x88, 0xe8, 0x09, 0x74, 0x38, 0x8d, 0x06, 0xaa, 0xc0, 0x69, 0x4f, + 0x73, 0x1a, 0xc4, 0x2b, 0x55, 0xd7, 0xab, 0xc4, 0xd1, 0x29, 0x8f, 0x35, 0xb9, 0xb4, 0xae, 0x70, + 0x54, 0x6b, 0x49, 0x30, 0x4e, 0x6b, 0x83, 0x3e, 0x04, 0x6f, 0x89, 0x4e, 0xea, 0xb2, 0x7d, 0x87, + 0x38, 0x0b, 0x76, 0xdb, 0xaa, 0x47, 0x91, 0x03, 0x43, 0xfe, 0xc0, 0xfe, 0x5e, 0xf1, 0x2d, 0xb8, + 0x9b, 0x06, 0xb8, 0x3b, 0xbc, 0xc9, 0x0e, 0xac, 0xb7, 0x5a, 0xa9, 0x1d, 0x18, 0xcd, 0xea, 0x40, + 0x46, 0x03, 0xdc, 0x1d, 0x5e, 0x84, 0xe1, 0x2c, 0x9f, 0x18, 0x9e, 0x9e, 0x59, 0xa2, 0x38, 0xc6, + 0x28, 0xb2, 0xef, 0x77, 0x2d, 0xb5, 0x06, 0xce, 0x68, 0x49, 0xcf, 0x94, 0xab, 0x59, 0xc3, 0x4f, + 0x90, 0x19, 0x67, 0x64, 0xde, 0xb6, 0xbf, 0x57, 0xbc, 0x8a, 0xbb, 0x6c, 0x83, 0xbb, 0xc6, 0x9e, + 0xd2, 0x95, 0x70, 0x22, 0x12, 0x5d, 0x99, 0xc8, 0xea, 0x4a, 0x76, 0x1b, 0xdc, 0x35, 0x76, 0xf4, + 0xfd, 0x0a, 0x9c, 0xd7, 0x5b, 0xed, 0x1b, 0x86, 0xeb, 0xd9, 0x0d, 0x47, 0x6b, 0x96, 0x89, 0xae, + 0xed, 0xde, 0xd0, 0xcc, 0xcd, 0x65, 0x63, 0x93, 0x88, 0x4b, 0xc7, 0x61, 0x3f, 0x1c, 0xe6, 0x78, + 0x5b, 0xaa, 0xae, 0xa7, 0x23, 0xc5, 0xd9, 0xf4, 0xd0, 0x8f, 0x2a, 0x70, 0xb1, 0xc9, 0xba, 0x98, + 0xd1, 0xa1, 0xc9, 0x5c, 0x1d, 0x62, 0x5c, 0x6c, 0xa5, 0x03, 0x5e, 0xdc, 0x91, 0x2a, 0x9b, 0x24, + 0x5e, 0x61, 0xbe, 0xd1, 0x70, 0x48, 0x83, 0x61, 0x0d, 0xb8, 0xcb, 0x54, 0xfe, 0x49, 0x5a, 0xc9, + 0x42, 0x8a, 0xb3, 0xe9, 0xa1, 0x97, 0xe1, 0x72, 0x26, 0xb0, 0x64, 0xb7, 0x2d, 0x8f, 0x3d, 0xc2, + 0xf4, 0x2d, 0xa8, 0xfb, 0x7b, 0xc5, 0xcb, 0x2b, 0x1d, 0x6b, 0xe2, 0x03, 0x30, 0xb1, 0xd8, 0x23, + 0x91, 0x30, 0x00, 0xa7, 0x99, 0x24, 0xf6, 0x62, 0xae, 0x74, 0xbc, 0x47, 0xe8, 0xfb, 0xff, 0x95, + 0xa8, 0xef, 0xff, 0x34, 0xeb, 0xd5, 0xfb, 0x8e, 0xae, 0x57, 0xdf, 0x74, 0xf8, 0xef, 0xc6, 0x11, + 0xe4, 0x33, 0x0a, 0x08, 0xff, 0x4d, 0x74, 0x31, 0x62, 0x76, 0x34, 0x1c, 0x33, 0x39, 0xf2, 0xb3, + 0xe2, 0x16, 0x52, 0xb3, 0xe2, 0xbe, 0x55, 0x0a, 0xc2, 0x3c, 0x12, 0x5e, 0x96, 0x38, 0xe6, 0x30, + 0x0a, 0x33, 0x7a, 0x10, 0x46, 0x82, 0x4b, 0xb5, 0x50, 0x76, 0xb2, 0x8c, 0x34, 0xe1, 0xed, 0x3b, + 0x84, 0xab, 0xbf, 0xaf, 0x00, 0x84, 0xc9, 0x9e, 0xd1, 0x7d, 0x7e, 0xb0, 0x1a, 0xde, 0xc1, 0xe0, + 0xc5, 0x40, 0x0e, 0x58, 0x73, 0xb0, 0x33, 0x02, 0x52, 0x61, 0xb0, 0xcd, 0xd2, 0x5b, 0x0a, 0x07, + 0x02, 0x66, 0x02, 0xb3, 0xce, 0x4a, 0xb0, 0x80, 0xa0, 0x75, 0x18, 0x6a, 0x1a, 0x16, 0xf3, 0xf5, + 0xe8, 0xcf, 0xe5, 0xeb, 0xc1, 0xee, 0x03, 0x2b, 0x1c, 0x05, 0xf6, 0x71, 0xa9, 0xbf, 0xa4, 0xc0, + 0xa9, 0x68, 0x54, 0x6c, 0x16, 0xf2, 0x46, 0x04, 0x4b, 0x15, 0xc1, 0xf8, 0x59, 0x53, 0x11, 0x04, + 0x12, 0xfb, 0xb0, 0xe8, 0xeb, 0x74, 0x0f, 0xaf, 0x0f, 0xe9, 0xc1, 0xb9, 0x0f, 0x78, 0x08, 0xf8, + 0xf1, 0xd3, 0x30, 0xc8, 0x2f, 0xda, 0x54, 0x8e, 0x4f, 0x09, 0x7e, 0x75, 0x33, 0x7f, 0xbe, 0x89, + 0x3c, 0x11, 0x71, 0xe4, 0xac, 0x9b, 0x85, 0x8e, 0x59, 0x37, 0x31, 0xf4, 0xe9, 0x8e, 0xd1, 0x8b, + 0x25, 0x52, 0x09, 0x57, 0xb8, 0x25, 0x52, 0x09, 0x57, 0x30, 0x45, 0x86, 0xbc, 0x88, 0x89, 0x4e, + 0x7f, 0x7e, 0x15, 0x10, 0x9f, 0x00, 0xc9, 0x50, 0x67, 0xa2, 0xa3, 0x91, 0x8e, 0x1f, 0x69, 0x7f, + 0x20, 0xbf, 0x73, 0x90, 0x98, 0xf2, 0x6e, 0x22, 0xed, 0xfb, 0x1f, 0xd2, 0x60, 0xe6, 0x87, 0xb4, + 0x09, 0x43, 0xe2, 0x53, 0x10, 0x17, 0x82, 0xf7, 0xf4, 0x90, 0x5b, 0x5e, 0xca, 0x62, 0xc5, 0x0b, + 0xb0, 0x8f, 0x9c, 0xde, 0x32, 0x9b, 0xda, 0x8e, 0xd1, 0x6c, 0x37, 0xd9, 0x2d, 0x60, 0x40, 0xae, + 0xca, 0x8a, 0xb1, 0x0f, 0x67, 0x55, 0xb9, 0x4f, 0x15, 0x93, 0xda, 0xe5, 0xaa, 0xbc, 0x18, 0xfb, + 0x70, 0xf4, 0x02, 0x0c, 0x37, 0xb5, 0x9d, 0x5a, 0xdb, 0x69, 0x10, 0x61, 0xa0, 0x93, 0xad, 0x14, + 0x6a, 0x7b, 0x86, 0x39, 0x67, 0x58, 0x9e, 0xeb, 0x39, 0x73, 0x15, 0xcb, 0xbb, 0xe5, 0xd4, 0x3c, + 0x66, 0x00, 0xc4, 0x76, 0xdd, 0x8a, 0xc0, 0x82, 0x03, 0x7c, 0xc8, 0x84, 0x89, 0xa6, 0xb6, 0xb3, + 0x6e, 0x69, 0x3c, 0x89, 0x82, 0x90, 0xb2, 0xf3, 0x50, 0x60, 0x5a, 0xb9, 0x95, 0x08, 0x2e, 0x1c, + 0xc3, 0x9d, 0x62, 0x0c, 0x3a, 0x76, 0x5c, 0xc6, 0xa0, 0xf3, 0x41, 0x94, 0x02, 0xae, 0xd2, 0x3f, + 0x9f, 0x1a, 0x1f, 0xb0, 0x63, 0x04, 0x82, 0x97, 0x82, 0x08, 0x04, 0x13, 0xf9, 0xad, 0x17, 0x3b, + 0x44, 0x1f, 0x68, 0xc3, 0x68, 0x5d, 0xf3, 0x34, 0x5e, 0xea, 0xce, 0x9c, 0xca, 0xff, 0x3a, 0x5d, + 0x0e, 0xd0, 0x84, 0x2c, 0x29, 0x2c, 0x73, 0xb1, 0x4c, 0x07, 0xdd, 0x82, 0x33, 0xf4, 0x63, 0x35, + 0x89, 0x17, 0x56, 0x61, 0x3a, 0xb3, 0x49, 0xf6, 0xfd, 0x30, 0x2f, 0xb5, 0x9b, 0x69, 0x15, 0x70, + 0x7a, 0xbb, 0x30, 0x96, 0xed, 0x54, 0x7a, 0x2c, 0x5b, 0xf4, 0xa9, 0x34, 0xb3, 0x1b, 0xc4, 0xe6, + 0xf4, 0xbd, 0xf9, 0x79, 0x43, 0x6e, 0xe3, 0x9b, 0x7f, 0xae, 0xc0, 0x8c, 0xd8, 0x65, 0xc2, 0x54, + 0xc6, 0x24, 0xce, 0x8a, 0x66, 0x69, 0x0d, 0xe2, 0x08, 0x3d, 0xf9, 0x5a, 0x0f, 0xfc, 0x21, 0x81, + 0x33, 0x08, 0x0d, 0x71, 0xff, 0xfe, 0x5e, 0xf1, 0xca, 0x41, 0xb5, 0x70, 0x66, 0xdf, 0x90, 0x03, + 0x43, 0xee, 0xae, 0xab, 0x7b, 0xa6, 0x2b, 0x64, 0xd0, 0xeb, 0x3d, 0x70, 0xd6, 0x1a, 0xc7, 0xc4, + 0x59, 0x6b, 0x98, 0x3b, 0x91, 0x97, 0x62, 0x9f, 0x10, 0xfa, 0x61, 0x05, 0xa6, 0xc4, 0xe3, 0x99, + 0x14, 0x7a, 0xe7, 0x4c, 0x7e, 0x3f, 0x99, 0x52, 0x1c, 0x99, 0x6f, 0x1e, 0xc3, 0xb4, 0x49, 0x09, + 0x28, 0x4e, 0x52, 0xa7, 0x87, 0x6a, 0xcb, 0x31, 0x6c, 0xc7, 0xf0, 0x76, 0x99, 0x21, 0xd2, 0x80, + 0x1f, 0xc1, 0x9e, 0x97, 0xe1, 0x00, 0x8a, 0x6a, 0x30, 0xc1, 0xb5, 0x36, 0x35, 0xcf, 0xd1, 0x3c, + 0xd2, 0xd8, 0x15, 0xd6, 0x42, 0x0f, 0xb2, 0x1c, 0xc3, 0x11, 0xc8, 0xbd, 0xbd, 0xe2, 0x19, 0xb1, + 0x36, 0x51, 0x00, 0x8e, 0xa1, 0x40, 0x1f, 0x88, 0x19, 0x6f, 0xcd, 0xe4, 0xcf, 0x3f, 0xc8, 0xd7, + 0xe2, 0x30, 0x26, 0x5c, 0xbd, 0x46, 0x93, 0xeb, 0x21, 0x07, 0xcc, 0xec, 0xe3, 0x30, 0x26, 0xef, + 0x9a, 0x43, 0x05, 0xb1, 0xf3, 0x00, 0x25, 0x07, 0x7b, 0xdc, 0x71, 0x7e, 0xd4, 0x9f, 0x56, 0x60, + 0x32, 0x2e, 0xbb, 0xa0, 0x2d, 0x18, 0x12, 0x8c, 0x4c, 0x50, 0x9d, 0xcf, 0x6b, 0xb5, 0x6c, 0x12, + 0xe1, 0x1e, 0xcc, 0x45, 0x61, 0x51, 0x84, 0x7d, 0xf4, 0xb2, 0x47, 0x42, 0xa1, 0x83, 0x47, 0xc2, + 0x13, 0x70, 0x36, 0x9d, 0xa5, 0xd1, 0x8b, 0x84, 0x46, 0xef, 0x57, 0x42, 0x69, 0x1d, 0x5c, 0x24, + 0xd8, 0xa5, 0x0b, 0x73, 0x98, 0xfa, 0x41, 0x88, 0xe7, 0x3e, 0x43, 0x2f, 0xc3, 0x88, 0xeb, 0x6e, + 0x71, 0x3b, 0x33, 0x31, 0xc8, 0x7c, 0x4f, 0x3d, 0x7e, 0x1e, 0x1a, 0x7e, 0xf7, 0x09, 0x7e, 0xe2, + 0x10, 0xfd, 0xc2, 0xf3, 0x5f, 0xfa, 0xda, 0xe5, 0x37, 0xfd, 0xc1, 0xd7, 0x2e, 0xbf, 0xe9, 0xab, + 0x5f, 0xbb, 0xfc, 0xa6, 0x0f, 0xef, 0x5f, 0x56, 0xbe, 0xb4, 0x7f, 0x59, 0xf9, 0x83, 0xfd, 0xcb, + 0xca, 0x57, 0xf7, 0x2f, 0x2b, 0x7f, 0xb9, 0x7f, 0x59, 0xf9, 0xa1, 0x7f, 0x77, 0xf9, 0x4d, 0x2f, + 0x3c, 0x1c, 0x52, 0xbf, 0xe6, 0x13, 0x0d, 0xff, 0x69, 0xdd, 0x6e, 0xd0, 0x4b, 0xab, 0xeb, 0xdf, + 0x5a, 0x19, 0xf5, 0xff, 0x1d, 0x00, 0x00, 0xff, 0xff, 0xd6, 0xf4, 0xb6, 0x47, 0x63, 0x25, 0x01, + 0x00, } func (m *APIServerLogging) Marshal() (dAtA []byte, err error) { @@ -15942,6 +15943,13 @@ func (m *SeedSettingLoadBalancerServices) MarshalToSizedBuffer(dAtA []byte) (int _ = i var l int _ = l + if m.Class != nil { + i -= len(*m.Class) + copy(dAtA[i:], *m.Class) + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.Class))) + i-- + dAtA[i] = 0x32 + } if m.ZonalIngress != nil { { size, err := m.ZonalIngress.MarshalToSizedBuffer(dAtA[:i]) @@ -22596,6 +22604,10 @@ func (m *SeedSettingLoadBalancerServices) Size() (n int) { l = m.ZonalIngress.Size() n += 1 + l + sovGenerated(uint64(l)) } + if m.Class != nil { + l = len(*m.Class) + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -26130,6 +26142,7 @@ func (this *SeedSettingLoadBalancerServices) String() string { `Zones:` + repeatedStringForZones + `,`, `ProxyProtocol:` + strings.Replace(this.ProxyProtocol.String(), "LoadBalancerServicesProxyProtocol", "LoadBalancerServicesProxyProtocol", 1) + `,`, `ZonalIngress:` + strings.Replace(this.ZonalIngress.String(), "SeedSettingLoadBalancerServicesZonalIngress", "SeedSettingLoadBalancerServicesZonalIngress", 1) + `,`, + `Class:` + valueToStringGenerated(this.Class) + `,`, `}`, }, "") return s @@ -53844,6 +53857,39 @@ func (m *SeedSettingLoadBalancerServices) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Class", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.Class = &s + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/core/v1beta1/generated.proto b/pkg/apis/core/v1beta1/generated.proto index 1e397da85de..658e7768912 100644 --- a/pkg/apis/core/v1beta1/generated.proto +++ b/pkg/apis/core/v1beta1/generated.proto @@ -2850,6 +2850,11 @@ message SeedSettingLoadBalancerServices { // Defaults to true. // +optional optional SeedSettingLoadBalancerServicesZonalIngress zonalIngress = 5; + + // Class configures the Service.spec.loadBalancerClass field for the load balancer services on the seed. + // Note that changing the loadBalancerClass of existing LoadBalancer services is denied by Kubernetes. + // +optional + optional string class = 6; } // SeedSettingLoadBalancerServicesZonalIngress controls the deployment of ingress gateways per availability zone. diff --git a/pkg/apis/core/v1beta1/types_seed.go b/pkg/apis/core/v1beta1/types_seed.go index f185a1e59e6..647b8964d9f 100644 --- a/pkg/apis/core/v1beta1/types_seed.go +++ b/pkg/apis/core/v1beta1/types_seed.go @@ -347,6 +347,10 @@ type SeedSettingLoadBalancerServices struct { // Defaults to true. // +optional ZonalIngress *SeedSettingLoadBalancerServicesZonalIngress `json:"zonalIngress,omitempty" protobuf:"bytes,5,opt,name=zonalIngress"` + // Class configures the Service.spec.loadBalancerClass field for the load balancer services on the seed. + // Note that changing the loadBalancerClass of existing LoadBalancer services is denied by Kubernetes. + // +optional + Class *string `json:"class,omitempty" protobuf:"bytes,6,opt,name=class"` } // SeedSettingLoadBalancerServicesZones controls settings, which are specific to the single-zone load balancers in a diff --git a/pkg/apis/core/v1beta1/zz_generated.conversion.go b/pkg/apis/core/v1beta1/zz_generated.conversion.go index e7aa9f0f336..1db27e77ee2 100644 --- a/pkg/apis/core/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/core/v1beta1/zz_generated.conversion.go @@ -6270,6 +6270,7 @@ func autoConvert_v1beta1_SeedSettingLoadBalancerServices_To_core_SeedSettingLoad out.Zones = *(*[]core.SeedSettingLoadBalancerServicesZones)(unsafe.Pointer(&in.Zones)) out.ProxyProtocol = (*core.LoadBalancerServicesProxyProtocol)(unsafe.Pointer(in.ProxyProtocol)) out.ZonalIngress = (*core.SeedSettingLoadBalancerServicesZonalIngress)(unsafe.Pointer(in.ZonalIngress)) + out.Class = (*string)(unsafe.Pointer(in.Class)) return nil } @@ -6284,6 +6285,7 @@ func autoConvert_core_SeedSettingLoadBalancerServices_To_v1beta1_SeedSettingLoad out.Zones = *(*[]SeedSettingLoadBalancerServicesZones)(unsafe.Pointer(&in.Zones)) out.ProxyProtocol = (*LoadBalancerServicesProxyProtocol)(unsafe.Pointer(in.ProxyProtocol)) out.ZonalIngress = (*SeedSettingLoadBalancerServicesZonalIngress)(unsafe.Pointer(in.ZonalIngress)) + out.Class = (*string)(unsafe.Pointer(in.Class)) return nil } diff --git a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go index bb6815fd427..5cfe903594a 100644 --- a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go @@ -4925,6 +4925,11 @@ func (in *SeedSettingLoadBalancerServices) DeepCopyInto(out *SeedSettingLoadBala *out = new(SeedSettingLoadBalancerServicesZonalIngress) (*in).DeepCopyInto(*out) } + if in.Class != nil { + in, out := &in.Class, &out.Class + *out = new(string) + **out = **in + } return } diff --git a/pkg/apis/core/validation/seed.go b/pkg/apis/core/validation/seed.go index c80e708943a..9495f9faf3c 100644 --- a/pkg/apis/core/validation/seed.go +++ b/pkg/apis/core/validation/seed.go @@ -158,6 +158,10 @@ func ValidateSeedSpec(seedSpec *core.SeedSpec, fldPath *field.Path, inTemplate b if seedSpec.Settings.LoadBalancerServices != nil { allErrs = append(allErrs, apivalidation.ValidateAnnotations(seedSpec.Settings.LoadBalancerServices.Annotations, fldPath.Child("settings", "loadBalancerServices", "annotations"))...) + if class := seedSpec.Settings.LoadBalancerServices.Class; class != nil { + allErrs = append(allErrs, kubernetescorevalidation.ValidateQualifiedName(*class, fldPath.Child("settings", "loadBalancerServices", "class"))...) + } + if policy := seedSpec.Settings.LoadBalancerServices.ExternalTrafficPolicy; policy != nil && !availableExternalTrafficPolicies.Has(string(*policy)) { allErrs = append(allErrs, field.NotSupported(fldPath.Child("settings", "loadBalancerServices", "externalTrafficPolicy"), *policy, sets.List(availableExternalTrafficPolicies))) } diff --git a/pkg/apis/core/validation/seed_test.go b/pkg/apis/core/validation/seed_test.go index c8f05f6416e..b8ca21de21e 100644 --- a/pkg/apis/core/validation/seed_test.go +++ b/pkg/apis/core/validation/seed_test.go @@ -1320,6 +1320,56 @@ var _ = Describe("Seed Validation Tests", func() { })), )) }) + + Describe("LoadBalancer class", func() { + BeforeEach(func() { + seed.Spec.Settings = &core.SeedSettings{ + LoadBalancerServices: &core.SeedSettingLoadBalancerServices{}, + } + }) + + It("should allow omitting the class", func() { + seed.Spec.Settings.LoadBalancerServices.Class = nil + + Expect(ValidateSeed(seed)).To(BeEmpty()) + }) + + It("should allow specifying a non-empty class", func() { + seed.Spec.Settings.LoadBalancerServices.Class = ptr.To("non-default") + + Expect(ValidateSeed(seed)).To(BeEmpty()) + }) + + It("should allow specifying a qualified class", func() { + seed.Spec.Settings.LoadBalancerServices.Class = ptr.To("stackit.cloud/yawol") + + Expect(ValidateSeed(seed)).To(BeEmpty()) + }) + + It("should deny specifying an empty class", func() { + seed.Spec.Settings.LoadBalancerServices.Class = ptr.To("") + + Expect(ValidateSeed(seed)).To(ContainElement( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.settings.loadBalancerServices.class"), + "Origin": Equal("format=qualified-name"), + })), + )) + }) + + It("should deny specifying a class that Kubernetes does not accept", func() { + seed.Spec.Settings.LoadBalancerServices.Class = ptr.To(".invalid-") + + Expect(ValidateSeed(seed)).To(ContainElement( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.settings.loadBalancerServices.class"), + "Origin": Equal("format=qualified-name"), + })), + )) + }) + }) }) Context("vertical pod autoscaler", func() { diff --git a/pkg/apis/core/zz_generated.deepcopy.go b/pkg/apis/core/zz_generated.deepcopy.go index 4c0893d3504..7648fb2f518 100644 --- a/pkg/apis/core/zz_generated.deepcopy.go +++ b/pkg/apis/core/zz_generated.deepcopy.go @@ -4925,6 +4925,11 @@ func (in *SeedSettingLoadBalancerServices) DeepCopyInto(out *SeedSettingLoadBala *out = new(SeedSettingLoadBalancerServicesZonalIngress) (*in).DeepCopyInto(*out) } + if in.Class != nil { + in, out := &in.Class, &out.Class + *out = new(string) + **out = **in + } return } diff --git a/pkg/apiserver/openapi/openapi_generated.go b/pkg/apiserver/openapi/openapi_generated.go index da3d893769c..f3848af0412 100644 --- a/pkg/apiserver/openapi/openapi_generated.go +++ b/pkg/apiserver/openapi/openapi_generated.go @@ -8440,6 +8440,13 @@ func schema_pkg_apis_core_v1beta1_SeedSettingLoadBalancerServices(ref common.Ref Ref: ref("github.com/gardener/gardener/pkg/apis/core/v1beta1.SeedSettingLoadBalancerServicesZonalIngress"), }, }, + "class": { + SchemaProps: spec.SchemaProps{ + Description: "Class configures the Service.spec.loadBalancerClass field for the load balancer services on the seed. Note that changing the loadBalancerClass of existing LoadBalancer services is denied by Kubernetes.", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/pkg/component/networking/istio/charts/istio/istio-ingress/templates/service.yaml b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/service.yaml index 541e1a13b1b..586d020f59c 100644 --- a/pkg/component/networking/istio/charts/istio/istio-ingress/templates/service.yaml +++ b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/service.yaml @@ -25,6 +25,9 @@ spec: {{- if .Values.loadBalancerIP }} loadBalancerIP: {{ .Values.loadBalancerIP }} {{- end }} +{{- with .Values.loadBalancerClass }} + loadBalancerClass: {{ . | quote }} +{{- end }} {{- if .Values.externalTrafficPolicy }} externalTrafficPolicy: {{ .Values.externalTrafficPolicy }} {{- end }} diff --git a/pkg/component/networking/istio/charts/istio/istio-ingress/values.yaml b/pkg/component/networking/istio/charts/istio/istio-ingress/values.yaml index ae8bf5b1510..048a36ef74f 100644 --- a/pkg/component/networking/istio/charts/istio/istio-ingress/values.yaml +++ b/pkg/component/networking/istio/charts/istio/istio-ingress/values.yaml @@ -22,6 +22,7 @@ ports: [] serviceName: istio-ingressgateway internalServiceName: istio-ingressgateway-internal ingressVersion: "1.27.1" +#loadBalancerClass: non-default-loadbalancer-class #externalTrafficPolicy: Cluster replicas: 2 cpuRequests: 300m diff --git a/pkg/component/networking/istio/ingress_gateway.go b/pkg/component/networking/istio/ingress_gateway.go index 62f86edb8cf..d86ec9a4051 100644 --- a/pkg/component/networking/istio/ingress_gateway.go +++ b/pkg/component/networking/istio/ingress_gateway.go @@ -35,6 +35,7 @@ type IngressGatewayValues struct { Annotations map[string]string Labels map[string]string NetworkPolicyLabels map[string]string + LoadBalancerClass *string ExternalTrafficPolicy *corev1.ServiceExternalTrafficPolicy Image string IstiodNamespace string @@ -104,6 +105,7 @@ func (i *istiod) generateIstioIngressGatewayChart(ctx context.Context) (*chartre "labels": istioIngressGateway.Labels, "networkPolicyLabels": istioIngressGateway.NetworkPolicyLabels, "annotations": istioIngressGateway.Annotations, + "loadBalancerClass": istioIngressGateway.LoadBalancerClass, "externalTrafficPolicy": istioIngressGateway.ExternalTrafficPolicy, "dualStack": istioIngressGateway.DualStack, "deployNamespace": false, diff --git a/pkg/component/networking/istio/istio_test.go b/pkg/component/networking/istio/istio_test.go index 5715e30ae27..a877a7f25b7 100644 --- a/pkg/component/networking/istio/istio_test.go +++ b/pkg/component/networking/istio/istio_test.go @@ -191,6 +191,11 @@ var _ = Describe("istiod", func() { return string(data) } + istioIngressServiceClass = func() string { + data, _ := os.ReadFile("./test_charts/ingress_service_class.yaml") + return string(data) + } + istioIngressServiceETPCluster = func() string { data, _ := os.ReadFile("./test_charts/ingress_service_etp_cluster.yaml") return string(data) @@ -552,6 +557,36 @@ var _ = Describe("istiod", func() { }) }) + Context("LoadBalancer class", func() { + BeforeEach(func() { + igw[0].LoadBalancerClass = ptr.To("non-default-loadbalancer-class") + istiod = NewIstio( + c, + renderer, + Values{ + Istiod: IstiodValues{ + Enabled: true, + Image: "foo/bar", + Namespace: deployNS, + TrustDomain: "foo.local", + Zones: []string{"a", "b", "c"}, + }, + IngressGateway: igw, + }, + ) + }) + + It("should successfully deploy correct loadBalancerClass", func() { + Expect(c.Get(ctx, client.ObjectKeyFromObject(managedResourceIstioSecret), managedResourceIstioSecret)).To(Succeed()) + + var err error + istioManifests, err := test.ExtractManifestsFromManagedResourceData(managedResourceIstioSecret.Data) + Expect(err).NotTo(HaveOccurred()) + + Expect(istioManifests).To(ContainElement(istioIngressServiceClass())) + }) + }) + Context("external traffic policy cluster", func() { BeforeEach(func() { externalTrafficPolicy = corev1.ServiceExternalTrafficPolicyCluster diff --git a/pkg/component/networking/istio/test_charts/ingress_service_class.yaml b/pkg/component/networking/istio/test_charts/ingress_service_class.yaml new file mode 100644 index 00000000000..9e388d229b1 --- /dev/null +++ b/pkg/component/networking/istio/test_charts/ingress_service_class.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingressgateway + namespace: test-ingress + annotations: + networking.resources.gardener.cloud/from-world-to-ports: '[{"port":8132,"protocol":"TCP"},{"port":8443,"protocol":"TCP"},{"port":9443,"protocol":"TCP"}]' + networking.resources.gardener.cloud/namespace-selectors: '[{"matchLabels":{"gardener.cloud/role":"extension"}},{"matchLabels":{"gardener.cloud/role":"shoot"}},{"matchLabels":{"kubernetes.io/metadata.name":"garden"}}]' + networking.resources.gardener.cloud/pod-label-selector-namespace-alias: all-istio-ingresses + networking.resources.gardener.cloud/from-all-seed-scrape-targets-allowed-ports: '[{"port":15022,"protocol":"TCP"}]' + foo: bar + labels: + app.kubernetes.io/version: 1.27.1 + app: istio-ingressgateway + foo: bar +spec: + type: LoadBalancer + selector: + app: istio-ingressgateway + foo: bar + ports: + - name: foo + port: 999 + targetPort: 999 + loadBalancerClass: "non-default-loadbalancer-class" diff --git a/pkg/component/shared/istio.go b/pkg/component/shared/istio.go index e60d4572e4a..b3dd8b39c95 100644 --- a/pkg/component/shared/istio.go +++ b/pkg/component/shared/istio.go @@ -44,6 +44,7 @@ func NewIstio( labels map[string]string, toKubeAPIServerPolicyLabel string, lbAnnotations map[string]string, + loadBalancerClass *string, externalTrafficPolicy *corev1.ServiceExternalTrafficPolicy, serviceExternalIP *string, servicePorts []corev1.ServicePort, @@ -98,6 +99,7 @@ func NewIstio( Image: igwImage.String(), IstiodNamespace: v1beta1constants.IstioSystemNamespace, Annotations: lbAnnotations, + LoadBalancerClass: loadBalancerClass, ExternalTrafficPolicy: externalTrafficPolicy, MinReplicas: minReplicas, MaxReplicas: maxReplicas, @@ -145,6 +147,7 @@ func AddIstioIngressGateway( namespace string, annotations map[string]string, labels map[string]string, + loadBalancerClass *string, externalTrafficPolicy *corev1.ServiceExternalTrafficPolicy, serviceExternalIP *string, zone *string, @@ -188,6 +191,7 @@ func AddIstioIngressGateway( Namespace: namespace, MinReplicas: minReplicas, MaxReplicas: maxReplicas, + LoadBalancerClass: loadBalancerClass, ExternalTrafficPolicy: externalTrafficPolicy, LoadBalancerIP: serviceExternalIP, Image: templateValues.Image, diff --git a/pkg/component/shared/istio_test.go b/pkg/component/shared/istio_test.go index c83244eba31..2ac77db6b8c 100644 --- a/pkg/component/shared/istio_test.go +++ b/pkg/component/shared/istio_test.go @@ -39,6 +39,7 @@ type istioTestValues struct { labels map[string]string kubeAPIServerPolicyLabel string lbAnnotations map[string]string + loadBalancerClass *string externalTrafficPolicy *corev1.ServiceExternalTrafficPolicy serviceExternalIP *string servicePorts []corev1.ServicePort @@ -70,6 +71,7 @@ func createIstio(testValues istioTestValues) istio.Interface { testValues.labels, testValues.kubeAPIServerPolicyLabel, testValues.lbAnnotations, + testValues.loadBalancerClass, testValues.externalTrafficPolicy, testValues.serviceExternalIP, testValues.servicePorts, @@ -121,6 +123,7 @@ func checkIstio(istioDeploy istio.Interface, testValues istioTestValues) { Image: testValues.ingressImageName, IstiodNamespace: "istio-system", Annotations: testValues.lbAnnotations, + LoadBalancerClass: testValues.loadBalancerClass, ExternalTrafficPolicy: testValues.externalTrafficPolicy, MinReplicas: minReplicas, MaxReplicas: maxReplicas, @@ -145,6 +148,7 @@ func checkAdditionalIstioGateway(cl client.Client, namespace string, annotations map[string]string, labels map[string]string, + loadBalancerClass *string, externalTrafficPolicy *corev1.ServiceExternalTrafficPolicy, serviceExternalIP *string, zone *string, @@ -174,6 +178,7 @@ func checkAdditionalIstioGateway(cl client.Client, Image: ingressValues[0].Image, IstiodNamespace: "istio-system", Annotations: annotations, + LoadBalancerClass: loadBalancerClass, ExternalTrafficPolicy: externalTrafficPolicy, MinReplicas: minReplicas, MaxReplicas: maxReplicas, @@ -219,6 +224,7 @@ var _ = Describe("Istio", func() { labels: map[string]string{"some": "labelValue"}, kubeAPIServerPolicyLabel: "to-all-test-kube-apiserver", lbAnnotations: map[string]string{"some": "annotationValue"}, + loadBalancerClass: ptr.To("non-default-load-balancer-class"), externalTrafficPolicy: &trafficPolicy, serviceExternalIP: ptr.To("1.2.3.4"), servicePorts: []corev1.ServicePort{{Port: 443}}, @@ -317,6 +323,7 @@ var _ = Describe("Istio", func() { namespace string annotations map[string]string labels map[string]string + loadBalancerClass *string externalTrafficPolicy corev1.ServiceExternalTrafficPolicy serviceExternalIP *string zone *string @@ -330,6 +337,7 @@ var _ = Describe("Istio", func() { labels = map[string]string{ "additional": "istio-ingress-label", } + loadBalancerClass = ptr.To("non-default-load-balancer-class") externalTrafficPolicy = corev1.ServiceExternalTrafficPolicyCluster serviceExternalIP = ptr.To("1.1.1.1") }) @@ -344,6 +352,7 @@ var _ = Describe("Istio", func() { namespace, annotations, labels, + loadBalancerClass, &externalTrafficPolicy, serviceExternalIP, zone, @@ -365,6 +374,7 @@ var _ = Describe("Istio", func() { namespace, annotations, labels, + loadBalancerClass, &externalTrafficPolicy, serviceExternalIP, zone, @@ -378,6 +388,7 @@ var _ = Describe("Istio", func() { namespace, annotations, labels, + loadBalancerClass, &externalTrafficPolicy, serviceExternalIP, zone, @@ -399,6 +410,7 @@ var _ = Describe("Istio", func() { namespace, annotations, labels, + loadBalancerClass, &externalTrafficPolicy, serviceExternalIP, zone, @@ -412,6 +424,7 @@ var _ = Describe("Istio", func() { namespace, annotations, labels, + loadBalancerClass, &externalTrafficPolicy, serviceExternalIP, zone, @@ -436,6 +449,7 @@ var _ = Describe("Istio", func() { namespace, annotations, labels, + loadBalancerClass, &externalTrafficPolicy, serviceExternalIP, zone, @@ -449,6 +463,7 @@ var _ = Describe("Istio", func() { namespace, annotations, labels, + loadBalancerClass, &externalTrafficPolicy, serviceExternalIP, zone, @@ -471,6 +486,7 @@ var _ = Describe("Istio", func() { namespace, annotations, labels, + loadBalancerClass, &externalTrafficPolicy, serviceExternalIP, zone, @@ -484,6 +500,7 @@ var _ = Describe("Istio", func() { namespace, annotations, labels, + loadBalancerClass, &externalTrafficPolicy, serviceExternalIP, zone, diff --git a/pkg/gardenlet/apis/config/v1alpha1/types.go b/pkg/gardenlet/apis/config/v1alpha1/types.go index d75147ced42..0eecac89e33 100644 --- a/pkg/gardenlet/apis/config/v1alpha1/types.go +++ b/pkg/gardenlet/apis/config/v1alpha1/types.go @@ -671,6 +671,11 @@ type ExposureClassHandler struct { type LoadBalancerServiceConfig struct { // Annotations is a key value map to annotate the underlying load balancer services. Annotations map[string]string `json:"annotations"` + // Class configures the Service.spec.loadBalancerClass field for the load balancer services of the exposure class + // handler. This setting in independent from Seed.spec.settings.loadBalancerServices.class. + // Note that changing the loadBalancerClass of existing LoadBalancer services is denied by Kubernetes. + // +optional + Class *string `json:"class,omitempty"` } // MonitoringConfig contains settings for the monitoring stack. diff --git a/pkg/gardenlet/apis/config/v1alpha1/validation/validation.go b/pkg/gardenlet/apis/config/v1alpha1/validation/validation.go index 9a7e83c192d..b5746b0b21f 100644 --- a/pkg/gardenlet/apis/config/v1alpha1/validation/validation.go +++ b/pkg/gardenlet/apis/config/v1alpha1/validation/validation.go @@ -22,6 +22,7 @@ import ( gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/logger" validationutils "github.com/gardener/gardener/pkg/utils/validation" + kubernetescorevalidation "github.com/gardener/gardener/pkg/utils/validation/kubernetes/core" ) // ValidateGardenletConfiguration validates a GardenletConfiguration object. @@ -106,20 +107,7 @@ func ValidateGardenletConfiguration(cfg *gardenletconfigv1alpha1.GardenletConfig } } - exposureClassHandlersPath := fldPath.Child("exposureClassHandlers") - for i, handler := range cfg.ExposureClassHandlers { - handlerPath := exposureClassHandlersPath.Index(i) - - for _, errorMessage := range validation.IsDNS1123Label(handler.Name) { - allErrs = append(allErrs, field.Invalid(handlerPath.Child("name"), handler.Name, errorMessage)) - } - - if handler.SNI != nil && handler.SNI.Ingress != nil && handler.SNI.Ingress.ServiceExternalIP != nil { - if ip := net.ParseIP(*handler.SNI.Ingress.ServiceExternalIP); ip == nil { - allErrs = append(allErrs, field.Invalid(handlerPath.Child("sni", "ingress", "serviceExternalIP"), handler.SNI.Ingress.ServiceExternalIP, "external service ip is invalid")) - } - } - } + allErrs = append(allErrs, validateExposureClassHandlers(cfg.ExposureClassHandlers, fldPath.Child("exposureClassHandlers"))...) if nodeTolerationCfg := cfg.NodeToleration; nodeTolerationCfg != nil { nodeTolerationConfigPath := fldPath.Child("nodeToleration") @@ -331,3 +319,27 @@ func validateBastionControllerConfiguration(cfg *gardenletconfigv1alpha1.Bastion return allErrs } + +func validateExposureClassHandlers(handlers []gardenletconfigv1alpha1.ExposureClassHandler, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + for i, handler := range handlers { + handlerPath := fldPath.Index(i) + + for _, errorMessage := range validation.IsDNS1123Label(handler.Name) { + allErrs = append(allErrs, field.Invalid(handlerPath.Child("name"), handler.Name, errorMessage)) + } + + if class := handler.LoadBalancerService.Class; class != nil { + allErrs = append(allErrs, kubernetescorevalidation.ValidateQualifiedName(*class, handlerPath.Child("loadBalancerService", "class"))...) + } + + if handler.SNI != nil && handler.SNI.Ingress != nil && handler.SNI.Ingress.ServiceExternalIP != nil { + if ip := net.ParseIP(*handler.SNI.Ingress.ServiceExternalIP); ip == nil { + allErrs = append(allErrs, field.Invalid(handlerPath.Child("sni", "ingress", "serviceExternalIP"), handler.SNI.Ingress.ServiceExternalIP, "external service ip is invalid")) + } + } + } + + return allErrs +} diff --git a/pkg/gardenlet/apis/config/v1alpha1/validation/validation_test.go b/pkg/gardenlet/apis/config/v1alpha1/validation/validation_test.go index f8859c619e6..12339cc0de5 100644 --- a/pkg/gardenlet/apis/config/v1alpha1/validation/validation_test.go +++ b/pkg/gardenlet/apis/config/v1alpha1/validation/validation_test.go @@ -629,6 +629,50 @@ var _ = Describe("GardenletConfiguration", func() { })))) }) + Describe("LoadBalancer class", func() { + It("should allow omitting the class", func() { + cfg.ExposureClassHandlers[0].LoadBalancerService.Class = nil + + Expect(ValidateGardenletConfiguration(cfg, nil)).To(BeEmpty()) + }) + + It("should allow specifying a non-empty class", func() { + cfg.ExposureClassHandlers[0].LoadBalancerService.Class = ptr.To("non-default") + + Expect(ValidateGardenletConfiguration(cfg, nil)).To(BeEmpty()) + }) + + It("should allow specifying a qualified class", func() { + cfg.ExposureClassHandlers[0].LoadBalancerService.Class = ptr.To("stackit.cloud/yawol") + + Expect(ValidateGardenletConfiguration(cfg, nil)).To(BeEmpty()) + }) + + It("should deny specifying an empty class", func() { + cfg.ExposureClassHandlers[0].LoadBalancerService.Class = ptr.To("") + + Expect(ValidateGardenletConfiguration(cfg, nil)).To(ContainElement( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("exposureClassHandlers[0].loadBalancerService.class"), + "Origin": Equal("format=qualified-name"), + })), + )) + }) + + It("should deny specifying a class that Kubernetes does not accept", func() { + cfg.ExposureClassHandlers[0].LoadBalancerService.Class = ptr.To(".invalid-") + + Expect(ValidateGardenletConfiguration(cfg, nil)).To(ContainElement( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("exposureClassHandlers[0].loadBalancerService.class"), + "Origin": Equal("format=qualified-name"), + })), + )) + }) + }) + Context("serviceExternalIP", func() { It("should allow to use an external service ip as loadbalancer ip is valid", func() { cfg.ExposureClassHandlers[0].SNI.Ingress.ServiceExternalIP = ptr.To("1.1.1.1") diff --git a/pkg/gardenlet/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/gardenlet/apis/config/v1alpha1/zz_generated.deepcopy.go index 7a2b5a03cab..164179a0c84 100644 --- a/pkg/gardenlet/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/gardenlet/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -690,6 +690,11 @@ func (in *LoadBalancerServiceConfig) DeepCopyInto(out *LoadBalancerServiceConfig (*out)[key] = val } } + if in.Class != nil { + in, out := &in.Class, &out.Class + *out = new(string) + **out = **in + } return } diff --git a/pkg/gardenlet/controller/seed/seed/components.go b/pkg/gardenlet/controller/seed/seed/components.go index 07c359d637c..a9da60186aa 100644 --- a/pkg/gardenlet/controller/seed/seed/components.go +++ b/pkg/gardenlet/controller/seed/seed/components.go @@ -338,6 +338,7 @@ func (r *Reconciler) newIstio(ctx context.Context, seed *seedpkg.Seed, isGardenC labels, gardenerutils.NetworkPolicyLabel(v1beta1constants.LabelNetworkPolicyShootNamespaceAlias+"-"+v1beta1constants.DeploymentNameKubeAPIServer, kubeapiserverconstants.Port), seed.GetLoadBalancerServiceAnnotations(), + seed.GetLoadBalancerServiceClass(), seed.GetLoadBalancerServiceExternalTrafficPolicy(), r.Config.SNI.Ingress.ServiceExternalIP, servicePorts, @@ -366,6 +367,7 @@ func (r *Reconciler) newIstio(ctx context.Context, seed *seedpkg.Seed, isGardenC sharedcomponent.GetIstioNamespaceForZone(*r.Config.SNI.Ingress.Namespace, zone), seed.GetZonalLoadBalancerServiceAnnotations(zone), sharedcomponent.GetIstioZoneLabels(labels, &zone), + seed.GetLoadBalancerServiceClass(), seed.GetZonalLoadBalancerServiceExternalTrafficPolicy(zone), nil, &zone, @@ -388,6 +390,7 @@ func (r *Reconciler) newIstio(ctx context.Context, seed *seedpkg.Seed, isGardenC // handler.LoadBalancerService.Annotations must put last to override non-exposure class related keys. utils.MergeStringMaps(seed.GetLoadBalancerServiceAnnotations(), handler.LoadBalancerService.Annotations), sharedcomponent.GetIstioZoneLabels(gardenerutils.GetMandatoryExposureClassHandlerSNILabels(handler.SNI.Ingress.Labels, handler.Name), nil), + handler.LoadBalancerService.DeepCopy().Class, seed.GetLoadBalancerServiceExternalTrafficPolicy(), handler.SNI.Ingress.ServiceExternalIP, nil, @@ -410,6 +413,7 @@ func (r *Reconciler) newIstio(ctx context.Context, seed *seedpkg.Seed, isGardenC // handler.LoadBalancerService.Annotations must put last to override non-exposure class related keys. utils.MergeStringMaps(seed.GetZonalLoadBalancerServiceAnnotations(zone), handler.LoadBalancerService.Annotations), sharedcomponent.GetIstioZoneLabels(gardenerutils.GetMandatoryExposureClassHandlerSNILabels(handler.SNI.Ingress.Labels, handler.Name), &zone), + handler.LoadBalancerService.DeepCopy().Class, seed.GetZonalLoadBalancerServiceExternalTrafficPolicy(zone), nil, &zone, diff --git a/pkg/gardenlet/operation/seed/seed.go b/pkg/gardenlet/operation/seed/seed.go index 294cff49381..2c884039265 100644 --- a/pkg/gardenlet/operation/seed/seed.go +++ b/pkg/gardenlet/operation/seed/seed.go @@ -168,6 +168,16 @@ func (s *Seed) GetLoadBalancerServiceAnnotations() map[string]string { return nil } +// GetLoadBalancerServiceClass returns the load balancer class set for the seed if any. +func (s *Seed) GetLoadBalancerServiceClass() *string { + seed := s.GetInfo() + if seed.Spec.Settings != nil && seed.Spec.Settings.LoadBalancerServices != nil { + // return copy to prevent accidental mutation by components + return seed.Spec.Settings.LoadBalancerServices.DeepCopy().Class + } + return nil +} + // GetLoadBalancerServiceExternalTrafficPolicy indicates the external traffic policy for the seed if any. func (s *Seed) GetLoadBalancerServiceExternalTrafficPolicy() *corev1.ServiceExternalTrafficPolicy { seed := s.GetInfo() diff --git a/pkg/gardenlet/operation/seed/seed_test.go b/pkg/gardenlet/operation/seed/seed_test.go index 91c152ddebc..50f092c4e01 100644 --- a/pkg/gardenlet/operation/seed/seed_test.go +++ b/pkg/gardenlet/operation/seed/seed_test.go @@ -9,6 +9,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" . "github.com/gardener/gardener/pkg/gardenlet/operation/seed" @@ -122,6 +123,47 @@ var _ = Describe("seed", func() { }) }) + Describe("#GetLoadBalancerServiceClass", func() { + var seed *Seed + + BeforeEach(func() { + seed = &Seed{} + }) + + It("should return a copied class value if specified", func() { + class := ptr.To("my-load-balancer-class") + + seed.SetInfo(&gardencorev1beta1.Seed{ + Spec: gardencorev1beta1.SeedSpec{ + Settings: &gardencorev1beta1.SeedSettings{ + LoadBalancerServices: &gardencorev1beta1.SeedSettingLoadBalancerServices{ + Class: class, + }, + }, + }, + }) + + Expect(seed.GetLoadBalancerServiceClass()).To(And( + HaveValue(Equal("my-load-balancer-class")), + Not(BeIdenticalTo(class)), + )) + }) + + It("should return nil if class is unspecified", func() { + seed.SetInfo(&gardencorev1beta1.Seed{ + Spec: gardencorev1beta1.SeedSpec{ + Settings: &gardencorev1beta1.SeedSettings{ + LoadBalancerServices: &gardencorev1beta1.SeedSettingLoadBalancerServices{ + Class: nil, + }, + }, + }, + }) + + Expect(seed.GetLoadBalancerServiceClass()).To(BeNil()) + }) + }) + Describe("#GetLoadBalancerServiceExternalTrafficPolicy", func() { It("should return the traffic policy", func() { var ( diff --git a/pkg/operator/controller/garden/garden/components.go b/pkg/operator/controller/garden/garden/components.go index 605870a8363..8d63407e946 100644 --- a/pkg/operator/controller/garden/garden/components.go +++ b/pkg/operator/controller/garden/garden/components.go @@ -907,6 +907,7 @@ func (r *Reconciler) newIstio(ctx context.Context, garden *operatorv1alpha1.Gard annotations, nil, nil, + nil, []corev1.ServicePort{ {Name: "tcp", Port: 443, TargetPort: intstr.FromInt32(9443)}, }, diff --git a/pkg/utils/validation/kubernetes/core/validation.go b/pkg/utils/validation/kubernetes/core/validation.go index e621a760ad6..0b90775c1a1 100644 --- a/pkg/utils/validation/kubernetes/core/validation.go +++ b/pkg/utils/validation/kubernetes/core/validation.go @@ -132,3 +132,13 @@ func ValidateResourceQuantityValue(resource string, value resource.Quantity, fld return allErrs } + +// ValidateQualifiedName validates if value is what Kubernetes calls a "qualified name". +// See https://github.com/kubernetes/kubernetes/blob/v1.34.0/pkg/apis/core/validation/validation.go#L144-L151 +func ValidateQualifiedName(value string, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + for _, msg := range validation.IsQualifiedName(value) { + allErrs = append(allErrs, field.Invalid(fldPath, value, msg).WithOrigin("format=qualified-name")) + } + return allErrs +} diff --git a/pkg/utils/validation/kubernetes/core/validation_test.go b/pkg/utils/validation/kubernetes/core/validation_test.go index cfdf5730b5e..ad2db9bb223 100644 --- a/pkg/utils/validation/kubernetes/core/validation_test.go +++ b/pkg/utils/validation/kubernetes/core/validation_test.go @@ -20,6 +20,8 @@ limitations under the License. package core_test import ( + "strings" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" @@ -153,4 +155,20 @@ var _ = Describe("Validation", func() { "Field": Equal("resourceQuantity"), "Detail": Equal("must be an integer"), }))))) + + DescribeTable("#ValidateQualifiedName", + func(name string, errMatcher types.GomegaMatcher) { + Expect(ValidateQualifiedName(name, field.NewPath("qualifiedName"))).To(errMatcher) + }, + + Entry("valid qualified name (simple)", "my-name", BeEmpty()), + Entry("valid qualified name (with dot)", "my.name", BeEmpty()), + Entry("valid qualified name (with dash)", "my-name.test", BeEmpty()), + Entry("valid qualified name (max length)", strings.Repeat("a", 63), BeEmpty()), + Entry("invalid qualified name (too long)", strings.Repeat("a", 64), Not(BeEmpty())), + Entry("invalid qualified name (invalid char)", "my-name[0]", Not(BeEmpty())), + Entry("invalid qualified name (starts with dot)", ".myname", Not(BeEmpty())), + Entry("invalid qualified name (ends with dash)", "myname-", Not(BeEmpty())), + Entry("invalid qualified name (empty)", "", Not(BeEmpty())), + ) }) From 14928a127ad188f63e7861111ae7d41e2742a192 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 7 Nov 2025 14:51:30 +0100 Subject: [PATCH 056/176] Update quay.io/prometheus/alertmanager Docker tag to v0.29.0 (#13350) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index acfef962dc2..54e1df5d6d3 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -227,7 +227,7 @@ images: - name: alertmanager sourceRepository: github.com/prometheus/alertmanager repository: quay.io/prometheus/alertmanager - tag: v0.28.1 + tag: v0.29.0 labels: - name: gardener.cloud/cve-categorisation value: From 68129d955a8abfd29a18263c4d809f477dd37c12 Mon Sep 17 00:00:00 2001 From: Dominik Froehlich Date: Fri, 7 Nov 2025 16:32:23 +0100 Subject: [PATCH 057/176] feat: add readiness probe to vpn tunnel-controller (#13366) --- pkg/component/networking/vpn/shoot/shoot.go | 13 +++++++++++++ pkg/component/networking/vpn/shoot/shoot_test.go | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/pkg/component/networking/vpn/shoot/shoot.go b/pkg/component/networking/vpn/shoot/shoot.go index ade0f6c9d54..d92d30b0173 100644 --- a/pkg/component/networking/vpn/shoot/shoot.go +++ b/pkg/component/networking/vpn/shoot/shoot.go @@ -667,6 +667,19 @@ func (v *vpnShoot) tunnelControllerContainer() *corev1.Container { Add: []corev1.Capability{"NET_ADMIN"}, }, }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt32(8080), + }, + }, + SuccessThreshold: 2, // Check twice to buy enough time to establish more than one kube apiserver route. + FailureThreshold: 1, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + TimeoutSeconds: 2, + }, } } diff --git a/pkg/component/networking/vpn/shoot/shoot_test.go b/pkg/component/networking/vpn/shoot/shoot_test.go index d6cf28623ea..50c57bd063b 100644 --- a/pkg/component/networking/vpn/shoot/shoot_test.go +++ b/pkg/component/networking/vpn/shoot/shoot_test.go @@ -706,6 +706,19 @@ var _ = Describe("VPNShoot", func() { }, }, ImagePullPolicy: corev1.PullIfNotPresent, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt32(8080), + }, + }, + SuccessThreshold: 2, + FailureThreshold: 1, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + TimeoutSeconds: 2, + }, }) } From 615272edec336264ba884d80594b23d40b6540e9 Mon Sep 17 00:00:00 2001 From: Sebastian Stauch Date: Fri, 7 Nov 2025 16:32:30 +0100 Subject: [PATCH 058/176] Reapply server block imports (#13375) * Reapply "support server block imports for node-local-dns (#13160)" (#13344) This reverts commit f34fc0bfb2d272d02d2afbb0dc24d25c74a2a33f. * update image tage of coredns-config-adapter to v0.4.0 * update documentation --- docs/usage/networking/custom-dns-config.md | 6 +- docs/usage/networking/node-local-dns.md | 2 +- imagevector/containers.go | 2 + imagevector/containers.yaml | 17 +++++ pkg/component/networking/coredns/coredns.go | 5 +- .../networking/nodelocaldns/nodelocaldns.go | 19 +++--- .../nodelocaldns/nodelocaldns_test.go | 53 +++++++++++++++- .../networking/nodelocaldns/resources.go | 63 +++++++++++++++++-- .../operation/botanist/nodelocaldns.go | 20 +++--- .../controller/dnsrecord/actuator.go | 3 +- .../controller/dnsrecord/actuator_test.go | 5 +- skaffold-gardenadm.yaml | 2 + skaffold-operator.yaml | 2 + skaffold.yaml | 2 + 14 files changed, 173 insertions(+), 28 deletions(-) diff --git a/docs/usage/networking/custom-dns-config.md b/docs/usage/networking/custom-dns-config.md index 706f162f69c..fef710d0b2e 100644 --- a/docs/usage/networking/custom-dns-config.md +++ b/docs/usage/networking/custom-dns-config.md @@ -102,9 +102,9 @@ This should bring the cluster DNS back to functioning state. ## Node Local DNS -Starting with Gardener v1.128, custom DNS configurations are fully supported in NodeLocalDNS. In this version, the `coredns-custom` `ConfigMap` is mounted into the NodeLocalDNS pod, allowing custom override and server configurations to be imported into the DNS server. Prior to Gardener v1.128, custom DNS configurations might not function as expected with NodeLocalDNS. -With NodeLocalDNS, ordinary DNS queries targeting upstream DNS servers (i.e., non-Kubernetes domains) are sent directly to the upstream DNS server, bypassing CoreDNS. Therefore, configurations for non-Kubernetes entities, such as the `istio.server` block in the [custom DNS configuration](custom-dns-config.md) example, may not have any effect when NodeLocalDNS is enabled on landscapes with Gardener prior to v1.128. -If you require custom DNS configurations for non-Kubernetes domains, you need to disable forwarding to upstream DNS with Gardener v1.127 and below. This can be done by setting the `disableForwardToUpstreamDNS` option in the Shoot resource to `true`: +Starting with Gardener v1.132, custom DNS configurations are fully supported in NodeLocalDNS. In this version, the `coredns-custom` `ConfigMap` is mounted into the NodeLocalDNS pod, allowing custom override and server configurations to be imported into the DNS server. The server configuration is read by a sidecar container and a new configuration file with the correct bind statement and port mappings is generated and imported into NodeLocalDNS pods. Prior to Gardener v1.132, custom DNS configurations might not function as expected with NodeLocalDNS. +With NodeLocalDNS, ordinary DNS queries targeting upstream DNS servers (i.e., non-Kubernetes domains) are sent directly to the upstream DNS server, bypassing CoreDNS. Therefore, configurations for non-Kubernetes entities, such as the `istio.server` block in the [custom DNS configuration](custom-dns-config.md) example, may not have any effect when NodeLocalDNS is enabled on landscapes with Gardener prior to v1.132. +If you require custom DNS configurations for non-Kubernetes domains, you need to disable forwarding to upstream DNS with Gardener v1.131 and below. This can be done by setting the `disableForwardToUpstreamDNS` option in the Shoot resource to `true`: ```yaml ... spec: diff --git a/docs/usage/networking/node-local-dns.md b/docs/usage/networking/node-local-dns.md index ca95fa01af6..a83340e572f 100644 --- a/docs/usage/networking/node-local-dns.md +++ b/docs/usage/networking/node-local-dns.md @@ -48,5 +48,5 @@ For more information about `node-local-dns`, please refer to the [KEP](https://g ## Known Issues -Custom DNS configuration may not work as expected in conjunction with `NodeLocalDNS` prior to gardener v1.128. +Custom DNS configuration may not work as expected in conjunction with `NodeLocalDNS` prior to gardener v1.132. Please refer to [Custom DNS Configuration](custom-dns-config.md#node-local-dns). diff --git a/imagevector/containers.go b/imagevector/containers.go index c655a64e28e..0ea93ce1eab 100644 --- a/imagevector/containers.go +++ b/imagevector/containers.go @@ -25,6 +25,8 @@ const ( ContainerImageNameConfigmapReloader = "configmap-reloader" // ContainerImageNameCoredns is a constant for an image in the image vector with name 'coredns'. ContainerImageNameCoredns = "coredns" + // ContainerImageNameCorednsConfigAdapter is a constant for an image in the image vector with name 'coredns-config-adapter'. + ContainerImageNameCorednsConfigAdapter = "coredns-config-adapter" // ContainerImageNameCortex is a constant for an image in the image vector with name 'cortex'. ContainerImageNameCortex = "cortex" // ContainerImageNameDependencyWatchdog is a constant for an image in the image vector with name 'dependency-watchdog'. diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 54e1df5d6d3..2c77224573c 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -494,6 +494,23 @@ images: value: - type: 'githubTeam' teamname: 'gardener/gardener-core-networking-maintainers' + - name: coredns-config-adapter + sourceRepository: github.com/gardener/coredns-config-adapter + repository: europe-docker.pkg.dev/gardener-project/releases/gardener/coredns-config-adapter + tag: "v0.4.0" + labels: + - name: 'gardener.cloud/cve-categorisation' + value: + network_exposure: 'private' + authentication_enforced: false + user_interaction: 'end-user' + confidentiality_requirement: 'low' + integrity_requirement: 'high' + availability_requirement: 'high' + - name: 'cloud.gardener.cnudie/responsibles' + value: + - type: 'githubTeam' + teamname: 'gardener/gardener-core-networking-maintainers' - name: node-problem-detector sourceRepository: github.com/kubernetes/node-problem-detector repository: registry.k8s.io/node-problem-detector/node-problem-detector diff --git a/pkg/component/networking/coredns/coredns.go b/pkg/component/networking/coredns/coredns.go index 68175e44986..e5db3bcaa4a 100644 --- a/pkg/component/networking/coredns/coredns.go +++ b/pkg/component/networking/coredns/coredns.go @@ -55,6 +55,9 @@ const ( containerName = "coredns" serviceName = "kube-dns" // this is due to legacy reasons + // CustomConfigMapName is the name of the custom CoreDNS ConfigMap. + CustomConfigMapName = "coredns-custom" + portNameMetrics = "metrics" portMetrics = 9153 @@ -319,7 +322,7 @@ import custom/*.server configMapCustom = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "coredns-custom", + Name: CustomConfigMapName, Namespace: metav1.NamespaceSystem, Annotations: map[string]string{resourcesv1alpha1.Ignore: "true"}, }, diff --git a/pkg/component/networking/nodelocaldns/nodelocaldns.go b/pkg/component/networking/nodelocaldns/nodelocaldns.go index 571c70e10b5..52891283d63 100644 --- a/pkg/component/networking/nodelocaldns/nodelocaldns.go +++ b/pkg/component/networking/nodelocaldns/nodelocaldns.go @@ -64,6 +64,8 @@ const ( metricsPortName = "metrics" errorMetricsPortName = "errormetrics" + sideCarName = "coredns-config-adapter" + domain = gardencorev1beta1.DefaultDomain serviceName = "kube-dns-upstream" livenessProbePort = 8099 @@ -74,13 +76,14 @@ const ( daemonSetPollInterval = 5 * time.Second - volumeMountNameCleanUp = "cleanup-script" - volumeMountPathCleanUp = "/scripts" - volumeMountNameXtablesLock = "xtables-lock" - volumeMountPathXtablesLock = "/run/xtables.lock" - volumeMountPathCustomConfig = "/etc/custom" - volumeMountNameCustomConfig = "custom-config-volume" - customConfigMapName = "coredns-custom" + volumeMountNameCleanUp = "cleanup-script" + volumeMountPathCleanUp = "/scripts" + volumeMountNameXtablesLock = "xtables-lock" + volumeMountPathXtablesLock = "/run/xtables.lock" + volumeMountPathCustomConfig = "/etc/custom" + volumeMountNameCustomConfig = "custom-config-volume" + volumeMountNameGeneratedConfig = "generated-config" + volumeMountPathGeneratedConfig = "/etc/generated-config" ) var ( @@ -103,6 +106,8 @@ type Values struct { Image string // AlpineImage is the container image used for the cleanup DaemonSet. AlpineImage string + // CorednsConfigAdapterImage is the container image used for the coredns config adapter sidecar. + CorednsConfigAdapterImage string // VPAEnabled marks whether VerticalPodAutoscaler is enabled for the shoot. VPAEnabled bool // Config is the node local configuration for the shoot spec diff --git a/pkg/component/networking/nodelocaldns/nodelocaldns_test.go b/pkg/component/networking/nodelocaldns/nodelocaldns_test.go index 3518128667c..2a0ff6eeed3 100644 --- a/pkg/component/networking/nodelocaldns/nodelocaldns_test.go +++ b/pkg/component/networking/nodelocaldns/nodelocaldns_test.go @@ -31,6 +31,7 @@ import ( resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component" + "github.com/gardener/gardener/pkg/component/networking/coredns" . "github.com/gardener/gardener/pkg/component/networking/nodelocaldns" "github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references" "github.com/gardener/gardener/pkg/utils" @@ -350,6 +351,7 @@ data: cache 30 reload } + import generated-config/custom-server-block.server immutable: true kind: ConfigMap metadata: @@ -451,6 +453,41 @@ status: Type: corev1.SeccompProfileTypeRuntimeDefault, }, }, + InitContainers: []corev1.Container{ + { + Name: "coredns-config-adapter", + Image: values.CorednsConfigAdapterImage, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To[int64](65532), + RunAsGroup: ptr.To[int64](65532), + }, + Args: []string{ + "-inputDir=/etc/custom", + "-outputDir=/etc/generated-config", + "-bind=bind " + bindIP(values), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "custom-config-volume", + MountPath: "/etc/custom", + ReadOnly: true, + }, + { + MountPath: "/etc/generated-config", + Name: "generated-config", + }, + }, + RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + }, + }, Containers: []corev1.Container{ { Name: "node-cache", @@ -532,6 +569,10 @@ status: MountPath: "/etc/custom", ReadOnly: true, }, + { + MountPath: "/etc/generated-config", + Name: "generated-config", + }, }, }, }, @@ -577,13 +618,19 @@ status: VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "coredns-custom", + Name: coredns.CustomConfigMapName, }, DefaultMode: ptr.To[int32](420), Optional: ptr.To(true), }, }, }, + { + Name: "generated-config", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, }, }, }, @@ -601,6 +648,8 @@ spec: containerPolicies: - containerName: '*' controlledValues: RequestsOnly + - containerName: coredns-config-adapter + mode: "Off" targetRef: apiVersion: apps/v1 kind: DaemonSet @@ -720,6 +769,7 @@ ip6.arpa:53 { cache 30 reload } +import generated-config/custom-server-block.server `, } configMapHash = utils.ComputeConfigMapChecksum(configMapData)[:8] @@ -994,6 +1044,7 @@ ip6.arpa:53 { cache 30 reload } +import generated-config/custom-server-block.server `, } configMapHash = utils.ComputeConfigMapChecksum(configMapData)[:8] diff --git a/pkg/component/networking/nodelocaldns/resources.go b/pkg/component/networking/nodelocaldns/resources.go index 78fdf7ecd4d..b184a9ef61b 100644 --- a/pkg/component/networking/nodelocaldns/resources.go +++ b/pkg/component/networking/nodelocaldns/resources.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/component/networking/coredns" nodelocaldnsconstants "github.com/gardener/gardener/pkg/component/networking/nodelocaldns/constants" "github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" @@ -100,6 +101,7 @@ ip6.arpa:53 { cache 30 reload } +import generated-config/custom-server-block.server `, }, } @@ -208,6 +210,41 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc v1beta1constants.LabelNodeLocalDNS: "true", v1beta1constants.LabelWorkerPool: worker.Name, }, + InitContainers: []corev1.Container{ + { + Name: sideCarName, + Image: n.values.CorednsConfigAdapterImage, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To[int64](65532), + RunAsGroup: ptr.To[int64](65532), + }, + Args: []string{ + "-inputDir=" + volumeMountPathCustomConfig, + "-outputDir=" + volumeMountPathGeneratedConfig, + "-bind=bind " + n.bindIP(), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: volumeMountNameCustomConfig, + MountPath: volumeMountPathCustomConfig, + ReadOnly: true, + }, + { + MountPath: volumeMountPathGeneratedConfig, + Name: volumeMountNameGeneratedConfig, + }, + }, + RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + }, + }, Containers: []corev1.Container{ { Name: containerName, @@ -289,6 +326,10 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc MountPath: volumeMountPathCustomConfig, ReadOnly: true, }, + { + MountPath: volumeMountPathGeneratedConfig, + Name: volumeMountNameGeneratedConfig, + }, }, }, }, @@ -334,13 +375,19 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: customConfigMapName, + Name: coredns.CustomConfigMapName, }, DefaultMode: ptr.To[int32](420), Optional: ptr.To(true), }, }, }, + { + Name: volumeMountNameGeneratedConfig, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, }, }, }, @@ -366,10 +413,16 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc UpdateMode: &vpaUpdateMode, }, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ - ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{{ - ContainerName: vpaautoscalingv1.DefaultContainerResourcePolicy, - ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), - }}, + ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: vpaautoscalingv1.DefaultContainerResourcePolicy, + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + }, + { + ContainerName: sideCarName, + Mode: ptr.To(vpaautoscalingv1.ContainerScalingModeOff), + }, + }, }, }, } diff --git a/pkg/gardenlet/operation/botanist/nodelocaldns.go b/pkg/gardenlet/operation/botanist/nodelocaldns.go index f1ab2b75b73..4691d8715e3 100644 --- a/pkg/gardenlet/operation/botanist/nodelocaldns.go +++ b/pkg/gardenlet/operation/botanist/nodelocaldns.go @@ -35,17 +35,23 @@ func (b *Botanist) DefaultNodeLocalDNS() (nodelocaldns.Interface, error) { return nil, err } + imageCorednsConfigAdapter, err := imagevector.Containers().FindImage(imagevector.ContainerImageNameCorednsConfigAdapter) + if err != nil { + return nil, err + } + return nodelocaldns.New( b.SeedClientSet.Client(), b.Shoot.ControlPlaneNamespace, nodelocaldns.Values{ - Image: image.String(), - AlpineImage: imageAlpine.String(), - VPAEnabled: b.Shoot.WantsVerticalPodAutoscaler, - Config: v1beta1helper.GetNodeLocalDNS(b.Shoot.GetInfo().Spec.SystemComponents), - Workers: b.Shoot.GetInfo().Spec.Provider.Workers, - KubeProxyConfig: b.Shoot.GetInfo().Spec.Kubernetes.KubeProxy, - Log: b.Logger, + Image: image.String(), + AlpineImage: imageAlpine.String(), + CorednsConfigAdapterImage: imageCorednsConfigAdapter.String(), + VPAEnabled: b.Shoot.WantsVerticalPodAutoscaler, + Config: v1beta1helper.GetNodeLocalDNS(b.Shoot.GetInfo().Spec.SystemComponents), + Workers: b.Shoot.GetInfo().Spec.Provider.Workers, + KubeProxyConfig: b.Shoot.GetInfo().Spec.Kubernetes.KubeProxy, + Log: b.Logger, }, ), nil } diff --git a/pkg/provider-local/controller/dnsrecord/actuator.go b/pkg/provider-local/controller/dnsrecord/actuator.go index a12611e09d7..905d40f3207 100644 --- a/pkg/provider-local/controller/dnsrecord/actuator.go +++ b/pkg/provider-local/controller/dnsrecord/actuator.go @@ -21,6 +21,7 @@ import ( v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/component/networking/coredns" ) // Actuator implements the DNSRecord actuator for the local DNS provider. @@ -85,7 +86,7 @@ func (a *Actuator) Restore(ctx context.Context, log logr.Logger, dnsRecord *exte } func (a *Actuator) patchCoreDNSConfigMap(ctx context.Context, mutate func(configMap *corev1.ConfigMap)) error { - configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "coredns-custom", Namespace: "gardener-extension-provider-local-coredns"}} + configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: coredns.CustomConfigMapName, Namespace: "gardener-extension-provider-local-coredns"}} _, err := controllerutil.CreateOrPatch(ctx, a.Client, configMap, func() error { mutate(configMap) return nil diff --git a/pkg/provider-local/controller/dnsrecord/actuator_test.go b/pkg/provider-local/controller/dnsrecord/actuator_test.go index fe3d9303184..fb6edac999f 100644 --- a/pkg/provider-local/controller/dnsrecord/actuator_test.go +++ b/pkg/provider-local/controller/dnsrecord/actuator_test.go @@ -22,6 +22,7 @@ import ( "github.com/gardener/gardener/extensions/pkg/controller/dnsrecord" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/component/networking/coredns" "github.com/gardener/gardener/pkg/logger" . "github.com/gardener/gardener/pkg/provider-local/controller/dnsrecord" ) @@ -103,14 +104,14 @@ var _ = Describe("Actuator", func() { } emptyConfigMap = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "coredns-custom", + Name: coredns.CustomConfigMapName, Namespace: extensionNamespace.Name, }, Data: map[string]string{"test": "data"}, } configMapWithRule = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "coredns-custom", + Name: coredns.CustomConfigMapName, Namespace: extensionNamespace.Name, }, Data: map[string]string{ diff --git a/skaffold-gardenadm.yaml b/skaffold-gardenadm.yaml index fff45bdb73d..63fecc64b89 100644 --- a/skaffold-gardenadm.yaml +++ b/skaffold-gardenadm.yaml @@ -1132,6 +1132,8 @@ build: - pkg/component/kubernetes/proxy - pkg/component/kubernetes/proxy/resources/cleanup.sh - pkg/component/kubernetes/proxy/resources/conntrack-fix.sh + - pkg/component/networking/coredns + - pkg/component/networking/coredns/constants - pkg/component/nodemanagement/machinecontrollermanager - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml index 7f5dfe7f84f..234857b7643 100644 --- a/skaffold-operator.yaml +++ b/skaffold-operator.yaml @@ -1146,6 +1146,8 @@ build: - pkg/component/kubernetes/proxy - pkg/component/kubernetes/proxy/resources/cleanup.sh - pkg/component/kubernetes/proxy/resources/conntrack-fix.sh + - pkg/component/networking/coredns + - pkg/component/networking/coredns/constants - pkg/component/nodemanagement/machinecontrollermanager - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml diff --git a/skaffold.yaml b/skaffold.yaml index 7bde9200a08..1d8c51412d5 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -740,6 +740,8 @@ build: - pkg/component/kubernetes/proxy - pkg/component/kubernetes/proxy/resources/cleanup.sh - pkg/component/kubernetes/proxy/resources/conntrack-fix.sh + - pkg/component/networking/coredns + - pkg/component/networking/coredns/constants - pkg/component/nodemanagement/machinecontrollermanager - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machineclasses.yaml - pkg/component/nodemanagement/machinecontrollermanager/templates/crd-machine.sapcloud.io_machinedeployments.yaml From 6c2ef38f8f8fff59393c7f1d656b1a12007841e6 Mon Sep 17 00:00:00 2001 From: Jeremy Rickards <9338170+rickardsjp@users.noreply.github.com> Date: Fri, 7 Nov 2025 16:32:39 +0100 Subject: [PATCH 059/176] Stop deploying `cache-node-exporter` ServiceMonitor on unmanaged `Seed`s (#13382) Gardener does not deploy the node-exporter DaemonSet on unmanaged seed clusters, i.e. seed clusters that are not simultaneously also shoot clusters. With this change, it also stops deploying the ServiceMonitor in the `kube-system` namespace that is used by the prometheus-cache to scrape node-exporter instances on managed seeds. --- .../monitoring/prometheus/cache/servicemonitors.go | 13 +++++++++---- .../prometheus/cache/servicemonitors_test.go | 8 ++++++-- pkg/gardenlet/controller/seed/seed/components.go | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pkg/component/observability/monitoring/prometheus/cache/servicemonitors.go b/pkg/component/observability/monitoring/prometheus/cache/servicemonitors.go index 532df2a0a30..b5cac22f417 100644 --- a/pkg/component/observability/monitoring/prometheus/cache/servicemonitors.go +++ b/pkg/component/observability/monitoring/prometheus/cache/servicemonitors.go @@ -12,9 +12,12 @@ import ( ) // CentralServiceMonitors returns the central ServiceMonitor resources for the cache prometheus. -func CentralServiceMonitors() []*monitoringv1.ServiceMonitor { - return []*monitoringv1.ServiceMonitor{ - { +func CentralServiceMonitors(seedIsShoot bool) []*monitoringv1.ServiceMonitor { + var serviceMonitors []*monitoringv1.ServiceMonitor + + if seedIsShoot { + // add cache-node-exporter ServiceMonitor only to ManagedSeeds. + serviceMonitors = append(serviceMonitors, &monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: "node-exporter", Namespace: metav1.NamespaceSystem, @@ -53,6 +56,8 @@ func CentralServiceMonitors() []*monitoringv1.ServiceMonitor { ), }}, }, - }, + }) } + + return serviceMonitors } diff --git a/pkg/component/observability/monitoring/prometheus/cache/servicemonitors_test.go b/pkg/component/observability/monitoring/prometheus/cache/servicemonitors_test.go index 1b61e1f8078..636101f153b 100644 --- a/pkg/component/observability/monitoring/prometheus/cache/servicemonitors_test.go +++ b/pkg/component/observability/monitoring/prometheus/cache/servicemonitors_test.go @@ -16,8 +16,8 @@ import ( var _ = Describe("ServiceMonitors", func() { Describe("#CentralServiceMonitors", func() { - It("should return the expected objects", func() { - Expect(cache.CentralServiceMonitors()).To(HaveExactElements(&monitoringv1.ServiceMonitor{ + It("should return the service monitors for ManagedSeeds", func() { + Expect(cache.CentralServiceMonitors(true)).To(HaveExactElements(&monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: "node-exporter", Namespace: "kube-system", @@ -58,5 +58,9 @@ var _ = Describe("ServiceMonitors", func() { }, })) }) + + It("should not return the service monitors for unmanaged Seeds", func() { + Expect(cache.CentralServiceMonitors(false)).To(BeEmpty()) + }) }) }) diff --git a/pkg/gardenlet/controller/seed/seed/components.go b/pkg/gardenlet/controller/seed/seed/components.go index a9da60186aa..f6df0b2ca97 100644 --- a/pkg/gardenlet/controller/seed/seed/components.go +++ b/pkg/gardenlet/controller/seed/seed/components.go @@ -601,7 +601,7 @@ func (r *Reconciler) newCachePrometheus(log logr.Logger, seed *seedpkg.Seed, see }, CentralConfigs: prometheus.CentralConfigs{ AdditionalScrapeConfigs: additionalScrapeConfigs, - ServiceMonitors: cacheprometheus.CentralServiceMonitors(), + ServiceMonitors: cacheprometheus.CentralServiceMonitors(seedIsShoot), PrometheusRules: cacheprometheus.CentralPrometheusRules(), }, AdditionalResources: []client.Object{ From f3e7bc71fcc43484bbc9c5a25ea91e6cb2a53cd2 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Sat, 8 Nov 2025 10:28:26 +0100 Subject: [PATCH 060/176] [GEP-28] `gardenadm init`: Deploy/restore `Infrastructure` (#13353) * Document new credentials handling in provider-local * Add kind kubeconfig to secret of shoot with managed infra * Use provider client in `Infrastructure` controller * Side: clean up deletion of no longer existing objects * `gardenadm init`: Deploy/restore `Infrastructure` If the shoot's infrastructure is managed by gardener, `gardenadm init` will deploy the `Infrastructure` object into the self-hosted shoot. It restores the extension state from the `ShootState` compiled by `gardenadm bootstrap` from the bootstrap cluster. The shoot networks might only be known after the `Infrastructure` reconciliation, so kube-proxy and `Network` deployment depend on a successful `Infrastructure` reconciliation. * Add e2e test coverage * Rename seedClient to runtimeClient The wording "runtime client" fits more cases while still being simple enough to understand. * Address review suggestions * Rework dev-setup structure as suggested --- .../overlays/managed-infra/kustomization.yaml | 2 +- .../.gitignore | 1 + .../kustomization.yaml | 22 ++++++++++ .../secretbinding.yaml | 9 ++++ docs/extensions/provider-local.md | 9 ++++ hack/kind-up.sh | 7 +++ pkg/gardenadm/cmd/init/init.go | 18 ++++++-- .../controller/infrastructure/actuator.go | 30 +++++++++---- pkg/provider-local/local/client.go | 44 +++++++++++++++++++ test/e2e/gardenadm/managedinfra/gardenadm.go | 23 +++++++++- 10 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/.gitignore create mode 100644 dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/kustomization.yaml create mode 100644 dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/secretbinding.yaml create mode 100644 pkg/provider-local/local/client.go diff --git a/dev-setup/gardenadm/resources/overlays/managed-infra/kustomization.yaml b/dev-setup/gardenadm/resources/overlays/managed-infra/kustomization.yaml index 111f5bf56bc..8a7c4732bf2 100644 --- a/dev-setup/gardenadm/resources/overlays/managed-infra/kustomization.yaml +++ b/dev-setup/gardenadm/resources/overlays/managed-infra/kustomization.yaml @@ -8,7 +8,7 @@ resources: - ../../base components: -- ../../../../gardenconfig/components/credentials/secret-project-garden +- ../../../../gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig patches: - path: patch-shoot-managed-infra.yaml diff --git a/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/.gitignore b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/.gitignore new file mode 100644 index 00000000000..f940e0b554e --- /dev/null +++ b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/.gitignore @@ -0,0 +1 @@ +kubeconfig diff --git a/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/kustomization.yaml b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/kustomization.yaml new file mode 100644 index 00000000000..269acef9d28 --- /dev/null +++ b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/kustomization.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: +- secretbinding.yaml + +secretGenerator: +# Instead of creating an empty "cloudprovider" secret, we create a secret containing the kubeconfig +# for the local kind cluster. If the kubeconfig key in the cloudprovider key is present, provider-local +# uses it to manage infrastructure resources instead of using its in-cluster ServiceAccount. +# See docs/extensions/provider-local.md#credentials for details. +# This is only needed for self-hosted shoots in the local setup because provider-local needs to manage +# Kubernetes resources in a different cluster than the one it is running in. +# Most notably, putting the initial kind cluster kubeconfig into the cloudprovider secret will effectively +# prevent a control plane migration to another kind cluster in the local setup. +# The kubeconfig file is copied to this directory by hack/kind-up.sh. +- name: local + namespace: garden + options: + disableNameSuffixHash: true + files: + - kubeconfig diff --git a/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/secretbinding.yaml b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/secretbinding.yaml new file mode 100644 index 00000000000..0c5e7325d06 --- /dev/null +++ b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/secretbinding.yaml @@ -0,0 +1,9 @@ +apiVersion: core.gardener.cloud/v1beta1 +kind: SecretBinding +metadata: + name: local + namespace: garden +provider: + type: local +secretRef: + name: local diff --git a/docs/extensions/provider-local.md b/docs/extensions/provider-local.md index ea959155e1f..0b3db158d45 100644 --- a/docs/extensions/provider-local.md +++ b/docs/extensions/provider-local.md @@ -63,6 +63,15 @@ This CoreDNS instance is responsible for enabling the components running in the It contains a static configuration to resolve the DNS names based on `local.gardener.cloud` to `istio-ingressgateway.istio-ingress.svc`. +### Credentials + +By default, provider-local doesn't require any cloud provider credentials in the shoot's `{Secret,Credentials}Binding` as all infrastructure resources are deployed in the cluster where it runs. +For this, provider-local uses its in-cluster config to connect to the Kubernetes API server of the seed/bootstrap cluster. +However, provider-local also supports overwriting the credentials by specifying a `kubeconfig` in the shoot credentials secret. +This is used in the local setup for [self-hosted shoots with managed infrastructure](../proposals/28-self-hosted-shoot-clusters.md#managed-infrastructure). +Here, provider-local initially creates the infrastructure resources (e.g., Service, NetworkPolicies, machine Pods, etc.) in the kind cluster (during `gardenadm bootstrap`). +Later on, `gardenadm init` restores the extension resources within the self-hosted shoot cluster itself, and provider-local – running in the shoot cluster now – needs credentials to access the kind cluster API server to keep managing those resources. + ### Controllers There are controllers for all resources in the `extensions.gardener.cloud/v1alpha1` API group except for `BackupBucket` and `BackupEntry`s. diff --git a/hack/kind-up.sh b/hack/kind-up.sh index 9e229967ca3..2d8b9875c1c 100755 --- a/hack/kind-up.sh +++ b/hack/kind-up.sh @@ -327,6 +327,13 @@ if [[ "$KUBECONFIG" != "$PATH_KUBECONFIG" ]]; then fi fi +# Prepare a kubeconfig that can be used by provider-local as the provider credentials to talk to the kind cluster +# from within the kind cluster and also from within a self-hosted shoot. +# See docs/extensions/provider-local.md#credentials. +if [[ "$CLUSTER_NAME" == "gardener-operator-local" ]] ; then + sed "s/127\.0\.0\.1:[0-9]\+/$CLUSTER_NAME-control-plane:6443/g" "$PATH_KUBECONFIG" > "$(dirname "$0")/../dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/kubeconfig" +fi + # Prepare garden.local.gardener.cloud hostname that can be used everywhere to talk to the garden cluster. # Historically, we used the docker container name for this, but this differs between clusters with different names # and doesn't work in IPv6 kind clusters: https://github.com/kubernetes-sigs/kind/issues/3114 diff --git a/pkg/gardenadm/cmd/init/init.go b/pkg/gardenadm/cmd/init/init.go index fb5950fb03f..f429d8df853 100644 --- a/pkg/gardenadm/cmd/init/init.go +++ b/pkg/gardenadm/cmd/init/init.go @@ -84,7 +84,7 @@ func run(ctx context.Context, opts *Options) error { Name: "Deploying control plane namespace", Fn: b.DeployControlPlaneNamespace, }) - _ = g.Add(flow.Task{ + deployCloudProviderSecret = g.Add(flow.Task{ Name: "Deploying cloud provider account secret", Fn: b.DeployCloudProviderSecret, SkipIf: b.Shoot.Credentials == nil, @@ -163,6 +163,18 @@ func run(ctx context.Context, opts *Options) error { Fn: b.ApplyNetworkPolicies, Dependencies: flow.NewTaskIDs(waitUntilGardenerResourceManagerReady, deployExtensionControllers), }) + deployInfrastructure = g.Add(flow.Task{ + Name: "Deploying Shoot infrastructure", + Fn: b.DeployInfrastructure, + SkipIf: !b.Shoot.HasManagedInfrastructure(), + Dependencies: flow.NewTaskIDs(initializeSecretsManagement, deployCloudProviderSecret, waitUntilExtensionControllersReady), + }) + waitUntilInfrastructureReady = g.Add(flow.Task{ + Name: "Waiting until Shoot infrastructure has been reconciled", + Fn: b.WaitForInfrastructure, + SkipIf: !b.Shoot.HasManagedInfrastructure(), + Dependencies: flow.NewTaskIDs(deployInfrastructure), + }) deployShootNamespaces = g.Add(flow.Task{ Name: "Deploying shoot namespaces system component", Fn: b.Shoot.Components.SystemComponents.Namespaces.Deploy, @@ -177,12 +189,12 @@ func run(ctx context.Context, opts *Options) error { Name: "Deploying kube-proxy system component", Fn: b.DeployKubeProxy, SkipIf: !kubeProxyEnabled, - Dependencies: flow.NewTaskIDs(waitUntilShootNamespacesReady, waitUntilExtensionControllersReady), + Dependencies: flow.NewTaskIDs(waitUntilShootNamespacesReady, waitUntilInfrastructureReady), }) deployNetwork = g.Add(flow.Task{ Name: "Deploying shoot network plugin", Fn: b.DeployNetwork, - Dependencies: flow.NewTaskIDs(waitUntilShootNamespacesReady, waitUntilExtensionControllersReady), + Dependencies: flow.NewTaskIDs(waitUntilShootNamespacesReady, waitUntilInfrastructureReady), }) waitUntilNetworkReady = g.Add(flow.Task{ Name: "Waiting until shoot network plugin has been reconciled", diff --git a/pkg/provider-local/controller/infrastructure/actuator.go b/pkg/provider-local/controller/infrastructure/actuator.go index dadcc1808af..3538422b760 100644 --- a/pkg/provider-local/controller/infrastructure/actuator.go +++ b/pkg/provider-local/controller/infrastructure/actuator.go @@ -27,17 +27,25 @@ import ( ) type actuator struct { - client client.Client + // runtimeClient uses provider-local's in-cluster config, e.g., for the seed/bootstrap cluster it runs in. + // It's used to interact with extension objects. By default, it's also used as the provider client to interact with + // infrastructure resources, unless a kubeconfig is specified in the cloudprovider secret. + runtimeClient client.Client } // NewActuator creates a new Actuator that updates the status of the handled Infrastructure resources. func NewActuator(mgr manager.Manager) infrastructure.Actuator { return &actuator{ - client: mgr.GetClient(), + runtimeClient: mgr.GetClient(), } } -func (a *actuator) Reconcile(ctx context.Context, _ logr.Logger, infrastructure *extensionsv1alpha1.Infrastructure, cluster *extensionscontroller.Cluster) error { +func (a *actuator) Reconcile(ctx context.Context, log logr.Logger, infrastructure *extensionsv1alpha1.Infrastructure, cluster *extensionscontroller.Cluster) error { + providerClient, err := local.GetProviderClient(ctx, log, a.runtimeClient, infrastructure.Spec.SecretRef) + if err != nil { + return fmt.Errorf("could not create client for infrastructure resources: %w", err) + } + networkPolicyAllowMachinePods := emptyNetworkPolicy("allow-machine-pods", infrastructure.Namespace) networkPolicyAllowMachinePods.Spec = networkingv1.NetworkPolicySpec{ Ingress: []networkingv1.NetworkPolicyIngressRule{{ @@ -101,7 +109,7 @@ func (a *actuator) Reconcile(ctx context.Context, _ logr.Logger, infrastructure } for _, obj := range objects { - if err := a.client.Patch(ctx, obj, client.Apply, local.FieldOwner, client.ForceOwnership); err != nil { + if err := providerClient.Patch(ctx, obj, client.Apply, local.FieldOwner, client.ForceOwnership); err != nil { return err } } @@ -122,14 +130,18 @@ func (a *actuator) Reconcile(ctx context.Context, _ logr.Logger, infrastructure if services := cluster.Shoot.Spec.Networking.Services; services != nil { infrastructure.Status.Networking.Services = []string{*services} } - return a.client.Status().Patch(ctx, infrastructure, patch) + + return a.runtimeClient.Status().Patch(ctx, infrastructure, patch) } -func (a *actuator) Delete(ctx context.Context, _ logr.Logger, infrastructure *extensionsv1alpha1.Infrastructure, _ *extensionscontroller.Cluster) error { - return kubernetesutils.DeleteObjects(ctx, a.client, +func (a *actuator) Delete(ctx context.Context, log logr.Logger, infrastructure *extensionsv1alpha1.Infrastructure, _ *extensionscontroller.Cluster) error { + providerClient, err := local.GetProviderClient(ctx, log, a.runtimeClient, infrastructure.Spec.SecretRef) + if err != nil { + return fmt.Errorf("could not create client for infrastructure resources: %w", err) + } + + return kubernetesutils.DeleteObjects(ctx, providerClient, emptyNetworkPolicy("allow-machine-pods", infrastructure.Namespace), - emptyNetworkPolicy("allow-to-istio-ingress-gateway", infrastructure.Namespace), - emptyNetworkPolicy("allow-to-provider-local-coredns", infrastructure.Namespace), emptyService(infrastructure.Namespace), &metav1.PartialObjectMetadata{TypeMeta: metav1.TypeMeta{APIVersion: "crd.projectcalico.org/v1", Kind: "IPPool"}, ObjectMeta: metav1.ObjectMeta{Name: IPPoolName(infrastructure.Namespace, string(gardencorev1beta1.IPFamilyIPv4))}}, &metav1.PartialObjectMetadata{TypeMeta: metav1.TypeMeta{APIVersion: "crd.projectcalico.org/v1", Kind: "IPPool"}, ObjectMeta: metav1.ObjectMeta{Name: IPPoolName(infrastructure.Namespace, string(gardencorev1beta1.IPFamilyIPv6))}}, diff --git a/pkg/provider-local/local/client.go b/pkg/provider-local/local/client.go new file mode 100644 index 00000000000..b782254fe84 --- /dev/null +++ b/pkg/provider-local/local/client.go @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package local + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/gardener/gardener/pkg/client/kubernetes" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" +) + +// GetProviderClient returns a Kubernetes client for the cluster in which provider-local should manage infrastructure +// resources, e.g., Services, NetworkPolicies, machine Pods, etc. If the cloudprovider secret contains a kubeconfig, +// a client for that kubeconfig is created. Otherwise, the given client for the runtime cluster is returned. +// See https://github.com/gardener/gardener/blob/master/docs/extensions/provider-local.md#credentials. +func GetProviderClient(ctx context.Context, log logr.Logger, runtimeClient client.Client, secretRef corev1.SecretReference) (client.Client, error) { + cloudProviderSecret, err := kubernetesutils.GetSecretByReference(ctx, runtimeClient, &secretRef) + if err != nil { + return nil, fmt.Errorf("could not retrieve cloudprovider secret: %w", err) + } + + if len(cloudProviderSecret.Data[kubernetes.KubeConfig]) == 0 { + log.Info("Using in-cluster config for provider client as no kubeconfig is specified in the cloudprovider secret") + return runtimeClient, nil + } + + clientSet, err := kubernetes.NewClientFromBytes(cloudProviderSecret.Data[kubernetes.KubeConfig], + kubernetes.WithClientOptions(client.Options{Scheme: kubernetes.SeedScheme}), + kubernetes.WithDisabledCachedClient(), + ) + if err != nil { + return nil, fmt.Errorf("could not create client from cloudprovider secret: %w", err) + } + + log.Info("Using kubeconfig from cloudprovider secret for provider client") + return clientSet.Client(), nil +} diff --git a/test/e2e/gardenadm/managedinfra/gardenadm.go b/test/e2e/gardenadm/managedinfra/gardenadm.go index bc216288ec4..b7670dcd22e 100644 --- a/test/e2e/gardenadm/managedinfra/gardenadm.go +++ b/test/e2e/gardenadm/managedinfra/gardenadm.go @@ -200,7 +200,11 @@ var _ = Describe("gardenadm managed infrastructure scenario tests", Label("garde kubeconfig = strings.ReplaceAll(string(kubeconfigSecret.Data["kubeconfig"]), "api.root.garden.local.gardener.cloud", fmt.Sprintf("localhost:%d", localPort)) }, SpecTimeout(time.Minute)) - var shootClientSet kubernetes.Interface + var ( + shootClientSet kubernetes.Interface + shootKomega Komega + ) + It("should connect to the shoot", func(ctx SpecContext) { By("Forward port to control plane machine pod") fw, err := kubernetes.SetupPortForwarder(portForwardCtx, RuntimeClient.RESTConfig(), technicalID, machinePodName(ctx, technicalID, 0), localPort, 443) @@ -222,6 +226,8 @@ var _ = Describe("gardenadm managed infrastructure scenario tests", Label("garde ) return err }).Should(Succeed()) + + shootKomega = New(shootClientSet.Client()) }, SpecTimeout(time.Minute)) It("should restore all persisted secrets in the shoot", func(ctx SpecContext) { @@ -244,6 +250,21 @@ var _ = Describe("gardenadm managed infrastructure scenario tests", Label("garde Expect(shootmigration.ComparePersistedSecrets(persistedSecrets, shootSecrets)).To(Succeed()) }, SpecTimeout(time.Minute)) + It("should deploy/restore the infrastructure in the shoot", func(ctx SpecContext) { + infra := &extensionsv1alpha1.Infrastructure{ObjectMeta: metav1.ObjectMeta{Name: shootName, Namespace: "kube-system"}} + Eventually(ctx, shootKomega.Object(infra)).Should(BeHealthy(health.CheckExtensionObject)) + }, SpecTimeout(time.Minute)) + + It("should create infrastructure resources in the bootstrap cluster", func(ctx SpecContext) { + // Verify that the infrastructure created in the shoot has corresponding resources in the bootstrap cluster, but + // not in the shoot itself. This is specific to provider-local as the infrastructure resources are managed in the + // bootstrap cluster. This check is unrelated to gardenadm functionality, but serves as an additional verification + // that the functionality of using a non-default provider client in provider-local works as expected. + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "machines", Namespace: technicalID}} + Consistently(ctx, shootKomega.Get(service)).Should(BeNotFoundError()) + Eventually(ctx, Get(service)).Should(Succeed()) + }, SpecTimeout(time.Minute)) + It("should finish successfully", func(ctx SpecContext) { Wait(ctx, session) Eventually(ctx, session.Out).Should(gbytes.Say("work in progress")) From 3918a061e47c841da5f9799a8ec114311a1bfe6c Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 10 Nov 2025 10:57:42 +0100 Subject: [PATCH 061/176] Update module github.com/fluent/fluent-operator/v3 to v3.5.0 (#13306) * Update module github.com/fluent/fluent-operator/v3 to v3.5.0 * chore: Run `make generate` * chore: Remove toolchain from go.mod and keep bump to `go 1.24.5` --------- Co-authored-by: Marc Vornetran --- ...rd-fluentbit.fluent.io_clusterfilters.yaml | 35 +++++--- ...bit.fluent.io_clusterfluentbitconfigs.yaml | 6 +- ...crd-fluentbit.fluent.io_clusterinputs.yaml | 84 +++++++++++++++++-- ...rd-fluentbit.fluent.io_clusteroutputs.yaml | 63 ++++++++++++-- ...rd-fluentbit.fluent.io_clusterparsers.yaml | 13 +++ ...10-crd-fluentbit.fluent.io_collectors.yaml | 4 + .../10-crd-fluentbit.fluent.io_filters.yaml | 35 +++++--- ...-fluentbit.fluent.io_fluentbitconfigs.yaml | 6 +- .../10-crd-fluentbit.fluent.io_outputs.yaml | 63 ++++++++++++-- .../10-crd-fluentbit.fluent.io_parsers.yaml | 13 +++ go.mod | 4 +- go.sum | 4 +- ...rd-fluentbit.fluent.io_clusterfilters.yaml | 35 +++++--- ...bit.fluent.io_clusterfluentbitconfigs.yaml | 6 +- ...crd-fluentbit.fluent.io_clusterinputs.yaml | 84 +++++++++++++++++-- ...rd-fluentbit.fluent.io_clusteroutputs.yaml | 63 ++++++++++++-- ...rd-fluentbit.fluent.io_clusterparsers.yaml | 13 +++ .../crd-fluentbit.fluent.io_collectors.yaml | 4 + .../crd-fluentbit.fluent.io_filters.yaml | 35 +++++--- ...-fluentbit.fluent.io_fluentbitconfigs.yaml | 6 +- .../crd-fluentbit.fluent.io_outputs.yaml | 63 ++++++++++++-- .../crd-fluentbit.fluent.io_parsers.yaml | 13 +++ 22 files changed, 552 insertions(+), 100 deletions(-) diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfilters.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfilters.yaml index 624039566a0..4a6e32eb438 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfilters.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfilters.yaml @@ -85,7 +85,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -131,7 +131,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -279,6 +279,10 @@ spec: Include Kubernetes namespace metadata only and no pod metadata. If this is set, the values of Labels and Annotations are ignored. type: boolean + ownerReferences: + description: Include Kubernetes owner references in the + extra metadata. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. @@ -287,7 +291,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -313,6 +317,11 @@ spec: instead of calling Kube Server API to enhance the log. This could mitigate the Kube API heavy traffic issue for large cluster. type: boolean + useTagForMeta: + description: If true, Kubernetes metadata (e.g., pod_name, + container_name, namespace_name etc) will be extracted + from the tag itself. + type: boolean type: object logToMetrics: description: LogToMetrics defines a Log to Metrics Filter configuration. @@ -387,7 +396,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -425,7 +434,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -548,7 +557,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -671,7 +680,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -711,7 +720,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -751,7 +760,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -788,7 +797,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -826,7 +835,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -864,7 +873,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -901,7 +910,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 2aaeb309652..665720d7141 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -299,6 +299,10 @@ spec: type: string emitterStorageType: type: string + enableChunkTrace: + description: Enable input/output tracing on debug images, controlled + more granualry via the http API + type: boolean flushSeconds: description: Interval to flush output type: number @@ -341,7 +345,7 @@ spec: type: boolean httpListen: description: Address to listen - pattern: ^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}$ + pattern: ^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(([A-Fa-f0-9:]+:+)+[A-Fa-f0-9]*)$ type: string httpPort: description: Port to listen diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterinputs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterinputs.yaml index f3813ba4343..d809f0dea06 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterinputs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterinputs.yaml @@ -91,7 +91,7 @@ spec: format: int32 type: integer tag: - description: Tag name associated to all records comming from this + description: Tag name associated to all records coming from this plugin. type: string type: object @@ -156,7 +156,7 @@ spec: properties: bufferMaxSize: description: |- - Specify maximum buffer memory size used to recieve a forward message. + Specify maximum buffer memory size used to receive a forward message. The value must be according to the Unit Size specification. pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ type: string @@ -188,7 +188,7 @@ spec: a separate thread which helps to desaturate the main pipeline. type: string unixPath: - description: Specify the path to unix socket to recieve a forward + description: Specify the path to unix socket to receive a forward message. If set, Listen and port are ignnored. type: string unixPerm: @@ -353,7 +353,7 @@ spec: description: API Server end-point type: string tag: - description: Tag name associated to all records comming from this + description: Tag name associated to all records coming from this plugin. type: string tlsDebug: @@ -432,7 +432,7 @@ spec: host operating system, default is 5 seconds. type: string tag: - description: Tag name associated to all records comming from this + description: Tag name associated to all records coming from this plugin. type: string type: object @@ -507,7 +507,7 @@ spec: description: 'The interval to scrape metrics, default: 10s' type: string tag: - description: Tag name associated to all records comming from this + description: Tag name associated to all records coming from this plugin type: string type: object @@ -578,6 +578,74 @@ spec: description: Specify the key where the source address will be injected. type: string + tls: + description: Specify TLS connector options. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: |- + Set TLS debug verbosity level. + It accept the following values: 0 (No debug), 1 (Error), 2 (State change), 3 (Informational) and 4 Verbose + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object unixPerm: description: 'If Mode is set to unix_tcp or unix_udp, set the permission of the Unix socket file, default: 0644' @@ -692,6 +760,10 @@ spec: description: Specify the database file to keep track of monitored files and offsets. type: string + dbLocking: + description: Specify that the database will be accessed only by + Fluent Bit. + type: boolean dbSync: description: 'Set a default synchronization (I/O) method. Values: Extra, Full, Normal, Off.' diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusteroutputs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusteroutputs.yaml index f53fd251fb4..051a0b7eac0 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusteroutputs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusteroutputs.yaml @@ -65,6 +65,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -240,6 +252,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName @@ -1981,7 +2005,7 @@ spec: 192.168.1.4:9092.' type: string dynamicTopic: - description: adds unknown topics (found in Topic_Key) to Topics. + description: Adds unknown topics (found in Topic_Key) to Topics. So in Topics only a default topic needs to be configured type: boolean format: @@ -2067,6 +2091,7 @@ spec: type: string region: description: The AWS region. + minLength: 1 type: string roleARN: description: ARN of an IAM role to assume (for cross account access). @@ -2323,6 +2348,21 @@ spec: items: type: string type: array + structuredMetadata: + additionalProperties: + type: string + description: |- + Stream structured metadata for API request. It can be multiple comma separated key=value pairs. + This is used for high cardinality data that isn't suited for using labels. + Only supported in Loki 3.0+ with schema v13 and TSDB storage. + type: object + structuredMetadataKeys: + description: |- + Optional list of record keys that will be placed as structured metadata. + This allows using record accessor patterns (e.g. $kubernetes['pod_name']) to reference record keys. + items: + type: string + type: array tenantID: description: |- Tenant ID used by default to push logs to Loki. @@ -2682,7 +2722,7 @@ spec: type: integer replaceDots: description: When enabled, replace field name dots with underscore, - required by Elasticsearch 2.0-2.3. + required by Opensearch 2.0-2.3. type: boolean suppressTypeName: description: When enabled, mapping types is removed and Type option @@ -2781,12 +2821,12 @@ spec: for the current output logical destination. type: string traceError: - description: When enabled print the elasticsearch API calls to - stdout when elasticsearch returns an error + description: When enabled print the Opensearch API calls to stdout + when Opensearch returns an error type: boolean traceOutput: - description: When enabled print the elasticsearch API calls to - stdout (for diag only) + description: When enabled print the Opensearch API calls to stdout + (for diag only) type: boolean type: description: Type name @@ -3437,6 +3477,11 @@ spec: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean + Workers: + description: Specify number of worker threads to use to output + to S3 + format: int32 + type: integer tls: description: Fluent Bit provides integrated support for Transport Layer Security (TLS) and it predecessor Secure Sockets Layer @@ -3876,7 +3921,7 @@ spec: description: Set resource types of data type: string resourceLabels: - description: Optional list of comma seperated strings. Setting + description: Optional list of comma separated strings. Setting these fields overrides the Stackdriver monitored resource API values items: @@ -3958,6 +4003,10 @@ spec: description: Identifier for a task within a namespace. Required if Resource is generic_task type: string + textPayloadKey: + description: The key used to select the text payload from the + record + type: string workers: description: Number of dedicated threads for the Stackdriver Output Plugin diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterparsers.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterparsers.yaml index 0e2983036e5..55f4e0d914a 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_clusterparsers.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_clusterparsers.yaml @@ -74,7 +74,20 @@ spec: type: object logfmt: description: Logfmt defines logfmt parser configuration. + properties: + timeFormat: + description: Time_Format, eg. %Y-%m-%dT%H:%M:%S %z + type: string + timeKeep: + description: Time_Keep + type: boolean + timeKey: + description: Time_Key + type: string type: object + x-kubernetes-validations: + - message: logfmt must not be null + rule: self != null ltsv: description: LTSV defines ltsv parser configuration. properties: diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_collectors.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_collectors.yaml index c99f8c00ad4..704bde01752 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_collectors.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_collectors.yaml @@ -1502,6 +1502,10 @@ spec: - verbs type: object type: array + replicas: + description: Replicas is the number of collector replicas to deploy + format: int32 + type: integer resources: description: Compute Resources required by container. properties: diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_filters.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_filters.yaml index 55135835718..f21ee911c8e 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_filters.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_filters.yaml @@ -85,7 +85,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -131,7 +131,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -279,6 +279,10 @@ spec: Include Kubernetes namespace metadata only and no pod metadata. If this is set, the values of Labels and Annotations are ignored. type: boolean + ownerReferences: + description: Include Kubernetes owner references in the + extra metadata. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. @@ -287,7 +291,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -313,6 +317,11 @@ spec: instead of calling Kube Server API to enhance the log. This could mitigate the Kube API heavy traffic issue for large cluster. type: boolean + useTagForMeta: + description: If true, Kubernetes metadata (e.g., pod_name, + container_name, namespace_name etc) will be extracted + from the tag itself. + type: boolean type: object logToMetrics: description: LogToMetrics defines a Log to Metrics Filter configuration. @@ -387,7 +396,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -425,7 +434,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -548,7 +557,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -671,7 +680,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -711,7 +720,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -751,7 +760,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -788,7 +797,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -826,7 +835,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -864,7 +873,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -901,7 +910,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbitconfigs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbitconfigs.yaml index aafe67c640f..9a5ef6585db 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbitconfigs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_fluentbitconfigs.yaml @@ -331,6 +331,10 @@ spec: type: string emitterStorageType: type: string + enableChunkTrace: + description: Enable input/output tracing on debug images, controlled + more granualry via the http API + type: boolean flushSeconds: description: Interval to flush output type: number @@ -373,7 +377,7 @@ spec: type: boolean httpListen: description: Address to listen - pattern: ^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}$ + pattern: ^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(([A-Fa-f0-9:]+:+)+[A-Fa-f0-9]*)$ type: string httpPort: description: Port to listen diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_outputs.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_outputs.yaml index 7da1e0b4203..97a511871ba 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_outputs.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_outputs.yaml @@ -65,6 +65,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -240,6 +252,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName @@ -1981,7 +2005,7 @@ spec: 192.168.1.4:9092.' type: string dynamicTopic: - description: adds unknown topics (found in Topic_Key) to Topics. + description: Adds unknown topics (found in Topic_Key) to Topics. So in Topics only a default topic needs to be configured type: boolean format: @@ -2067,6 +2091,7 @@ spec: type: string region: description: The AWS region. + minLength: 1 type: string roleARN: description: ARN of an IAM role to assume (for cross account access). @@ -2323,6 +2348,21 @@ spec: items: type: string type: array + structuredMetadata: + additionalProperties: + type: string + description: |- + Stream structured metadata for API request. It can be multiple comma separated key=value pairs. + This is used for high cardinality data that isn't suited for using labels. + Only supported in Loki 3.0+ with schema v13 and TSDB storage. + type: object + structuredMetadataKeys: + description: |- + Optional list of record keys that will be placed as structured metadata. + This allows using record accessor patterns (e.g. $kubernetes['pod_name']) to reference record keys. + items: + type: string + type: array tenantID: description: |- Tenant ID used by default to push logs to Loki. @@ -2682,7 +2722,7 @@ spec: type: integer replaceDots: description: When enabled, replace field name dots with underscore, - required by Elasticsearch 2.0-2.3. + required by Opensearch 2.0-2.3. type: boolean suppressTypeName: description: When enabled, mapping types is removed and Type option @@ -2781,12 +2821,12 @@ spec: for the current output logical destination. type: string traceError: - description: When enabled print the elasticsearch API calls to - stdout when elasticsearch returns an error + description: When enabled print the Opensearch API calls to stdout + when Opensearch returns an error type: boolean traceOutput: - description: When enabled print the elasticsearch API calls to - stdout (for diag only) + description: When enabled print the Opensearch API calls to stdout + (for diag only) type: boolean type: description: Type name @@ -3437,6 +3477,11 @@ spec: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean + Workers: + description: Specify number of worker threads to use to output + to S3 + format: int32 + type: integer tls: description: Fluent Bit provides integrated support for Transport Layer Security (TLS) and it predecessor Secure Sockets Layer @@ -3876,7 +3921,7 @@ spec: description: Set resource types of data type: string resourceLabels: - description: Optional list of comma seperated strings. Setting + description: Optional list of comma separated strings. Setting these fields overrides the Stackdriver monitored resource API values items: @@ -3958,6 +4003,10 @@ spec: description: Identifier for a task within a namespace. Required if Resource is generic_task type: string + textPayloadKey: + description: The key used to select the text payload from the + record + type: string workers: description: Number of dedicated threads for the Stackdriver Output Plugin diff --git a/example/seed-crds/10-crd-fluentbit.fluent.io_parsers.yaml b/example/seed-crds/10-crd-fluentbit.fluent.io_parsers.yaml index 6bfe5e95284..1c117577242 100644 --- a/example/seed-crds/10-crd-fluentbit.fluent.io_parsers.yaml +++ b/example/seed-crds/10-crd-fluentbit.fluent.io_parsers.yaml @@ -74,7 +74,20 @@ spec: type: object logfmt: description: Logfmt defines logfmt parser configuration. + properties: + timeFormat: + description: Time_Format, eg. %Y-%m-%dT%H:%M:%S %z + type: string + timeKeep: + description: Time_Keep + type: boolean + timeKey: + description: Time_Key + type: string type: object + x-kubernetes-validations: + - message: logfmt must not be null + rule: self != null ltsv: description: LTSV defines ltsv parser configuration. properties: diff --git a/go.mod b/go.mod index 77e2501e53a..73870b6409e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/gardener/gardener -go 1.24.0 +go 1.24.5 require ( github.com/Masterminds/semver/v3 v3.4.0 @@ -12,7 +12,7 @@ require ( github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 github.com/docker/cli v28.5.2+incompatible - github.com/fluent/fluent-operator/v3 v3.3.0 + github.com/fluent/fluent-operator/v3 v3.5.0 github.com/gardener/cert-management v0.19.0 github.com/gardener/dependency-watchdog v1.6.0 github.com/gardener/etcd-druid/api v0.33.0 diff --git a/go.sum b/go.sum index 10ee032a51a..3fcce88f487 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,8 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fluent/fluent-operator/v3 v3.3.0 h1:zBtt8IOVSyTiywnmom3V2byqIi2ZXMCCKBUx/4bnFBk= -github.com/fluent/fluent-operator/v3 v3.3.0/go.mod h1:x54zzJ60QYJ6jnN7n9/Mseyaz9oWjSO99hbhVXJaar0= +github.com/fluent/fluent-operator/v3 v3.5.0 h1:soNOaXLmN7VQg1mlHDDKGQ9itYpvgycWstMqXvOD66g= +github.com/fluent/fluent-operator/v3 v3.5.0/go.mod h1:6phBFSu5/+81dbwfizIprNirr7VlbnkhcawbSe7r7NM= github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfilters.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfilters.yaml index 624039566a0..4a6e32eb438 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfilters.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfilters.yaml @@ -85,7 +85,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -131,7 +131,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -279,6 +279,10 @@ spec: Include Kubernetes namespace metadata only and no pod metadata. If this is set, the values of Labels and Annotations are ignored. type: boolean + ownerReferences: + description: Include Kubernetes owner references in the + extra metadata. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. @@ -287,7 +291,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -313,6 +317,11 @@ spec: instead of calling Kube Server API to enhance the log. This could mitigate the Kube API heavy traffic issue for large cluster. type: boolean + useTagForMeta: + description: If true, Kubernetes metadata (e.g., pod_name, + container_name, namespace_name etc) will be extracted + from the tag itself. + type: boolean type: object logToMetrics: description: LogToMetrics defines a Log to Metrics Filter configuration. @@ -387,7 +396,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -425,7 +434,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -548,7 +557,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -671,7 +680,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -711,7 +720,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -751,7 +760,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -788,7 +797,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -826,7 +835,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -864,7 +873,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -901,7 +910,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 2aaeb309652..665720d7141 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -299,6 +299,10 @@ spec: type: string emitterStorageType: type: string + enableChunkTrace: + description: Enable input/output tracing on debug images, controlled + more granualry via the http API + type: boolean flushSeconds: description: Interval to flush output type: number @@ -341,7 +345,7 @@ spec: type: boolean httpListen: description: Address to listen - pattern: ^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}$ + pattern: ^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(([A-Fa-f0-9:]+:+)+[A-Fa-f0-9]*)$ type: string httpPort: description: Port to listen diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterinputs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterinputs.yaml index f3813ba4343..d809f0dea06 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterinputs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterinputs.yaml @@ -91,7 +91,7 @@ spec: format: int32 type: integer tag: - description: Tag name associated to all records comming from this + description: Tag name associated to all records coming from this plugin. type: string type: object @@ -156,7 +156,7 @@ spec: properties: bufferMaxSize: description: |- - Specify maximum buffer memory size used to recieve a forward message. + Specify maximum buffer memory size used to receive a forward message. The value must be according to the Unit Size specification. pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ type: string @@ -188,7 +188,7 @@ spec: a separate thread which helps to desaturate the main pipeline. type: string unixPath: - description: Specify the path to unix socket to recieve a forward + description: Specify the path to unix socket to receive a forward message. If set, Listen and port are ignnored. type: string unixPerm: @@ -353,7 +353,7 @@ spec: description: API Server end-point type: string tag: - description: Tag name associated to all records comming from this + description: Tag name associated to all records coming from this plugin. type: string tlsDebug: @@ -432,7 +432,7 @@ spec: host operating system, default is 5 seconds. type: string tag: - description: Tag name associated to all records comming from this + description: Tag name associated to all records coming from this plugin. type: string type: object @@ -507,7 +507,7 @@ spec: description: 'The interval to scrape metrics, default: 10s' type: string tag: - description: Tag name associated to all records comming from this + description: Tag name associated to all records coming from this plugin type: string type: object @@ -578,6 +578,74 @@ spec: description: Specify the key where the source address will be injected. type: string + tls: + description: Specify TLS connector options. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: |- + Set TLS debug verbosity level. + It accept the following values: 0 (No debug), 1 (Error), 2 (State change), 3 (Informational) and 4 Verbose + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object unixPerm: description: 'If Mode is set to unix_tcp or unix_udp, set the permission of the Unix socket file, default: 0644' @@ -692,6 +760,10 @@ spec: description: Specify the database file to keep track of monitored files and offsets. type: string + dbLocking: + description: Specify that the database will be accessed only by + Fluent Bit. + type: boolean dbSync: description: 'Set a default synchronization (I/O) method. Values: Extra, Full, Normal, Off.' diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusteroutputs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusteroutputs.yaml index f53fd251fb4..051a0b7eac0 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusteroutputs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusteroutputs.yaml @@ -65,6 +65,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -240,6 +252,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName @@ -1981,7 +2005,7 @@ spec: 192.168.1.4:9092.' type: string dynamicTopic: - description: adds unknown topics (found in Topic_Key) to Topics. + description: Adds unknown topics (found in Topic_Key) to Topics. So in Topics only a default topic needs to be configured type: boolean format: @@ -2067,6 +2091,7 @@ spec: type: string region: description: The AWS region. + minLength: 1 type: string roleARN: description: ARN of an IAM role to assume (for cross account access). @@ -2323,6 +2348,21 @@ spec: items: type: string type: array + structuredMetadata: + additionalProperties: + type: string + description: |- + Stream structured metadata for API request. It can be multiple comma separated key=value pairs. + This is used for high cardinality data that isn't suited for using labels. + Only supported in Loki 3.0+ with schema v13 and TSDB storage. + type: object + structuredMetadataKeys: + description: |- + Optional list of record keys that will be placed as structured metadata. + This allows using record accessor patterns (e.g. $kubernetes['pod_name']) to reference record keys. + items: + type: string + type: array tenantID: description: |- Tenant ID used by default to push logs to Loki. @@ -2682,7 +2722,7 @@ spec: type: integer replaceDots: description: When enabled, replace field name dots with underscore, - required by Elasticsearch 2.0-2.3. + required by Opensearch 2.0-2.3. type: boolean suppressTypeName: description: When enabled, mapping types is removed and Type option @@ -2781,12 +2821,12 @@ spec: for the current output logical destination. type: string traceError: - description: When enabled print the elasticsearch API calls to - stdout when elasticsearch returns an error + description: When enabled print the Opensearch API calls to stdout + when Opensearch returns an error type: boolean traceOutput: - description: When enabled print the elasticsearch API calls to - stdout (for diag only) + description: When enabled print the Opensearch API calls to stdout + (for diag only) type: boolean type: description: Type name @@ -3437,6 +3477,11 @@ spec: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean + Workers: + description: Specify number of worker threads to use to output + to S3 + format: int32 + type: integer tls: description: Fluent Bit provides integrated support for Transport Layer Security (TLS) and it predecessor Secure Sockets Layer @@ -3876,7 +3921,7 @@ spec: description: Set resource types of data type: string resourceLabels: - description: Optional list of comma seperated strings. Setting + description: Optional list of comma separated strings. Setting these fields overrides the Stackdriver monitored resource API values items: @@ -3958,6 +4003,10 @@ spec: description: Identifier for a task within a namespace. Required if Resource is generic_task type: string + textPayloadKey: + description: The key used to select the text payload from the + record + type: string workers: description: Number of dedicated threads for the Stackdriver Output Plugin diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterparsers.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterparsers.yaml index 0e2983036e5..55f4e0d914a 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterparsers.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_clusterparsers.yaml @@ -74,7 +74,20 @@ spec: type: object logfmt: description: Logfmt defines logfmt parser configuration. + properties: + timeFormat: + description: Time_Format, eg. %Y-%m-%dT%H:%M:%S %z + type: string + timeKeep: + description: Time_Keep + type: boolean + timeKey: + description: Time_Key + type: string type: object + x-kubernetes-validations: + - message: logfmt must not be null + rule: self != null ltsv: description: LTSV defines ltsv parser configuration. properties: diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_collectors.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_collectors.yaml index c99f8c00ad4..704bde01752 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_collectors.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_collectors.yaml @@ -1502,6 +1502,10 @@ spec: - verbs type: object type: array + replicas: + description: Replicas is the number of collector replicas to deploy + format: int32 + type: integer resources: description: Compute Resources required by container. properties: diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_filters.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_filters.yaml index 55135835718..f21ee911c8e 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_filters.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_filters.yaml @@ -85,7 +85,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -131,7 +131,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -279,6 +279,10 @@ spec: Include Kubernetes namespace metadata only and no pod metadata. If this is set, the values of Labels and Annotations are ignored. type: boolean + ownerReferences: + description: Include Kubernetes owner references in the + extra metadata. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. @@ -287,7 +291,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -313,6 +317,11 @@ spec: instead of calling Kube Server API to enhance the log. This could mitigate the Kube API heavy traffic issue for large cluster. type: boolean + useTagForMeta: + description: If true, Kubernetes metadata (e.g., pod_name, + container_name, namespace_name etc) will be extracted + from the tag itself. + type: boolean type: object logToMetrics: description: LogToMetrics defines a Log to Metrics Filter configuration. @@ -387,7 +396,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -425,7 +434,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -548,7 +557,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -671,7 +680,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -711,7 +720,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -751,7 +760,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -788,7 +797,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -826,7 +835,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -864,7 +873,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the @@ -901,7 +910,7 @@ spec: retryLimit: description: 'RetryLimit describes how many times fluent-bit should retry to send data to a specific output. If set - to false fluent-bit will try indefinetly. If set to any + to false fluent-bit will try indefinitely. If set to any integer N>0 it will try at most N+1 times. Leading zeros are not allowed (values such as 007, 0150, 01 do not work). If this property is not defined fluent-bit will use the diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbitconfigs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbitconfigs.yaml index aafe67c640f..9a5ef6585db 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbitconfigs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_fluentbitconfigs.yaml @@ -331,6 +331,10 @@ spec: type: string emitterStorageType: type: string + enableChunkTrace: + description: Enable input/output tracing on debug images, controlled + more granualry via the http API + type: boolean flushSeconds: description: Interval to flush output type: number @@ -373,7 +377,7 @@ spec: type: boolean httpListen: description: Address to listen - pattern: ^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}$ + pattern: ^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(([A-Fa-f0-9:]+:+)+[A-Fa-f0-9]*)$ type: string httpPort: description: Port to listen diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_outputs.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_outputs.yaml index 7da1e0b4203..97a511871ba 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_outputs.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_outputs.yaml @@ -65,6 +65,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -240,6 +252,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName @@ -1981,7 +2005,7 @@ spec: 192.168.1.4:9092.' type: string dynamicTopic: - description: adds unknown topics (found in Topic_Key) to Topics. + description: Adds unknown topics (found in Topic_Key) to Topics. So in Topics only a default topic needs to be configured type: boolean format: @@ -2067,6 +2091,7 @@ spec: type: string region: description: The AWS region. + minLength: 1 type: string roleARN: description: ARN of an IAM role to assume (for cross account access). @@ -2323,6 +2348,21 @@ spec: items: type: string type: array + structuredMetadata: + additionalProperties: + type: string + description: |- + Stream structured metadata for API request. It can be multiple comma separated key=value pairs. + This is used for high cardinality data that isn't suited for using labels. + Only supported in Loki 3.0+ with schema v13 and TSDB storage. + type: object + structuredMetadataKeys: + description: |- + Optional list of record keys that will be placed as structured metadata. + This allows using record accessor patterns (e.g. $kubernetes['pod_name']) to reference record keys. + items: + type: string + type: array tenantID: description: |- Tenant ID used by default to push logs to Loki. @@ -2682,7 +2722,7 @@ spec: type: integer replaceDots: description: When enabled, replace field name dots with underscore, - required by Elasticsearch 2.0-2.3. + required by Opensearch 2.0-2.3. type: boolean suppressTypeName: description: When enabled, mapping types is removed and Type option @@ -2781,12 +2821,12 @@ spec: for the current output logical destination. type: string traceError: - description: When enabled print the elasticsearch API calls to - stdout when elasticsearch returns an error + description: When enabled print the Opensearch API calls to stdout + when Opensearch returns an error type: boolean traceOutput: - description: When enabled print the elasticsearch API calls to - stdout (for diag only) + description: When enabled print the Opensearch API calls to stdout + (for diag only) type: boolean type: description: Type name @@ -3437,6 +3477,11 @@ spec: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean + Workers: + description: Specify number of worker threads to use to output + to S3 + format: int32 + type: integer tls: description: Fluent Bit provides integrated support for Transport Layer Security (TLS) and it predecessor Secure Sockets Layer @@ -3876,7 +3921,7 @@ spec: description: Set resource types of data type: string resourceLabels: - description: Optional list of comma seperated strings. Setting + description: Optional list of comma separated strings. Setting these fields overrides the Stackdriver monitored resource API values items: @@ -3958,6 +4003,10 @@ spec: description: Identifier for a task within a namespace. Required if Resource is generic_task type: string + textPayloadKey: + description: The key used to select the text payload from the + record + type: string workers: description: Number of dedicated threads for the Stackdriver Output Plugin diff --git a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_parsers.yaml b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_parsers.yaml index 6bfe5e95284..1c117577242 100644 --- a/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_parsers.yaml +++ b/pkg/component/observability/logging/fluentoperator/assets/crd-fluentbit.fluent.io_parsers.yaml @@ -74,7 +74,20 @@ spec: type: object logfmt: description: Logfmt defines logfmt parser configuration. + properties: + timeFormat: + description: Time_Format, eg. %Y-%m-%dT%H:%M:%S %z + type: string + timeKeep: + description: Time_Keep + type: boolean + timeKey: + description: Time_Key + type: string type: object + x-kubernetes-validations: + - message: logfmt must not be null + rule: self != null ltsv: description: LTSV defines ltsv parser configuration. properties: From be49ff2e41643c1d735de889812c771513f69a44 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 10 Nov 2025 10:57:49 +0100 Subject: [PATCH 062/176] Update module github.com/containerd/containerd to v2 (#13311) * Update module github.com/containerd/containerd to v2 * chore: Update containerd to v2.1.4 * chore: Adapt import paths for containerd/v2 * chore: Run `go mod tidy` * chore: Import github.com/containerd/errdefs instead of full containerd/v1 module (PR review) * chore: Run `go mod tidy` --------- Co-authored-by: Marc Vornetran --- go.mod | 36 ++++---- go.sum | 82 +++++++++---------- pkg/nodeagent/controller/healthcheck/add.go | 6 +- .../controller/healthcheck/containerd.go | 2 +- .../registry/containerd_extractor.go | 20 ++--- .../nodeagent/healthcheck/healthcheck_test.go | 2 +- 6 files changed, 72 insertions(+), 76 deletions(-) diff --git a/go.mod b/go.mod index 73870b6409e..58e043e227c 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,8 @@ require ( github.com/ahmetb/gen-crd-api-reference-docs v0.3.0 github.com/andybalholm/brotli v1.2.0 github.com/bramvdbogaerde/go-scp v1.5.0 - github.com/containerd/containerd v1.7.29 + github.com/containerd/containerd/v2 v2.1.4 + github.com/containerd/errdefs v1.0.0 github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 github.com/docker/cli v28.5.2+incompatible @@ -24,7 +25,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/gnostic-models v0.7.0 github.com/google/go-cmp v0.7.0 - github.com/google/go-containerregistry v0.20.0 + github.com/google/go-containerregistry v0.20.1 github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/ironcore-dev/vgopath v0.1.5 @@ -90,13 +91,11 @@ require ( require ( cel.dev/expr v0.24.0 // indirect dario.cat/mergo v1.0.2 // indirect - github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect - github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/BurntSushi/toml v1.5.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Microsoft/hcsshim v0.11.7 // indirect + github.com/Microsoft/hcsshim v0.13.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/PaesslerAG/gval v1.2.4 // indirect github.com/PaesslerAG/jsonpath v0.1.2-0.20240726212847-3a740cf7976f // indirect @@ -108,16 +107,18 @@ require ( github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/cgroups v1.1.0 // indirect - github.com/containerd/containerd/api v1.8.0 // indirect - github.com/containerd/continuity v0.4.4 // indirect - github.com/containerd/errdefs v0.3.0 // indirect + github.com/containerd/cgroups/v3 v3.0.5 // indirect + github.com/containerd/containerd v1.7.27 // indirect + github.com/containerd/containerd/api v1.9.0 // indirect + github.com/containerd/continuity v0.4.5 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/platforms v0.2.1 // indirect + github.com/containerd/platforms v1.0.0-rc.1 // indirect + github.com/containerd/plugin v1.0.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/containerd/ttrpc v1.2.7 // indirect - github.com/containerd/typeurl/v2 v2.1.1 // indirect + github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect @@ -183,10 +184,10 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.5.0 // indirect - github.com/moby/sys/mountinfo v0.7.1 // indirect - github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/sys/signal v0.7.0 // indirect - github.com/moby/sys/user v0.3.0 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/signal v0.7.1 // indirect + github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -197,8 +198,8 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/nexucis/lamenv v0.5.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/runtime-spec v1.1.0 // indirect - github.com/opencontainers/selinux v1.11.0 // indirect + github.com/opencontainers/runtime-spec v1.2.1 // indirect + github.com/opencontainers/selinux v1.12.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/perses/common v0.27.1-0.20250326140707-96e439b14e0e // indirect github.com/perses/perses v0.51.0 // indirect @@ -266,7 +267,6 @@ require ( golang.org/x/sys v0.37.0 // indirect golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect golang.org/x/term v0.36.0 // indirect - google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect google.golang.org/grpc v1.76.0 // indirect diff --git a/go.sum b/go.sum index 3fcce88f487..a1b9dea6e15 100644 --- a/go.sum +++ b/go.sum @@ -14,7 +14,7 @@ cloud.google.com/go/auth v0.13.0/go.mod h1:COOjD9gwfKNKz+IIduatIhYJQIc0mG3H102r/ cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU= cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/compute v1.28.0 h1:OPtBxMcheSS+DWfci803qvPly3d4w7Eu5ztKBcFfzwk= +cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= @@ -23,10 +23,8 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= -github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.1 h1:DSDNVxqkoXJiko6x8a90zidoYqnYYa6c1MTzDKzKkTo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.1/go.mod h1:zGqV2R4Cr/k8Uye5w+dgQ06WJtEcbQG/8J7BB6hnCr4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 h1:F0gBpfdPLGsw+nsgk6aqqkZS1jiixa5WwFe3fk/T3Ys= @@ -67,8 +65,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= -github.com/Microsoft/hcsshim v0.11.7/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= +github.com/Microsoft/hcsshim v0.13.0 h1:/BcXOiS6Qi7N9XqUcv27vkIuVOkBEcWstd2pMlWSeaA= +github.com/Microsoft/hcsshim v0.13.0/go.mod h1:9KWJ/8DgU+QzYGupX4tzMhRQE8h6w90lH6HAaclpEok= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -131,28 +129,34 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= -github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/containerd v1.7.29 h1:90fWABQsaN9mJhGkoVnuzEY+o1XDPbg9BTC9QTAHnuE= -github.com/containerd/containerd v1.7.29/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs= -github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= -github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= -github.com/containerd/continuity v0.4.4 h1:/fNVfTJ7wIl/YPMHjf+5H32uFhl63JucB34PlCpMKII= -github.com/containerd/continuity v0.4.4/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= -github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4= -github.com/containerd/errdefs v0.3.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= +github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= +github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII= +github.com/containerd/containerd v1.7.27/go.mod h1:xZmPnl75Vc+BLGt4MIfu6bp+fy03gdHAn9bz+FreFR0= +github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= +github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= +github.com/containerd/containerd/v2 v2.1.4 h1:/hXWjiSFd6ftrBOBGfAZ6T30LJcx1dBjdKEeI8xucKQ= +github.com/containerd/containerd/v2 v2.1.4/go.mod h1:8C5QV9djwsYDNhxfTCFjWtTBZrqjditQ4/ghHSYjnHM= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= -github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/containerd/platforms v1.0.0-rc.1 h1:83KIq4yy1erSRgOVHNk1HYdPvzdJ5CnsWaRoJX4C41E= +github.com/containerd/platforms v1.0.0-rc.1/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4= +github.com/containerd/plugin v1.0.0 h1:c8Kf1TNl6+e2TtMHZt+39yAPDbouRH9WAToRjex483Y= +github.com/containerd/plugin v1.0.0/go.mod h1:hQfJe5nmWfImiqT1q8Si3jLv3ynMUIBB47bQ+KexvO8= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= -github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= -github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40= +github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= @@ -335,7 +339,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -352,12 +355,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.20.0 h1:wRqHpOeVh3DnenOrPy9xDOLdnLatiGuuNRVelR2gSbg= -github.com/google/go-containerregistry v0.20.0/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/go-containerregistry v0.20.1 h1:eTgx9QNYugV4DN5mz4U8hiAGTi1ybXn0TPi4Smd8du0= +github.com/google/go-containerregistry v0.20.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -531,14 +533,14 @@ github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= -github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= -github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= -github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= -github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= -github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0= +github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= @@ -582,10 +584,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= -github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= -github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= +github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.12.0 h1:6n5JV4Cf+4y0KNXW48TLj5DwfXpvWlxXplUkdTrmPb8= +github.com/opencontainers/selinux v1.12.0/go.mod h1:BTPX+bjVbWGXw7ZZWUbdENt8w0htPSrlgOOysQaU62U= github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e h1:cxgCNo/R769CO23AK5TCh45H9SMUGZ8RukiF2/Qif3o= github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= github.com/operator-framework/operator-lib v0.18.0 h1:6OaWemt/CuyrjFMkLyk4O8Vj4CPHxt/m1DMuMAmPwXo= @@ -933,8 +935,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1017,8 +1017,6 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eYDVe+YvyR7Yc2vJGkO5/0UxD0/jZU= -google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:hL97c3SYopEHblzpxRL4lSs523++l8DYxGM1FQiYmb4= google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= @@ -1043,8 +1041,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/pkg/nodeagent/controller/healthcheck/add.go b/pkg/nodeagent/controller/healthcheck/add.go index 5b6747d13e0..55214859649 100644 --- a/pkg/nodeagent/controller/healthcheck/add.go +++ b/pkg/nodeagent/controller/healthcheck/add.go @@ -9,9 +9,9 @@ import ( "net" "os" - "github.com/containerd/containerd" - "github.com/containerd/containerd/defaults" - "github.com/containerd/containerd/namespaces" + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/defaults" + "github.com/containerd/containerd/v2/pkg/namespaces" corev1 "k8s.io/api/core/v1" "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/builder" diff --git a/pkg/nodeagent/controller/healthcheck/containerd.go b/pkg/nodeagent/controller/healthcheck/containerd.go index 1bdcbf68968..aebd6b43494 100644 --- a/pkg/nodeagent/controller/healthcheck/containerd.go +++ b/pkg/nodeagent/controller/healthcheck/containerd.go @@ -9,7 +9,7 @@ import ( "fmt" "time" - "github.com/containerd/containerd" + containerd "github.com/containerd/containerd/v2/client" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" "k8s.io/utils/clock" diff --git a/pkg/nodeagent/registry/containerd_extractor.go b/pkg/nodeagent/registry/containerd_extractor.go index 5631ed1b7b6..324fa225bf2 100644 --- a/pkg/nodeagent/registry/containerd_extractor.go +++ b/pkg/nodeagent/registry/containerd_extractor.go @@ -10,14 +10,14 @@ import ( "os" "path" - "github.com/containerd/containerd" - "github.com/containerd/containerd/defaults" - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/mount" - "github.com/containerd/containerd/namespaces" - "github.com/containerd/containerd/remotes/docker" - "github.com/containerd/containerd/remotes/docker/config" - "github.com/containerd/containerd/snapshots" + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/containerd/v2/core/remotes/docker" + "github.com/containerd/containerd/v2/core/remotes/docker/config" + "github.com/containerd/containerd/v2/core/snapshots" + "github.com/containerd/containerd/v2/defaults" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/errdefs" "github.com/opencontainers/image-spec/identity" "github.com/spf13/afero" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -61,12 +61,12 @@ func (e *containerdExtractor) CopyFromImage(ctx context.Context, imageRef string Hosts: config.ConfigureHosts(ctx, config.HostOptions{HostDir: config.HostDirFromRoot("/etc/containerd/certs.d")}), }) - image, err := client.Pull(ctx, imageRef, containerd.WithPullSnapshotter(containerd.DefaultSnapshotter), containerd.WithResolver(resolver), containerd.WithPullUnpack) + image, err := client.Pull(ctx, imageRef, containerd.WithPullSnapshotter(defaults.DefaultSnapshotter), containerd.WithResolver(resolver), containerd.WithPullUnpack) if err != nil { return fmt.Errorf("error pulling image: %w", err) } - snapshotter := client.SnapshotService(containerd.DefaultSnapshotter) + snapshotter := client.SnapshotService(defaults.DefaultSnapshotter) imageMountDirectory, err := fs.TempDir(nodeagentconfigv1alpha1.TempDir, "mount-image-") if err != nil { diff --git a/test/integration/nodeagent/healthcheck/healthcheck_test.go b/test/integration/nodeagent/healthcheck/healthcheck_test.go index d31bce5c4e8..3f4af4d0e42 100644 --- a/test/integration/nodeagent/healthcheck/healthcheck_test.go +++ b/test/integration/nodeagent/healthcheck/healthcheck_test.go @@ -13,7 +13,7 @@ import ( "net/http/httptest" "time" - "github.com/containerd/containerd" + containerd "github.com/containerd/containerd/v2/client" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" From ad41c362d46c9ef3b51ce60142bdef4650d20ef5 Mon Sep 17 00:00:00 2001 From: Vladimir Nachev Date: Mon, 10 Nov 2025 13:04:42 +0200 Subject: [PATCH 063/176] [GEP-26] Update the proposal to depict how extensions will use `WorkloadIdentity` (#13383) * Update GEP-26 to depict how extensions will use WorkloadIdentity * Apply review feedback Co-authored-by: Dimitar Mirchev --------- Co-authored-by: Dimitar Mirchev --- docs/proposals/26-workload-identity.md | 85 +++++++++++++++++++------- 1 file changed, 62 insertions(+), 23 deletions(-) diff --git a/docs/proposals/26-workload-identity.md b/docs/proposals/26-workload-identity.md index aa4bc333726..7a4ecd2e498 100644 --- a/docs/proposals/26-workload-identity.md +++ b/docs/proposals/26-workload-identity.md @@ -225,7 +225,7 @@ of `SecretBinding`, but on top of that will be extended to refer to In a nutshell, the changes introduced compared to `SecretBinding` are: -- `CredentialsBinding.credentialsRef` field will be mutable. It will refer to a +- `CredentialsBinding.credentialsRef` field will be immutable. It will refer to a `WorkloadIdentity` or `Secret` resource by its name and namespace. If the namespace is unset, the namespace of the `CredentialsBinding` will be used. - `quotas` and `provider` fields have the semantic as their respective @@ -271,38 +271,57 @@ cover clusters using WorkloadIdentity for authentication. While the infrastructure credentials for the shoot cluster are the main driver behind this GEP, various extensions can benefit of this feature as well. For -this purpose, a new optional field named `workloadIdentity` will be introduced -in `shoot.spec.extensions`, it will refer to workload identity by name assuming -the workloadIdentity resource is in the namespace of the shoot. The seed CRD -`extensions.extensions.gardener.cloud` will be also extended to reflect that the -extension is using workload identity letting extension controllers know when a -JWT or other kind of credentials are used. - -Similar approach can be taken to provide alternative to -`shoot.spec.dns.providers.secretName`, e.g. a new field `workloadIdentity` will -extended the `shoot.spec.dns.providers`. +this purpose, `shoot.spec.resources` will be extended to allow `WorkloadIdentity` +references next to the currently supported `Secret` and `ConfigMap`. Extensions, +via their own `shoot.spec.extensions.providerConfig`, will specify which +referenced resources they are using. This would allow extensions to reuse the +same `WorkloadIdentity` without the need for multiple resource references. For +every `WorkloadIdentity` referenced by the shoot, gardenlet will create a secret +in the control plane namespace bearing the workload identity features - the token +and the config. It will be up to the extension controller to make use of the +workload identity credentials available in the secret. Similar approach will be +taken to provide alternative to `shoot.spec.dns.providers.secretName`, e.g. a +new field `credentialsRef` will extended the `shoot.spec.dns.providers`. +Extension controllers are expected to implement admission validation for the +`WorkloadIdentity` resources of the types that they support instead of relying +on other extensions, i.e. infrastructure providers, to do the validation. ```yaml spec: extensions: - type: some-extension - workloadIdentity: - name: foo + providerConfig: + apiVersion: some-extension.extensions.gardener.cloud/v1alpha1 + kind: SomeExtensionConfig + resourcesRefs: + - name: resource-ref-1 dns: providers: - type: some-dns-provider - workloadIdentity: + credentialsRef: + apiVersion: security.gardener.cloud/v1alpha1 # or "v1", when secret is being used + kind: WorkloadIdentity # or "Secret", when secret is being used name: bar + resources: + - resourceName: resource-ref-1 + apiVersion: security.gardener.cloud/v1alpha1 # or "v1", when secret is being used + kind: WorkloadIdentity # or "Secret", when secret is being used + name: bar ``` #### Seed API Related Changes Wherever the Seed API is referring to secrets, it will be extended to refer to workload identities, as of now these are the fields `spec.backup.` and -`spec.dns.provider` and they will have new field `workloadIdentityRef` holding -the name and the namespace of a workload identity resource. The respective -`secretRef` fields will be made optional and validation will ensure only one of -`secretRef` and `workloadIdentityRef` is used at a time. +`spec.dns.provider` and they will have new field `credentialsRef` holding +the object reference to a credentials resource - `v1.Secret` or +`security.gardener.cloud/v1alpha1.WorkloadIdentity`. The respective +`secretRef` fields will be deprecated and removed in favour of `credentialsRef`. +During the migration phase, both fields will be allowed when the credential is +of type `v1.Secret` and validation will ensure both refer to the same secret. + +The workload identity credentials will be made available to the controllers in +the seed cluster via `Secret` containing the config and the token. ```yaml apiVersion: core.gardener.cloud/v1beta1 @@ -311,14 +330,35 @@ metadata: name: seed spec: backup: - workloadIdentityRef: + credentialsRef: + apiVersion: security.gardener.cloud/v1alpha1 # or "v1", when secret is being used + kind: WorkloadIdentity # or "Secret", when secret is being used name: backup-workloadidentity namespace: garden dns: provider: - workloadIdentityRef: + credentialsRef: + apiVersion: security.gardener.cloud/v1alpha1 # or "v1", when secret is being used + kind: WorkloadIdentity # or "Secret", when secret is being used name: ingress-workloadidentity namespace: garden + internal: + credentialsRef: + apiVersion: security.gardener.cloud/v1alpha1 # or "v1", when secret is being used + kind: WorkloadIdentity # or "Secret", when secret is being used + name: internal-workloadidentity + namespace: garden + defaults: + - credentialsRef: + apiVersion: security.gardener.cloud/v1alpha1 # or "v1", when secret is being used + kind: WorkloadIdentity # or "Secret", when secret is being used + name: default1-workloadidentity + namespace: garden + - credentialsRef: + apiVersion: security.gardener.cloud/v1alpha1 # or "v1", when secret is being used + kind: WorkloadIdentity # or "Secret", when secret is being used + name: default2-workloadidentity + namespace: garden ``` ### Gardener as OIDC Token Issuer @@ -482,9 +522,8 @@ type: Opaque The secret `cloudprovider` that now holds the static credentials will be reused to store the token and the provider config when the shoot is using workload identity as infrastructure credentials. For each extension using workload -identity, the secret will be named -`workloadidentity-extension-`, and for dns providers -`workloadidentity-dns-`. +identity, the secret will be named as per the naming convention for referenced +resources or other already established convention to name a secret in the seed. The reconciliation flow for a component using workload identity tokens will look like this: From 46f49378a1a972e4ef2c75f42417d66b05b71588 Mon Sep 17 00:00:00 2001 From: Shafeeque E S Date: Mon, 10 Nov 2025 18:17:46 +0530 Subject: [PATCH 064/176] Deploy `shoot-info` ConfigMap for workerless clusters as well (#13386) NB: Hide whitespace changes for this commit --- pkg/component/shoot/system/system.go | 28 +++++---- pkg/component/shoot/system/system_test.go | 71 +++++++++++++---------- 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/pkg/component/shoot/system/system.go b/pkg/component/shoot/system/system.go index 049a0d13204..9a724c5c222 100644 --- a/pkg/component/shoot/system/system.go +++ b/pkg/component/shoot/system/system.go @@ -174,14 +174,6 @@ func (s *shootSystem) computeResourcesData() (map[string][]byte, error) { if !s.values.IsWorkerless { var ( - shootInfoConfigMap = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: v1beta1constants.ConfigMapNameShootInfo, - Namespace: metav1.NamespaceSystem, - }, - Data: s.shootInfoData(), - } - port53 = intstr.FromInt32(53) port443 = intstr.FromInt32(kubeapiserverconstants.Port) port8053 = intstr.FromInt32(corednsconstants.PortServer) @@ -299,7 +291,6 @@ func (s *shootSystem) computeResourcesData() (map[string][]byte, error) { ) if err := registry.Add( - shootInfoConfigMap, networkPolicyAllowToShootAPIServer, networkPolicyAllowToDNS, networkPolicyAllowToKubelet, @@ -313,6 +304,20 @@ func (s *shootSystem) computeResourcesData() (map[string][]byte, error) { } } + shootInfoConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: v1beta1constants.ConfigMapNameShootInfo, + Namespace: metav1.NamespaceSystem, + }, + Data: s.shootInfoData(), + } + + if err := registry.Add( + shootInfoConfigMap, + ); err != nil { + return nil, err + } + if len(s.values.APIResourceList) > 0 { if err := registry.Add(s.readOnlyClusterRole()); err != nil { return nil, err @@ -360,7 +365,6 @@ func (s *shootSystem) shootInfoData() map[string]string { "provider": s.values.Object.Spec.Provider.Type, "region": s.values.Object.Spec.Region, "kubernetesVersion": s.values.Object.Spec.Kubernetes.Version, - "podNetwork": s.values.PodNetworkCIDRs[0].String(), "serviceNetwork": s.values.ServiceNetworkCIDRs[0].String(), "maintenanceBegin": s.values.Object.Spec.Maintenance.TimeWindow.Begin, "maintenanceEnd": s.values.Object.Spec.Maintenance.TimeWindow.End, @@ -370,6 +374,10 @@ func (s *shootSystem) shootInfoData() map[string]string { data["domain"] = *domain } + if len(s.values.PodNetworkCIDRs) > 0 { + data["podNetwork"] = s.values.PodNetworkCIDRs[0].String() + } + if len(s.values.NodeNetworkCIDRs) > 0 { data["nodeNetwork"] = s.values.NodeNetworkCIDRs[0].String() } diff --git a/pkg/component/shoot/system/system_test.go b/pkg/component/shoot/system/system_test.go index 0803c652914..dcb56f5d0a4 100644 --- a/pkg/component/shoot/system/system_test.go +++ b/pkg/component/shoot/system/system_test.go @@ -151,48 +151,57 @@ var _ = Describe("ShootSystem", func() { }) Context("shoot-info ConfigMap", func() { + var configMap *corev1.ConfigMap + + BeforeEach(func() { + configMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shoot-info", + Namespace: "kube-system", + }, + Data: map[string]string{ + "domain": domain, + "extensions": extension1 + `,` + extension2, + "kubernetesVersion": kubernetesVersion, + "maintenanceBegin": maintenanceBegin, + "maintenanceEnd": maintenanceEnd, + "nodeNetwork": nodeCIDRs[0].String(), + "nodeNetworks": nodeCIDRs[0].String() + "," + nodeCIDRs[1].String(), + + "projectName": projectName, + "provider": providerType, + "region": region, + "serviceNetwork": serviceCIDRs[0].String(), + "serviceNetworks": serviceCIDRs[0].String() + "," + serviceCIDRs[1].String(), + "shootNamespace": projectNamespace, + "shootName": shootName, + }, + } + }) + + It("should successfully deploy all resources", func() { + configMap.Data["podNetwork"] = podCIDRs[0].String() + configMap.Data["podNetworks"] = podCIDRs[0].String() + "," + podCIDRs[1].String() + + Expect(managedResource).To(contain(configMap)) + }) + When("shoot is workerless", func() { BeforeEach(func() { values.IsWorkerless = true + values.PodNetworkCIDRs = nil }) - It("should not deploy any ConfigMap", func() { + It("should successfully deploy all resources", func() { manifests, err := test.ExtractManifestsFromManagedResourceData(managedResourceSecret.Data) Expect(err).NotTo(HaveOccurred()) for _, manifest := range manifests { - Expect(manifest).NotTo(And(ContainSubstring("name: shoot-info"), ContainSubstring("kind: ConfigMap"))) + Expect(manifest).NotTo(ContainSubstring("podNetworks")) } - }) - }) - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "shoot-info", - Namespace: "kube-system", - }, - Data: map[string]string{ - "domain": domain, - "extensions": extension1 + `,` + extension2, - "kubernetesVersion": kubernetesVersion, - "maintenanceBegin": maintenanceBegin, - "maintenanceEnd": maintenanceEnd, - "nodeNetwork": nodeCIDRs[0].String(), - "nodeNetworks": nodeCIDRs[0].String() + "," + nodeCIDRs[1].String(), - "podNetwork": podCIDRs[0].String(), - "podNetworks": podCIDRs[0].String() + "," + podCIDRs[1].String(), - "projectName": projectName, - "provider": providerType, - "region": region, - "serviceNetwork": serviceCIDRs[0].String(), - "serviceNetworks": serviceCIDRs[0].String() + "," + serviceCIDRs[1].String(), - "shootNamespace": projectNamespace, - "shootName": shootName, - }, - } - - It("should successfully deploy all resources", func() { - Expect(managedResource).To(contain(configMap)) + Expect(managedResource).To(contain(configMap)) + }) }) }) From bc3bc162eabac5ff7e4c407abef5a62711cd6b24 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 10 Nov 2025 13:47:57 +0100 Subject: [PATCH 065/176] Update dependency kyverno/kyverno to v1.16.0 (#13391) --- example/provider-extensions/kyverno/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/provider-extensions/kyverno/kustomization.yaml b/example/provider-extensions/kyverno/kustomization.yaml index ad73a020620..fc5276abbce 100644 --- a/example/provider-extensions/kyverno/kustomization.yaml +++ b/example/provider-extensions/kyverno/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- https://github.com/kyverno/kyverno/releases/download/v1.15.2/install.yaml +- https://github.com/kyverno/kyverno/releases/download/v1.16.0/install.yaml - kyverno-poddisruptionbudget.yaml - kyverno-rbac.yaml From b589ab7729ae48ed2417195ee7e29f5991d3b440 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 10 Nov 2025 15:32:44 +0100 Subject: [PATCH 066/176] Update module github.com/containerd/containerd/v2 to v2.1.5 [SECURITY] (#13393) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 58e043e227c..4e3adca7610 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ahmetb/gen-crd-api-reference-docs v0.3.0 github.com/andybalholm/brotli v1.2.0 github.com/bramvdbogaerde/go-scp v1.5.0 - github.com/containerd/containerd/v2 v2.1.4 + github.com/containerd/containerd/v2 v2.1.5 github.com/containerd/errdefs v1.0.0 github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 diff --git a/go.sum b/go.sum index a1b9dea6e15..51fad1549a7 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFqu github.com/containerd/containerd v1.7.27/go.mod h1:xZmPnl75Vc+BLGt4MIfu6bp+fy03gdHAn9bz+FreFR0= github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= -github.com/containerd/containerd/v2 v2.1.4 h1:/hXWjiSFd6ftrBOBGfAZ6T30LJcx1dBjdKEeI8xucKQ= -github.com/containerd/containerd/v2 v2.1.4/go.mod h1:8C5QV9djwsYDNhxfTCFjWtTBZrqjditQ4/ghHSYjnHM= +github.com/containerd/containerd/v2 v2.1.5 h1:pWSmPxUszaLZKQPvOx27iD4iH+aM6o0BoN9+hg77cro= +github.com/containerd/containerd/v2 v2.1.5/go.mod h1:8C5QV9djwsYDNhxfTCFjWtTBZrqjditQ4/ghHSYjnHM= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= From 688a46f01daea633e4add06d8f7acb4237b2ba59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Nov 2025 14:32:54 +0000 Subject: [PATCH 067/176] Bump github.com/opencontainers/selinux from 1.12.0 to 1.13.0 (#13396) Bumps [github.com/opencontainers/selinux](https://github.com/opencontainers/selinux) from 1.12.0 to 1.13.0. - [Release notes](https://github.com/opencontainers/selinux/releases) - [Commits](https://github.com/opencontainers/selinux/compare/v1.12.0...v1.13.0) --- updated-dependencies: - dependency-name: github.com/opencontainers/selinux dependency-version: 1.13.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 5 +++-- go.sum | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4e3adca7610..d684adc8e7b 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( require ( cel.dev/expr v0.24.0 // indirect + cyphar.com/go-pathrs v0.2.1 // indirect dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/BurntSushi/toml v1.5.0 // indirect @@ -121,7 +122,7 @@ require ( github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect - github.com/cyphar/filepath-securejoin v0.4.1 // indirect + github.com/cyphar/filepath-securejoin v0.6.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -199,7 +200,7 @@ require ( github.com/nexucis/lamenv v0.5.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/runtime-spec v1.2.1 // indirect - github.com/opencontainers/selinux v1.12.0 // indirect + github.com/opencontainers/selinux v1.13.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/perses/common v0.27.1-0.20250326140707-96e439b14e0e // indirect github.com/perses/perses v0.51.0 // indirect diff --git a/go.sum b/go.sum index 51fad1549a7..df00e20d502 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCB cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= +cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -166,8 +168,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= -github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is= +github.com/cyphar/filepath-securejoin v0.6.0/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -586,8 +588,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.12.0 h1:6n5JV4Cf+4y0KNXW48TLj5DwfXpvWlxXplUkdTrmPb8= -github.com/opencontainers/selinux v1.12.0/go.mod h1:BTPX+bjVbWGXw7ZZWUbdENt8w0htPSrlgOOysQaU62U= +github.com/opencontainers/selinux v1.13.0 h1:Zza88GWezyT7RLql12URvoxsbLfjFx988+LGaWfbL84= +github.com/opencontainers/selinux v1.13.0/go.mod h1:XxWTed+A/s5NNq4GmYScVy+9jzXhGBVEOAyucdRUY8s= github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e h1:cxgCNo/R769CO23AK5TCh45H9SMUGZ8RukiF2/Qif3o= github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= github.com/operator-framework/operator-lib v0.18.0 h1:6OaWemt/CuyrjFMkLyk4O8Vj4CPHxt/m1DMuMAmPwXo= From 7d954d12312bbc3f2415d7be0e018f643720c48c Mon Sep 17 00:00:00 2001 From: Marcel Boehm Date: Mon, 10 Nov 2025 16:40:45 +0100 Subject: [PATCH 068/176] Build gardenadm image (#13394) --- .github/workflows/build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5eea8b5d15d..709ae8ffdce 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -97,6 +97,9 @@ jobs: - name: controller-manager target: controller-manager oci-repository: gardener/controller-manager + - name: gardenadm + target: gardenadm + oci-repository: gardener/gardenadm - name: gardenlet target: gardenlet oci-repository: gardener/gardenlet From 1f01c3b2d45746d24b74c780dc158becdb4b6931 Mon Sep 17 00:00:00 2001 From: Viktor Date: Mon, 10 Nov 2025 18:38:43 +0200 Subject: [PATCH 069/176] Add `RadaBDimitrova` as core reviewer (#13347) * Add RadaBDimitrova as core reviewer * Add Rada as v1.147 release responsible * fix table --- OWNERS_ALIASES | 1 + docs/development/process.md | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 85367cb0cc7..7f64e0b4c79 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -25,6 +25,7 @@ aliases: - marc1404 - oliver-goetz - plkokanov + - RadaBDimitrova - rfranzke - ScheererJ - shafeeqes diff --git a/docs/development/process.md b/docs/development/process.md index 6566337d4ea..8fd9c2139ea 100644 --- a/docs/development/process.md +++ b/docs/development/process.md @@ -25,9 +25,9 @@ Version | Week No | Begin Validation Phase | Due Date | Release Res ------- |------------| ---------------------- | -------------------|--------------------------------------------------------------| v1.130 | Week 41-42 | October 6, 2025 | October 19, 2025 | [@vitanovs](https://github.com/vitanovs) | v1.131 | Week 43-44 | October 20, 2025 | November 2, 2025 | [@dimitar-kostadinov](https://github.com/dimitar-kostadinov) | -v1.132 | Week 45-46 | November 3, 2025 | November 16, 2025 | [@LucaBernstein](https://github.com/LucaBernstein) | +v1.132 | Week 45-46 | November 3, 2025 | November 16, 2025 | [@LucaBernstein](https://github.com/LucaBernstein) | v1.133 | Week 47-48 | November 17, 2025 | November 30, 2025 | [@ialidzhikov](https://github.com/ialidzhikov) | -v1.134 | Week 49-50 | December 1, 2025 | December 14, 2025 | [@marc1404](https://github.com/marc1404) | +v1.134 | Week 49-50 | December 1, 2025 | December 14, 2025 | [@marc1404](https://github.com/marc1404) | v1.135 | Week 01-04 | December 29, 2025 | January 25, 2026 | [@timuthy](https://github.com/timuthy) | v1.136 | Week 05-06 | January 26, 2026 | February 8, 2026 | [@ary1992](https://github.com/ary1992) | v1.137 | Week 07-08 | February 9, 2026 | February 22, 2026 | [@acumino](https://github.com/acumino) | @@ -40,6 +40,7 @@ v1.143 | Week 19-20 | May 4, 2026 | May 17, 2026 | [@shafeeqes v1.144 | Week 21-22 | May 18, 2026 | May 31, 2026 | [@Kostov6](https://github.com/Kostov6) | v1.145 | Week 23-24 | June 1, 2026 | June 14, 2026 | [@vitanovs](https://github.com/vitanovs) | v1.146 | Week 25-26 | June 15, 2026 | June 28, 2026 | [@dimitar-kostadinov](https://github.com/dimitar-kostadinov) | +v1.147 | Week 27-28 | June 29, 2026 | July 12, 2026 | [@RadaBDimitrova](https://github.com/RadaBDimitrova) | Apart from the release of the next version, the release responsible is also taking care of potential hotfix releases of the last three minor versions. The release responsible is the main contact person for coordinating new feature PRs for the next minor versions or cherry-pick PRs for the last three minor versions. From ecbf549e962b64bd2555cc81dacaf39f7d4c7e30 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 10 Nov 2025 20:44:43 +0100 Subject: [PATCH 070/176] Update opentelemetry-operator to v0.139.0 (#13389) --- go.mod | 2 +- go.sum | 4 ++-- imagevector/containers.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index d684adc8e7b..300e77d4459 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/onsi/ginkgo/v2 v2.27.1 github.com/onsi/gomega v1.38.2 - github.com/open-telemetry/opentelemetry-operator v0.138.0 + github.com/open-telemetry/opentelemetry-operator v0.139.0 github.com/opencontainers/image-spec v1.1.1 github.com/pelletier/go-toml v1.9.5 github.com/perses/perses-operator v0.2.0 diff --git a/go.sum b/go.sum index df00e20d502..e7c0e81ba9c 100644 --- a/go.sum +++ b/go.sum @@ -580,8 +580,8 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= -github.com/open-telemetry/opentelemetry-operator v0.138.0 h1:PWNLD3i+sGpPLV2HpTylgGXuJB7iZp4cpoGDtw2XTBo= -github.com/open-telemetry/opentelemetry-operator v0.138.0/go.mod h1:RuM1oKvL0W9gNONH1mpV/1g08jGu7LugSl0BOkhuQhk= +github.com/open-telemetry/opentelemetry-operator v0.139.0 h1:HD4ptH5NQDroxpBRPpMG3puPhCUVtKVAoHtVx6FRPPw= +github.com/open-telemetry/opentelemetry-operator v0.139.0/go.mod h1:RuM1oKvL0W9gNONH1mpV/1g08jGu7LugSl0BOkhuQhk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 2c77224573c..c5beb248513 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -188,7 +188,7 @@ images: - name: opentelemetry-operator sourceRepository: github.com/open-telemetry/opentelemetry-operator repository: europe-docker.pkg.dev/gardener-project/releases/3rd/opentelemetry-operator/opentelemetry-operator - tag: "v0.138.0" + tag: "v0.139.0" labels: - name: 'gardener.cloud/cve-categorisation' value: From fec8de511b7ba382e168d758438a95d62af4d904 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Nov 2025 19:44:50 +0000 Subject: [PATCH 071/176] Bump github.com/containerd/containerd from 1.7.27 to 1.7.29 (#13397) Bumps [github.com/containerd/containerd](https://github.com/containerd/containerd) from 1.7.27 to 1.7.29. - [Release notes](https://github.com/containerd/containerd/releases) - [Changelog](https://github.com/containerd/containerd/blob/main/RELEASES.md) - [Commits](https://github.com/containerd/containerd/compare/v1.7.27...v1.7.29) --- updated-dependencies: - dependency-name: github.com/containerd/containerd dependency-version: 1.7.29 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 300e77d4459..1d814eac17a 100644 --- a/go.mod +++ b/go.mod @@ -109,7 +109,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/cgroups/v3 v3.0.5 // indirect - github.com/containerd/containerd v1.7.27 // indirect + github.com/containerd/containerd v1.7.29 // indirect github.com/containerd/containerd/api v1.9.0 // indirect github.com/containerd/continuity v0.4.5 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect diff --git a/go.sum b/go.sum index e7c0e81ba9c..6bb67842b17 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,8 @@ github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= -github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII= -github.com/containerd/containerd v1.7.27/go.mod h1:xZmPnl75Vc+BLGt4MIfu6bp+fy03gdHAn9bz+FreFR0= +github.com/containerd/containerd v1.7.29 h1:90fWABQsaN9mJhGkoVnuzEY+o1XDPbg9BTC9QTAHnuE= +github.com/containerd/containerd v1.7.29/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs= github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= github.com/containerd/containerd/v2 v2.1.5 h1:pWSmPxUszaLZKQPvOx27iD4iH+aM6o0BoN9+hg77cro= From 615c59321f1688cfd72f58ce518d80abeeea4f10 Mon Sep 17 00:00:00 2001 From: Jeremy Rickards <9338170+rickardsjp@users.noreply.github.com> Date: Mon, 10 Nov 2025 20:44:58 +0100 Subject: [PATCH 072/176] Adjust Istio Gateway dashboard to show correct resource consumption (#13402) * Update Istio dashboards to accurately reflect the total resource usage Prior to this change, a bug with stacked graphs caused Plutono to show only the resource usage of pods that were running at the end of the query time range. For time ranges longer than the default 30m, this caused the visual effect of istio ingressgateway resource consumption rising when pods were replaced. This commit removes the stacking and introduces a `_total` metric that sums over all *selected* istio ingressgateway pods. It can still be hidden by shift-clicking it in the legend below the graph. * Update dashboard schema --- .../istio-ingress-gateway-dashboard.json | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/pkg/component/observability/plutono/dashboards/garden-seed/istio/istio-ingress-gateway-dashboard.json b/pkg/component/observability/plutono/dashboards/garden-seed/istio/istio-ingress-gateway-dashboard.json index 52b2c0305d6..8b2bc9d6e6e 100644 --- a/pkg/component/observability/plutono/dashboards/garden-seed/istio/istio-ingress-gateway-dashboard.json +++ b/pkg/component/observability/plutono/dashboards/garden-seed/istio/istio-ingress-gateway-dashboard.json @@ -15,8 +15,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 23, - "iteration": 1679041083994, + "iteration": 1762761234731, "links": [], "panels": [ { @@ -58,15 +57,23 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.17", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ + { + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"$istio_proxy_pod\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "_total", + "refId": "B" + }, { "exemplar": true, "expr": "rate(container_cpu_usage_seconds_total{pod=~\"$istio_proxy_pod\"}[$__rate_interval])", @@ -161,15 +168,23 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.17", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ + { + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{pod=~\"istio-ingress.*\"})", + "hide": false, + "interval": "", + "legendFormat": "_total", + "refId": "A" + }, { "exemplar": true, "expr": "container_memory_working_set_bytes{pod=~\"$istio_proxy_pod\"}", @@ -264,7 +279,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.17", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", @@ -367,7 +382,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.17", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", @@ -470,7 +485,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.17", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", @@ -573,7 +588,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.17", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", @@ -676,7 +691,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.17", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", @@ -780,7 +795,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.17", + "pluginVersion": "7.5.43", "pointradius": 5, "points": false, "renderer": "flot", @@ -869,9 +884,13 @@ { "allValue": null, "current": { - "selected": false, - "text": "All", - "value": "$__all" + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] }, "datasource": "${datasource}", "definition": "container_cpu_usage_seconds_total{container=\"istio-proxy\"}", @@ -901,7 +920,6 @@ "allValue": null, "current": { "selected": true, - "tags": [], "text": [ "All" ], From 7a923e1f5ee13335197a3e4cd7a0ee8543e0ef29 Mon Sep 17 00:00:00 2001 From: Plamen Kokanov <35485709+plkokanov@users.noreply.github.com> Date: Mon, 10 Nov 2025 23:49:43 +0200 Subject: [PATCH 073/176] Add documentation on how to deprecate and remove `Shoot` API fields (#13300) * Add documentation on how to deprecate and remove `Shoot` API fields * Address review comments --- docs/development/changing-the-api.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/development/changing-the-api.md b/docs/development/changing-the-api.md index 3220dea6db8..4dd1b29b4d4 100644 --- a/docs/development/changing-the-api.md +++ b/docs/development/changing-the-api.md @@ -7,7 +7,7 @@ title: Changing the API This document describes the steps that need to be performed when changing the API. It provides guidance for API changes to both (Gardener system in general or component configurations). -Generally, as Gardener is a Kubernetes-native extension, it follows the same API conventions and guidelines like Kubernetes itself. The Kubernetes +Generally, as Gardener is a Kubernetes-native extension, it follows the same API conventions and guidelines like Kubernetes itself. The Kubernetes [API Conventions](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md) as well as [Changing the API](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md) topics already provide a good overview and general explanation of the basic concepts behind it. We are following the same approaches. @@ -55,6 +55,30 @@ The steps for removing a field from the code base is: Example of field removal can be found in the [Remove `seedTemplate` field from ManagedSeed API](https://github.com/gardener/gardener/pull/6972) PR. +#### Additional Steps When Removing a Field From the Shoot API + +Removing fields from the `Shoot` API requires special handling because the removal is tied to specific Kubernetes versions and backwards compatibility must be maintained. +When removing defaulted fields, extra care is needed because these fields already exist in storage even when users never explicitly set them. + +The deprecation and removal process takes two release cycles (three for defaulted fields): +1. In the first release: + - Mark the field as deprecated. + - Remove all code that uses the field (or ensure controllers can handle `nil` values for defaulted fields). + - Forbid the field from being set for Kubernetes versions starting from the next minor version that will be supported by Gardener in a future release. + - Show a warning to users that the field is deprecated and will be forbidden after a specific Kubernetes version. Use the [`WarningsOnCreate`](https://github.com/gardener/gardener/blob/0bd160008f123cb268100f3316ef01d09c7fe84e/pkg/apiserver/registry/core/shoot/strategy.go#L207-L209) and [`WarningsOnUpdate`](https://github.com/gardener/gardener/blob/0bd160008f123cb268100f3316ef01d09c7fe84e/pkg/apiserver/registry/core/shoot/strategy.go#L211-L214) functions for this. + - Update the [maintenance reconciler](https://github.com/gardener/gardener/blob/0bd160008f123cb268100f3316ef01d09c7fe84e/pkg/controllermanager/controller/shoot/maintenance/reconciler.go) to set the field to `nil` during forced upgrades to Kubernetes versions where the field is forbidden. +2. *For defaulted fields only, in a subsequent release:* + - Force the field to `nil` in the [`Canonicalize`](https://github.com/gardener/gardener/blob/0bd160008f123cb268100f3316ef01d09c7fe84e/pkg/apiserver/registry/core/shoot/strategy.go#L186-L190) function. + - Update the warning from step (1) to inform users that the field is being automatically set to `nil`. +3. When Gardener drops support for the last Kubernetes version that allowed the deprecated field, remove the field completely from the API. + +This approach prevents several problems: + - Avoids issues when `gardener-apiserver` is upgraded to a version that sets fields to `nil` while other controllers are still running older versions. + - Allows users to update `Shoot`s using `Get` and `{Update,Patch}` requests without manually setting deprecated fields to `nil`, even when those fields were automatically defaulted in previous releases. + - Shows warnings when users explicitly set deprecated fields that will be automatically changed to `nil` by the [`Canonicalize`](https://github.com/gardener/gardener/blob/0bd160008f123cb268100f3316ef01d09c7fe84e/pkg/apiserver/registry/core/shoot/strategy.go#L186-L190) function. This warning is important because the update will increase the `Shoot`'s generation since the user's request (with the deprecated field set) differs from what is present in storage (where the field is `nil`). + +Examples for the above can be seen in the following PRs: [Deprecate CA flag `maxEmptyBulkDelete`](https://github.com/gardener/gardener/pull/12115), [Forbid setting the `Shoot`'s `.spec.kubernetes.clusterAutoscaler.maxEmptyBulkDelete` field for Kubernetes versions >= 1.33](https://github.com/gardener/gardener/pull/12413), and [Explicitly set `maxEmptyBulkDelete` to nil in `Canonicalize`](https://github.com/gardener/gardener/pull/13054) + ## Component Configuration APIs Most Gardener components have a component configuration that follows similar principles to the Gardener API. From a000efea44203eb74cc14c49d75b8fde50e4b078 Mon Sep 17 00:00:00 2001 From: Tobias Schlicht <155947440+tobschli@users.noreply.github.com> Date: Mon, 10 Nov 2025 22:49:50 +0100 Subject: [PATCH 074/176] [GEP-28] Update `gardenadm connect` docs (#13392) * Add `make` target to build `gardenadm` locally * Update `gardenadm connect` docs * Implement PR feedback Co-authored-by: Rafael Franzke --------- Co-authored-by: Rafael Franzke --- Makefile | 3 ++ .../getting_started_locally_with_gardenadm.md | 48 +++++++++++++++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3c5c917dd27..4f6f9d79243 100644 --- a/Makefile +++ b/Makefile @@ -426,6 +426,9 @@ operator-seed-up operator-seed-dev: $(SKAFFOLD) $(HELM) $(KUBECTL) operator-up g TIMEOUT=900 ./hack/usage/wait-for.sh garden local VirtualGardenAPIServerAvailable RuntimeComponentsHealthy VirtualComponentsHealthy operator-seed-down: $(SKAFFOLD) $(HELM) $(KUBECTL) seed-down garden-down +# gardenadm +gardenadm: + BUILD_OUTPUT_FILE=./bin/ BUILD_PACKAGES=./cmd/gardenadm $(MAKE) build # gardenadm-{up,down} gardenadm-%: export SKAFFOLD_FILENAME = skaffold-gardenadm.yaml gardenadm-up: $(SKAFFOLD) $(KUBECTL) diff --git a/docs/deployment/getting_started_locally_with_gardenadm.md b/docs/deployment/getting_started_locally_with_gardenadm.md index 924190bcb95..d098ea84fdc 100644 --- a/docs/deployment/getting_started_locally_with_gardenadm.md +++ b/docs/deployment/getting_started_locally_with_gardenadm.md @@ -199,12 +199,52 @@ This will deploy [`gardener-operator`](../concepts/operator.md) and create a `Ga Find all information about it [here](getting_started_locally.md#alternative-way-to-set-up-garden-and-seed-leveraging-gardener-operator). Note, that in this setup, no `Seed` will be registered in the Gardener - it's just a plain garden cluster without the ability to create regular shoot clusters. -Once above command is finished, you can connect the self-hosted shoot cluster to this Gardener instance: +Once above command is finished, you can generate a bootstrap token using `gardenadm` to connect the shoot cluster to this Gardener instance. +For this, you must have installed the `gardenadm` binary locally. You can build it via: ```shell -$ kubectl -n gardenadm-unmanaged-infra exec -it machine-0 -- bash -root@machine-0:/# gardenadm connect -2025-07-03T12:21:49.586Z INFO Command is work in progress +make gardenadm +``` + +This will install it to `./bin/gardenadm`, from where you can call it. + +After you have completed this, you need to get a kubeconfig for [the Gardener instance](../concepts/operator.md#accessing-the-virtual-garden-cluster) you want to connect the self-hosted shoot to. +We will refer to the path of this kubeconfig as `` in the following. + +Now you can generate the bootstrap token and the full `gardenadm connect` command like this: + +```shell +$ KUBECONFIG= ./bin/gardenadm token create --print-connect-command --shoot-namespace=garden --shoot-name=root +# This will output a command similar to: +gardenadm connect --bootstrap-token ... --ca-certificate ... https://api.virtual-garden.local.gardener.cloud +``` + +Copy the full output, exec once again into one of the control-plane machines of your self-hosted shoot cluster, and paste and run the generated `gardenadm connect` command there: + +```shell +root@machine-0:/# gardenadm connect --bootstrap-token ... --ca-certificate ... https://api.virtual-garden.local.gardener.cloud +2025-11-10T08:12:32.287Z INFO Using resources from directory {"configDir": "/gardenadm/resources/"} +2025-11-10T08:12:32.334Z INFO Initializing gardenadm botanist with fake client set {"cloudProfile": {"apiVersion": "core.gardener.cloud/v1beta1", "kind": "CloudProfile", "name": "local"}, "project": {"apiVersion": "core.gardener.cloud/v1beta1", "kind": "Project", "name": "garden"}, "shoot": {"apiVersion": "core.gardener.cloud/v1beta1", "kind": "Shoot", "namespace": "garden", "name": "root"}} +2025-11-10T08:12:32.345Z INFO Starting {"flow": "connect"} +... +2025-11-10T08:13:04.571Z INFO Succeeded {"flow": "connect", "task": "Waiting until gardenlet is ready"} +2025-11-10T08:13:04.571Z INFO Finished {"flow": "connect"} +``` + +Once this is done, you can observe that there is now a `gardenlet` running in the self-hosted shoot cluster, which connects it to the Gardener instance: + +```shell +root@machine-0:/# kubectl get pods -n kube-system | grep gardenlet +gardenlet-6cbcb676f5-prh8f 1/1 Running 0 40m +gardenlet-6cbcb676f5-wwn8w 1/1 Running 0 40m +```` + +You can also observe that the self-hosted shoot cluster is now registered as a shoot cluster in Gardener: + +```shell +kubectl --kubeconfig= get Shoots -A +NAMESPACE NAME CLOUDPROFILE PROVIDER REGION K8S VERSION HIBERNATION LAST OPERATION STATUS AGE +garden root local local local 1.33.0 Awake healthy 42m ``` ## Running E2E Tests for `gardenadm` From 060533563b8cef47e066ea847e62284531747c40 Mon Sep 17 00:00:00 2001 From: Vladimir Nachev Date: Tue, 11 Nov 2025 08:52:44 +0200 Subject: [PATCH 075/176] Add support for (local) multiplatform container images build (#13324) --- Dockerfile | 28 +++++++++++++++------------- Makefile | 25 +++++++++++++------------ 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd5b8ac0586..20b2db47831 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,77 +1,79 @@ # builder -FROM golang:1.25.4 AS builder +FROM --platform=$BUILDPLATFORM golang:1.25.4 AS builder ARG GOPROXY=https://proxy.golang.org,direct ENV GOPROXY=$GOPROXY WORKDIR /go/src/github.com/gardener/gardener COPY . . ARG EFFECTIVE_VERSION -RUN make install EFFECTIVE_VERSION=$EFFECTIVE_VERSION +ARG TARGETOS +ARG TARGETARCH +RUN make build EFFECTIVE_VERSION=$EFFECTIVE_VERSION GOOS=$TARGETOS GOARCH=$TARGETARCH BUILD_OUTPUT_FILE="/output/bin/" # distroless-static FROM gcr.io/distroless/static-debian12:nonroot AS distroless-static # apiserver FROM distroless-static AS apiserver -COPY --from=builder /go/bin/gardener-apiserver /gardener-apiserver +COPY --from=builder /output/bin/gardener-apiserver /gardener-apiserver WORKDIR / ENTRYPOINT ["/gardener-apiserver"] # controller-manager FROM distroless-static AS controller-manager -COPY --from=builder /go/bin/gardener-controller-manager /gardener-controller-manager +COPY --from=builder /output/bin/gardener-controller-manager /gardener-controller-manager WORKDIR / ENTRYPOINT ["/gardener-controller-manager"] # scheduler FROM distroless-static AS scheduler -COPY --from=builder /go/bin/gardener-scheduler /gardener-scheduler +COPY --from=builder /output/bin/gardener-scheduler /gardener-scheduler WORKDIR / ENTRYPOINT ["/gardener-scheduler"] # gardenlet FROM distroless-static AS gardenlet -COPY --from=builder /go/bin/gardenlet /gardenlet +COPY --from=builder /output/bin/gardenlet /gardenlet WORKDIR / ENTRYPOINT ["/gardenlet"] # gardenadm FROM distroless-static AS gardenadm -COPY --from=builder /go/bin/gardenadm /gardenadm +COPY --from=builder /output/bin/gardenadm /gardenadm WORKDIR / ENTRYPOINT ["/gardenadm"] # admission-controller FROM distroless-static AS admission-controller -COPY --from=builder /go/bin/gardener-admission-controller /gardener-admission-controller +COPY --from=builder /output/bin/gardener-admission-controller /gardener-admission-controller WORKDIR / ENTRYPOINT ["/gardener-admission-controller"] # resource-manager FROM distroless-static AS resource-manager -COPY --from=builder /go/bin/gardener-resource-manager /gardener-resource-manager +COPY --from=builder /output/bin/gardener-resource-manager /gardener-resource-manager WORKDIR / ENTRYPOINT ["/gardener-resource-manager"] # node-agent FROM distroless-static AS node-agent -COPY --from=builder /go/bin/gardener-node-agent /gardener-node-agent +COPY --from=builder /output/bin/gardener-node-agent /gardener-node-agent WORKDIR / ENTRYPOINT ["/gardener-node-agent"] # operator FROM distroless-static AS operator -COPY --from=builder /go/bin/gardener-operator /gardener-operator +COPY --from=builder /output/bin/gardener-operator /gardener-operator WORKDIR / ENTRYPOINT ["/gardener-operator"] # gardener-extension-provider-local FROM distroless-static AS gardener-extension-provider-local -COPY --from=builder /go/bin/gardener-extension-provider-local /gardener-extension-provider-local +COPY --from=builder /output/bin/gardener-extension-provider-local /gardener-extension-provider-local WORKDIR / ENTRYPOINT ["/gardener-extension-provider-local"] # gardener-extension-admission-local FROM distroless-static AS gardener-extension-admission-local -COPY --from=builder /go/bin/gardener-extension-admission-local /gardener-extension-admission-local +COPY --from=builder /output/bin/gardener-extension-admission-local /gardener-extension-admission-local WORKDIR / ENTRYPOINT ["/gardener-extension-admission-local"] diff --git a/Makefile b/Makefile index 4f6f9d79243..6f4c2877141 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,7 @@ PARALLEL_E2E_TESTS ?= 5 GARDENER_RELEASE_DOWNLOAD_PATH := $(REPO_ROOT)/dev DEV_SETUP_WITH_LPP_RESIZE_SUPPORT ?= false DEV_SETUP_WITH_WORKLOAD_IDENTITY_SUPPORT ?= false +TARGET_PLATFORMS ?= linux/$(shell go env GOARCH) PRINT_HELP ?= ifneq ($(SEED_NAME),provider-extensions) @@ -91,18 +92,18 @@ build: .PHONY: docker-images docker-images: - @echo "Building docker images with version and tag $(EFFECTIVE_VERSION)" - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(APISERVER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(APISERVER_IMAGE_REPOSITORY):latest -f Dockerfile --target apiserver . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(CONTROLLER_MANAGER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(CONTROLLER_MANAGER_IMAGE_REPOSITORY):latest -f Dockerfile --target controller-manager . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(SCHEDULER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(SCHEDULER_IMAGE_REPOSITORY):latest -f Dockerfile --target scheduler . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(ADMISSION_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(ADMISSION_IMAGE_REPOSITORY):latest -f Dockerfile --target admission-controller . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):latest -f Dockerfile --target resource-manager . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(NODE_AGENT_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(NODE_AGENT_IMAGE_REPOSITORY):latest -f Dockerfile --target node-agent . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(OPERATOR_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(OPERATOR_IMAGE_REPOSITORY):latest -f Dockerfile --target operator . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(GARDENLET_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(GARDENLET_IMAGE_REPOSITORY):latest -f Dockerfile --target gardenlet . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(GARDENADM_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(GARDENADM_IMAGE_REPOSITORY):latest -f Dockerfile --target gardenadm . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):latest -f Dockerfile --target gardener-extension-provider-local . - @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(EXTENSION_ADMISSION_LOCAL_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(EXTENSION_ADMISSION_LOCAL_IMAGE_REPOSITORY):latest -f Dockerfile --target gardener-extension-admission-local . + @echo "Building docker images with version and tag $(EFFECTIVE_VERSION) for target platforms $(TARGET_PLATFORMS)" + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(APISERVER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(APISERVER_IMAGE_REPOSITORY):latest -f Dockerfile --target apiserver . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(CONTROLLER_MANAGER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(CONTROLLER_MANAGER_IMAGE_REPOSITORY):latest -f Dockerfile --target controller-manager . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(SCHEDULER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(SCHEDULER_IMAGE_REPOSITORY):latest -f Dockerfile --target scheduler . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(ADMISSION_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(ADMISSION_IMAGE_REPOSITORY):latest -f Dockerfile --target admission-controller . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):latest -f Dockerfile --target resource-manager . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(NODE_AGENT_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(NODE_AGENT_IMAGE_REPOSITORY):latest -f Dockerfile --target node-agent . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(OPERATOR_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(OPERATOR_IMAGE_REPOSITORY):latest -f Dockerfile --target operator . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(GARDENLET_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(GARDENLET_IMAGE_REPOSITORY):latest -f Dockerfile --target gardenlet . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(GARDENADM_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(GARDENADM_IMAGE_REPOSITORY):latest -f Dockerfile --target gardenadm . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):latest -f Dockerfile --target gardener-extension-provider-local . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) --platform $(TARGET_PLATFORMS) -t $(EXTENSION_ADMISSION_LOCAL_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(EXTENSION_ADMISSION_LOCAL_IMAGE_REPOSITORY):latest -f Dockerfile --target gardener-extension-admission-local . .PHONY: docker-push docker-push: From 39c61c6cc75f1cdbdf0e299fd593fe363fa10b89 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Tue, 11 Nov 2025 07:52:51 +0100 Subject: [PATCH 076/176] Update module github.com/containerd/containerd/v2 to v2.2.0 (#13400) --- go.mod | 16 ++++++++-------- go.sum | 39 ++++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 1d814eac17a..81f96642331 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ahmetb/gen-crd-api-reference-docs v0.3.0 github.com/andybalholm/brotli v1.2.0 github.com/bramvdbogaerde/go-scp v1.5.0 - github.com/containerd/containerd/v2 v2.1.5 + github.com/containerd/containerd/v2 v2.2.0 github.com/containerd/errdefs v1.0.0 github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 @@ -39,7 +39,7 @@ require ( github.com/perses/perses-operator v0.2.0 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.85.0 github.com/prometheus/blackbox_exporter v0.27.0 - github.com/prometheus/client_golang v1.23.0 + github.com/prometheus/client_golang v1.23.2 github.com/prometheus/common v0.67.2 github.com/robfig/cron v1.2.0 github.com/spf13/afero v1.15.0 @@ -96,7 +96,7 @@ require ( github.com/BurntSushi/toml v1.5.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Microsoft/hcsshim v0.13.0 // indirect + github.com/Microsoft/hcsshim v0.14.0-rc.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/PaesslerAG/gval v1.2.4 // indirect github.com/PaesslerAG/jsonpath v0.1.2-0.20240726212847-3a740cf7976f // indirect @@ -108,20 +108,20 @@ require ( github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/cgroups/v3 v3.0.5 // indirect + github.com/containerd/cgroups/v3 v3.1.0 // indirect github.com/containerd/containerd v1.7.29 // indirect - github.com/containerd/containerd/api v1.9.0 // indirect + github.com/containerd/containerd/api v1.10.0 // indirect github.com/containerd/continuity v0.4.5 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/platforms v1.0.0-rc.1 // indirect + github.com/containerd/platforms v1.0.0-rc.2 // indirect github.com/containerd/plugin v1.0.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/containerd/ttrpc v1.2.7 // indirect github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/cyphar/filepath-securejoin v0.6.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0 // indirect @@ -170,7 +170,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/labstack/echo/v4 v4.13.4 // indirect github.com/labstack/gommon v0.4.2 // indirect diff --git a/go.sum b/go.sum index 6bb67842b17..723004333db 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Microsoft/hcsshim v0.13.0 h1:/BcXOiS6Qi7N9XqUcv27vkIuVOkBEcWstd2pMlWSeaA= -github.com/Microsoft/hcsshim v0.13.0/go.mod h1:9KWJ/8DgU+QzYGupX4tzMhRQE8h6w90lH6HAaclpEok= +github.com/Microsoft/hcsshim v0.14.0-rc.1 h1:qAPXKwGOkVn8LlqgBN8GS0bxZ83hOJpcjxzmlQKxKsQ= +github.com/Microsoft/hcsshim v0.14.0-rc.1/go.mod h1:hTKFGbnDtQb1wHiOWv4v0eN+7boSWAHyK/tNAaYZL0c= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -131,14 +131,14 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= -github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= +github.com/containerd/cgroups/v3 v3.1.0 h1:azxYVj+91ZgSnIBp2eI3k9y2iYQSR/ZQIgh9vKO+HSY= +github.com/containerd/cgroups/v3 v3.1.0/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= github.com/containerd/containerd v1.7.29 h1:90fWABQsaN9mJhGkoVnuzEY+o1XDPbg9BTC9QTAHnuE= github.com/containerd/containerd v1.7.29/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs= -github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= -github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= -github.com/containerd/containerd/v2 v2.1.5 h1:pWSmPxUszaLZKQPvOx27iD4iH+aM6o0BoN9+hg77cro= -github.com/containerd/containerd/v2 v2.1.5/go.mod h1:8C5QV9djwsYDNhxfTCFjWtTBZrqjditQ4/ghHSYjnHM= +github.com/containerd/containerd/api v1.10.0 h1:5n0oHYVBwN4VhoX9fFykCV9dF1/BvAXeg2F8W6UYq1o= +github.com/containerd/containerd/api v1.10.0/go.mod h1:NBm1OAk8ZL+LG8R0ceObGxT5hbUYj7CzTmR3xh0DlMM= +github.com/containerd/containerd/v2 v2.2.0 h1:K7TqcXy+LnFmZaui2DgHsnp2gAHhVNWYaHlx7HXfys8= +github.com/containerd/containerd/v2 v2.2.0/go.mod h1:YCMjKjA4ZA7egdHNi3/93bJR1+2oniYlnS+c0N62HdE= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -149,8 +149,8 @@ github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/platforms v1.0.0-rc.1 h1:83KIq4yy1erSRgOVHNk1HYdPvzdJ5CnsWaRoJX4C41E= -github.com/containerd/platforms v1.0.0-rc.1/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4= +github.com/containerd/platforms v1.0.0-rc.2 h1:0SPgaNZPVWGEi4grZdV8VRYQn78y+nm6acgLGv/QzE4= +github.com/containerd/platforms v1.0.0-rc.2/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4= github.com/containerd/plugin v1.0.0 h1:c8Kf1TNl6+e2TtMHZt+39yAPDbouRH9WAToRjex483Y= github.com/containerd/plugin v1.0.0/go.mod h1:hQfJe5nmWfImiqT1q8Si3jLv3ynMUIBB47bQ+KexvO8= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= @@ -164,8 +164,9 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= +github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is= @@ -400,8 +401,8 @@ github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrR github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= @@ -475,8 +476,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= +github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -629,8 +630,8 @@ github.com/prometheus/blackbox_exporter v0.27.0/go.mod h1:X1tfplLxckV+1vPT0JF5q2 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= -github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -755,8 +756,8 @@ github.com/zitadel/oidc/v3 v3.38.1 h1:VTf1Bv/33UbSwJnIWbfEIdpUGYKfoHetuBNIqVTcjv github.com/zitadel/oidc/v3 v3.38.1/go.mod h1:muukzAasaWmn3vBwEVMglJfuTE0PKCvLJGombPwXIRw= github.com/zitadel/schema v1.3.1 h1:QT3kwiRIRXXLVAs6gCK/u044WmUVh6IlbLXUsn6yRQU= github.com/zitadel/schema v1.3.1/go.mod h1:071u7D2LQacy1HAN+YnMd/mx1qVE2isb0Mjeqg46xnU= -go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= -go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.etcd.io/etcd/api/v3 v3.6.4 h1:7F6N7toCKcV72QmoUKa23yYLiiljMrT4xCeBL9BmXdo= go.etcd.io/etcd/api/v3 v3.6.4/go.mod h1:eFhhvfR8Px1P6SEuLT600v+vrhdDTdcfMzmnxVXXSbk= go.etcd.io/etcd/client/pkg/v3 v3.6.4 h1:9HBYrjppeOfFjBjaMTRxT3R7xT0GLK8EJMVC4xg6ok0= From 44af42b103578bf398b80a0c85ddd5ef6157909b Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Tue, 11 Nov 2025 10:43:44 +0100 Subject: [PATCH 077/176] [GEP-28] `gardenadm connect`: Introduce `shoot/lease` controller in `gardenlet` (#13342) * Prefactor: Rename constant for `GardenletReady` condition We'll reuse it for `Shoot`s as well :) * Move `seed/lease` controller into generic package now in `pkg/gardenlet/controller/lease` instead of `pkg/gardenlet/controller/seed/lease` * Prefactor: Make `Lease` controller generic * Add `Lease` controller for self-hosted `Shoot`s to `gardenlet` * Adapt `Shoot{Authorizer,Restriction}` handlers * Enable `shoot-lease` reconciler for self-hosted shoots * Update `Processing` `Shoot` statuses to `Aborted` - now that the `gardenlet` has permissions for `shoots/status` updates - Also, filtering the seed names is no longer needed with https://github.com/gardener/gardener/blob/36c0959cec3c1145a36fbe98c335ed678d0aeb2d/cmd/gardenlet/app/app.go#L286-L288 * Enable periodic healthz check * Adapt `seed-lifecycle` controller in GCM Now it is the `gardenlet-lifecycle` controller reconciling both `Seed`s and `Shoot`s. * [GAPI] Maintain `shoot.gardener.cloud/is-self-hosted` label for better filtering Without this, the `gardenlet-lifecycle` (and potential future actors) would need to loop over all worker pools for each `Shoot`. This can become quite costly in large landscapes - filtering via label selector is fast and cheap. * Address PR review feedback * Address PR review feedback * Address PR review feedback --- cmd/gardenlet/app/app.go | 27 +-- docs/concepts/controller-manager.md | 26 +-- docs/concepts/gardenlet.md | 17 +- .../admission/shootrestriction/handler.go | 29 +-- .../shootrestriction/handler_test.go | 150 ++++++++++++++ .../webhook/auth/seed/authorizer.go | 9 +- .../webhook/auth/seed/authorizer_test.go | 38 ++++ .../webhook/auth/shoot/authorizer.go | 11 +- .../webhook/auth/shoot/authorizer_test.go | 119 ++++++++++- .../core/v1beta1/constants/types_constants.go | 2 + pkg/apis/core/v1beta1/types_seed.go | 2 - pkg/apis/core/v1beta1/types_utils.go | 3 + pkg/apiserver/registry/core/shoot/strategy.go | 10 + .../registry/core/shoot/strategy_test.go | 20 ++ .../garden/backupentry/backupentry.go | 2 +- pkg/controllermanager/controller/add.go | 9 + .../exposureclass/exposureclass_suite_test.go | 2 +- .../controller/gardenletlifecycle/add.go | 79 ++++++++ .../controller/gardenletlifecycle/add_test.go | 77 ++++++++ .../gardenletlifecycle_suite_test.go | 17 ++ .../reconciler.go | 127 ++++++++---- .../controller/managedseedset/add_test.go | 6 +- .../controller/managedseedset/replica.go | 2 +- .../controller/managedseedset/replica_test.go | 2 +- pkg/controllermanager/controller/seed/add.go | 7 - .../controller/seed/lifecycle/add.go | 42 ---- .../controller/shoot/conditions/add_test.go | 4 +- pkg/gardenlet/controller/add.go | 8 +- pkg/gardenlet/controller/lease/add.go | 49 +++++ .../{seed => }/lease/lease_suite_test.go | 0 pkg/gardenlet/controller/lease/reconciler.go | 142 +++++++++++++ .../{seed => }/lease/reconciler_test.go | 48 ++--- pkg/gardenlet/controller/seed/add.go | 8 +- pkg/gardenlet/controller/seed/lease/add.go | 66 +++---- .../controller/seed/lease/reconciler.go | 132 ------------- .../controller/seed/seed/reconciler.go | 2 +- pkg/gardenlet/controller/shoot/add.go | 10 + pkg/gardenlet/controller/shoot/lease/add.go | 38 ++++ pkg/scheduler/controller/shoot/reconciler.go | 2 +- .../controller/shoot/reconciler_test.go | 10 +- pkg/utils/graph/eventhandler_seed.go | 2 +- pkg/utils/graph/eventhandler_shoot.go | 10 +- pkg/utils/graph/graph_test.go | 76 ++++--- pkg/utils/kubernetes/health/seed.go | 2 +- pkg/utils/kubernetes/health/seed_test.go | 22 +-- .../global/customverbauthorizer/admission.go | 4 +- skaffold-gardenadm.yaml | 2 + skaffold-operator.yaml | 2 +- skaffold-seed.yaml | 2 + skaffold.yaml | 4 +- test/framework/k8s_utils_test.go | 2 +- .../lifecycle_suite_test.go | 23 ++- .../lifecycle_test.go | 6 +- .../managedseedset/managedseedset_test.go | 2 +- .../shoot/conditions/conditions_test.go | 12 +- .../shoot/migration/migration_test.go | 2 +- .../gardenlet/seed/lease/lease_suite_test.go | 13 +- .../gardenlet/seed/lease/lease_test.go | 160 ++++++++------- .../gardenlet/shoot/lease/lease_suite_test.go | 187 ++++++++++++++++++ .../gardenlet/shoot/lease/lease_test.go | 124 ++++++++++++ .../integration/scheduler/shoot/shoot_test.go | 2 +- 61 files changed, 1502 insertions(+), 511 deletions(-) create mode 100644 pkg/controllermanager/controller/gardenletlifecycle/add.go create mode 100644 pkg/controllermanager/controller/gardenletlifecycle/add_test.go create mode 100644 pkg/controllermanager/controller/gardenletlifecycle/gardenletlifecycle_suite_test.go rename pkg/controllermanager/controller/{seed/lifecycle => gardenletlifecycle}/reconciler.go (54%) delete mode 100644 pkg/controllermanager/controller/seed/lifecycle/add.go create mode 100644 pkg/gardenlet/controller/lease/add.go rename pkg/gardenlet/controller/{seed => }/lease/lease_suite_test.go (100%) create mode 100644 pkg/gardenlet/controller/lease/reconciler.go rename pkg/gardenlet/controller/{seed => }/lease/reconciler_test.go (88%) delete mode 100644 pkg/gardenlet/controller/seed/lease/reconciler.go create mode 100644 pkg/gardenlet/controller/shoot/lease/add.go rename test/integration/controllermanager/{seed/lifecycle => gardenletlifecycle}/lifecycle_suite_test.go (89%) rename test/integration/controllermanager/{seed/lifecycle => gardenletlifecycle}/lifecycle_test.go (98%) create mode 100644 test/integration/gardenlet/shoot/lease/lease_suite_test.go create mode 100644 test/integration/gardenlet/shoot/lease/lease_test.go diff --git a/cmd/gardenlet/app/app.go b/cmd/gardenlet/app/app.go index a9cde085cc4..0ebde01e40a 100644 --- a/cmd/gardenlet/app/app.go +++ b/cmd/gardenlet/app/app.go @@ -173,14 +173,8 @@ func run(ctx context.Context, cancel context.CancelFunc, log logr.Logger, cfg *g if err := mgr.AddReadyzCheck("runtime-informer-sync", gardenerhealthz.NewCacheSyncHealthz(mgr.GetCache())); err != nil { return err } - - if !gardenlet.IsResponsibleForSelfHostedShoot() { - // TODO(rfranzke): This healthz check is currently not enabled for self-hosted shoots because it depends on the - // seed-lease controller. This controller is currently not enabled, but it will be in the future. Once it is - // enabled, we can also enable this healthz check for self-hosted shoots. - if err := mgr.AddHealthzCheck("periodic-health", gardenerhealthz.CheckerFunc(healthManager)); err != nil { - return err - } + if err := mgr.AddHealthzCheck("periodic-health", gardenerhealthz.CheckerFunc(healthManager)); err != nil { + return err } var selfHostedShootMeta *types.NamespacedName @@ -368,6 +362,10 @@ func (g *garden) Start(ctx context.Context) error { Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: gardenlet.ResourcePrefixSelfHostedShoot + g.selfHostedShootMeta.Name}), Namespaces: map[string]cache.Config{g.selfHostedShootMeta.Namespace: {}}, }, + &coordinationv1.Lease{}: { + Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: gardenlet.ResourcePrefixSelfHostedShoot + g.selfHostedShootMeta.Name}), + Namespaces: map[string]cache.Config{g.selfHostedShootMeta.Namespace: {}}, + }, } return kubernetes.AggregatorCacheFunc( @@ -424,12 +422,11 @@ func (g *garden) Start(ctx context.Context) error { if err := g.registerSeed(ctx, gardenCluster.GetClient()); err != nil { return err } + } - // TODO(rfranzke): Move this out of this 'if'-condition once the "shoot gardenlets" have the needed permissions. - log.Info("Updating last operation status of processing Shoots to 'Aborted'") - if err := g.updateProcessingShootStatusToAborted(ctx, gardenCluster.GetClient()); err != nil { - return err - } + log.Info("Updating last operation status of processing Shoots to 'Aborted'") + if err := g.updateProcessingShootStatusToAborted(ctx, gardenCluster.GetClient()); err != nil { + return err } log.Info("Attempt to create seedmanagement.gardener.cloud/v1alpha1.Gardenlet object in garden cluster for self-upgrades if necessary") @@ -690,10 +687,6 @@ func (g *garden) updateProcessingShootStatusToAborted(ctx context.Context, garde var taskFns []flow.TaskFn for _, shoot := range shootList.Items { - if specSeedName, statusSeedName := gardenerutils.GetShootSeedNames(&shoot); gardenerutils.GetResponsibleSeedName(specSeedName, statusSeedName) != g.config.SeedConfig.Name { - continue - } - // Check if the status indicates that an operation is processing and mark it as "aborted". if shoot.Status.LastOperation == nil || shoot.Status.LastOperation.State != gardencorev1beta1.LastOperationStateProcessing { continue diff --git a/docs/concepts/controller-manager.md b/docs/concepts/controller-manager.md index d6e3ca1f6c3..50d62ea2213 100644 --- a/docs/concepts/controller-manager.md +++ b/docs/concepts/controller-manager.md @@ -162,6 +162,19 @@ In order to activate it, provide the following configuration: Consequently, to ensure that `ExposureClass`es in-use are always present in the system until the last referring `Shoot` gets deleted, the controller adds a finalizer which is only released when there is no `Shoot` referencing the `ExposureClass` anymore. +### [Gardenlet Lifecycle Controller](../../pkg/controllermanager/controller/gardenletlifecycle) + +The "Lifecycle" reconciler processes `Seed` and self-hosted `Shoot` objects which are enqueued every 10 seconds in order to check if the responsible +`gardenlet` is still responding and operable. Therefore, it checks renewals via `Lease` objects in the garden cluster +which are renewed regularly by the `gardenlet`. + +In case a `Lease` is not renewed for the configured amount in `config.controllers.seed.monitorPeriod.duration`: + +1. The reconciler assumes that the `gardenlet` stopped operating and updates the `GardenletReady` condition to `Unknown`. +2. Additionally, the conditions and constraints of all affected `Shoot` resources are set to `Unknown` as well, + because a striking `gardenlet` won't be able to maintain these conditions anymore. +3. For `Seed`s: If the gardenlet's client certificate has expired (identified based on the `.status.clientCertificateExpirationTimestamp` field in the `Seed` resource) and if it is managed by a `ManagedSeed`, then this will be triggered for a reconciliation. This will trigger the bootstrapping process again and allows gardenlets to obtain a fresh client certificate. + ### [`ManagedSeedSet` Controller](../../pkg/controllermanager/controller/managedseedset) `ManagedSeedSet` objects maintain a stable set of replicas of `ManagedSeed`s, i.e. they guarantee the availability of a specified number of identical `ManagedSeed`s on an equal number of identical `Shoot`s. @@ -321,19 +334,6 @@ This reconciler reconciles `Seed` objects and checks whether all `ControllerInst Concretely, all three conditions `Valid`, `Installed`, and `Healthy` must have status `True` and the `Progressing` condition must have status `False`. Based on this check, it maintains the `ExtensionsReady` condition in the respective `Seed`'s `.status.conditions` list. -#### ["Lifecycle" Reconciler](../../pkg/controllermanager/controller/seed/lifecycle) - -The "Lifecycle" reconciler processes `Seed` objects which are enqueued every 10 seconds in order to check if the responsible -`gardenlet` is still responding and operable. Therefore, it checks renewals via `Lease` objects of the seed in the garden cluster -which are renewed regularly by the `gardenlet`. - -In case a `Lease` is not renewed for the configured amount in `config.controllers.seed.monitorPeriod.duration`: - -1. The reconciler assumes that the `gardenlet` stopped operating and updates the `GardenletReady` condition to `Unknown`. -2. Additionally, the conditions and constraints of all `Shoot` resources scheduled on the affected seed are set to `Unknown` as well, - because a striking `gardenlet` won't be able to maintain these conditions any more. -3. If the gardenlet's client certificate has expired (identified based on the `.status.clientCertificateExpirationTimestamp` field in the `Seed` resource) and if it is managed by a `ManagedSeed`, then this will be triggered for a reconciliation. This will trigger the bootstrapping process again and allows gardenlets to obtain a fresh client certificate. - #### ["Reference" Reconciler](../../pkg/controllermanager/controller/seed/reference) Seed objects may specify references to other objects in the `garden` namespace in the garden cluster which are required for certain features. diff --git a/docs/concepts/gardenlet.md b/docs/concepts/gardenlet.md index e774a703786..8b502fa966b 100644 --- a/docs/concepts/gardenlet.md +++ b/docs/concepts/gardenlet.md @@ -177,7 +177,8 @@ the gardenlet is using `Lease` objects for heart beats of the seed cluster. Every two seconds, the gardenlet checks that the seed cluster's `/healthz` endpoint returns HTTP status code 200. If that is the case, the gardenlet renews the lease in the Garden cluster in the `gardener-system-seed-lease` namespace and updates -the `GardenletReady` condition in the `status.conditions` field of the `Seed` resource. For more information, see [this section](#lease-reconciler). +the `GardenletReady` condition in the `status.conditions` field of the `Seed` resource. +For more information, see [this section](#lease-reconciler). Similar to the `node-lifecycle-controller` inside the `kube-controller-manager`, the `gardener-controller-manager` features a `seed-lifecycle-controller` that sets @@ -486,6 +487,20 @@ A pod is considered stale when: - it was terminated with reason `NodeAffinity`. - it is stuck in termination (i.e., if its `deletionTimestamp` is more than `5m` ago). +#### ["Lease" Reconciler](../../pkg/gardenlet/controller/shoot/lease) + +This reconciler is only enabled for self-hosted shoot clusters. +It checks whether the connection to the shoot cluster's `/healthz` endpoint works. +If this succeeds, then it renews a `Lease` resource in the garden cluster in the same namespace as the `Shoot` the `gardenlet` is responsible for. +This indicates a heartbeat to the external world, and internally the `gardenlet` sets its health status to `true`. +In addition, the `GardenletReady` condition in the `status` of the `Shoot` is set to `True`. +The whole process is similar to what the `kubelet` does to report heartbeats for its `Node` resource and its `KubeletReady` condition. +For more information, see [this section](#heartbeats). + +If the connection to the `/healthz` endpoint or the update of the `Lease` fails, then the internal health status of `gardenlet` is set to `false`. +Also, this internal health status is set to `false` automatically after some time, in case the controller gets stuck for whatever reason. +This internal health status is available via the `gardenlet`'s `/healthz` endpoint and is used for the `livenessProbe` in the `gardenlet` pod. + #### ["State" Reconciler](../../pkg/gardenlet/controller/shoot/state) This reconciler periodically (default: every `6h`) performs backups of the state of `Shoot` clusters and persists them into `ShootState` resources into the same namespace as the `Shoot`s in the garden cluster. diff --git a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go index a51d28963d9..354257c635a 100644 --- a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go +++ b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go @@ -14,6 +14,7 @@ import ( "github.com/go-logr/logr" admissionv1 "k8s.io/api/admission/v1" certificatesv1 "k8s.io/api/certificates/v1" + coordinationv1 "k8s.io/api/coordination/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -34,6 +35,7 @@ var ( // group and the resource (but it ignores the version). certificateSigningRequestResource = certificatesv1.Resource("certificatesigningrequests") gardenletResource = seedmanagementv1alpha1.Resource("gardenlets") + leaseResource = coordinationv1.Resource("leases") projectResource = gardencorev1beta1.Resource("projects") shootResource = gardencorev1beta1.Resource("shoots") ) @@ -67,7 +69,10 @@ func (h *Handler) Handle(_ context.Context, request admission.Request) admission return h.admitCertificateSigningRequest(gardenletShootInfo, userType, request) case gardenletResource: - return h.admitGardenlet(gardenletShootInfo, request) + return h.admitCreateWithResourcePrefix(gardenletShootInfo, request) + + case leaseResource: + return h.admitCreateWithResourcePrefix(gardenletShootInfo, request) default: log.Info( @@ -108,23 +113,23 @@ func (h *Handler) admitCertificateSigningRequest(gardenletShootInfo types.Namesp return h.admit(gardenletShootInfo, types.NamespacedName{Name: name, Namespace: namespace}) } -func (h *Handler) admitGardenlet(gardenletShootInfo types.NamespacedName, request admission.Request) admission.Response { +func (h *Handler) admit(gardenletShootInfo, objectShootInfo types.NamespacedName) admission.Response { + // Allow request if the shoot the gardenlet is responsible for matches with the shoot related to the object. + if gardenletShootInfo.Name == objectShootInfo.Name && gardenletShootInfo.Namespace == objectShootInfo.Namespace { + return admission.Allowed("") + } + + return admission.Errored(http.StatusForbidden, fmt.Errorf("object does not belong to shoot %s", gardenletShootInfo)) +} + +func (h *Handler) admitCreateWithResourcePrefix(gardenletShootInfo types.NamespacedName, request admission.Request) admission.Response { if request.Operation != admissionv1.Create { return admission.Errored(http.StatusBadRequest, fmt.Errorf("unexpected operation: %q", request.Operation)) } if !strings.HasPrefix(request.Name, gardenletutils.ResourcePrefixSelfHostedShoot) { - return admission.Errored(http.StatusBadRequest, fmt.Errorf("the Gardenlet resources for self-hosted shoots must be prefixed with %q", gardenletutils.ResourcePrefixSelfHostedShoot)) + return admission.Errored(http.StatusBadRequest, fmt.Errorf("the resource for self-hosted shoots must be prefixed with %q", gardenletutils.ResourcePrefixSelfHostedShoot)) } return h.admit(gardenletShootInfo, types.NamespacedName{Name: strings.TrimPrefix(request.Name, gardenletutils.ResourcePrefixSelfHostedShoot), Namespace: request.Namespace}) } - -func (h *Handler) admit(gardenletShootInfo, objectShootInfo types.NamespacedName) admission.Response { - // Allow request if the shoot the gardenlet is responsible for matches with the shoot related to the object. - if gardenletShootInfo.Name == objectShootInfo.Name && gardenletShootInfo.Namespace == objectShootInfo.Namespace { - return admission.Allowed("") - } - - return admission.Errored(http.StatusForbidden, fmt.Errorf("object does not belong to shoot %s", gardenletShootInfo)) -} diff --git a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go index e11e6191642..117fca007db 100644 --- a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go +++ b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go @@ -16,6 +16,7 @@ import ( admissionv1 "k8s.io/api/admission/v1" authenticationv1 "k8s.io/api/authentication/v1" certificatesv1 "k8s.io/api/certificates/v1" + coordinationv1 "k8s.io/api/coordination/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer/json" @@ -25,6 +26,7 @@ import ( . "github.com/gardener/gardener/pkg/admissioncontroller/webhook/admission/shootrestriction" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + seedmanagementv1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/logger" mockcache "github.com/gardener/gardener/third_party/mock/controller-runtime/cache" @@ -274,6 +276,154 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d }) }) }) + + Context("when requested for Gardenlets", func() { + var name string + + BeforeEach(func() { + name = "self-hosted-shoot-foo" + + request.Name = name + request.UserInfo = gardenletUser + request.Resource = metav1.GroupVersionResource{ + Group: seedmanagementv1alpha1.SchemeGroupVersion.Group, + Version: seedmanagementv1alpha1.SchemeGroupVersion.Version, + Resource: "gardenlets", + } + }) + + DescribeTable("should not allow the request because no allowed verb", + func(operation admissionv1.Operation) { + request.Operation = operation + + Expect(handler.Handle(ctx, request)).To(Equal(admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: int32(http.StatusBadRequest), + Message: fmt.Sprintf("unexpected operation: %q", operation), + }, + }, + })) + }, + + Entry("update", admissionv1.Update), + Entry("delete", admissionv1.Delete), + ) + + Context("when operation is create", func() { + BeforeEach(func() { + request.Operation = admissionv1.Create + }) + + It("should return an error because resource name is not prefixed", func() { + request.Name = "foo" + + Expect(handler.Handle(ctx, request)).To(Equal(admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: int32(http.StatusBadRequest), + Message: `the resource for self-hosted shoots must be prefixed with "self-hosted-shoot-"`, + }, + }, + })) + }) + + It("should return an error because the requestor is not responsible for the resource", func() { + Expect(handler.Handle(ctx, request)).To(Equal(admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: int32(http.StatusForbidden), + Message: "object does not belong to shoot " + shootNamespace + "/" + shootName, + }, + }, + })) + }) + + It("should return success because the requestor is responsible for the resource", func() { + request.Name = "self-hosted-shoot-" + shootName + request.Namespace = shootNamespace + + Expect(handler.Handle(ctx, request)).To(Equal(responseAllowed)) + }) + }) + }) + + Context("when requested for Leases", func() { + var name string + + BeforeEach(func() { + name = "self-hosted-shoot-foo" + + request.Name = name + request.UserInfo = gardenletUser + request.Resource = metav1.GroupVersionResource{ + Group: coordinationv1.SchemeGroupVersion.Group, + Version: coordinationv1.SchemeGroupVersion.Version, + Resource: "leases", + } + }) + + DescribeTable("should not allow the request because no allowed verb", + func(operation admissionv1.Operation) { + request.Operation = operation + + Expect(handler.Handle(ctx, request)).To(Equal(admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: int32(http.StatusBadRequest), + Message: fmt.Sprintf("unexpected operation: %q", operation), + }, + }, + })) + }, + + Entry("update", admissionv1.Update), + Entry("delete", admissionv1.Delete), + ) + + Context("when operation is create", func() { + BeforeEach(func() { + request.Operation = admissionv1.Create + }) + + It("should return an error because resource name is not prefixed", func() { + request.Name = "foo" + + Expect(handler.Handle(ctx, request)).To(Equal(admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: int32(http.StatusBadRequest), + Message: `the resource for self-hosted shoots must be prefixed with "self-hosted-shoot-"`, + }, + }, + })) + }) + + It("should return an error because the requestor is not responsible for the resource", func() { + Expect(handler.Handle(ctx, request)).To(Equal(admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: int32(http.StatusForbidden), + Message: "object does not belong to shoot " + shootNamespace + "/" + shootName, + }, + }, + })) + }) + + It("should return success because the requestor is responsible for the resource", func() { + request.Name = "self-hosted-shoot-" + shootName + request.Namespace = shootNamespace + + Expect(handler.Handle(ctx, request)).To(Equal(responseAllowed)) + }) + }) + }) }) Context("gardenadm client", func() { diff --git a/pkg/admissioncontroller/webhook/auth/seed/authorizer.go b/pkg/admissioncontroller/webhook/auth/seed/authorizer.go index 1dce6811831..fbcc5e57f14 100644 --- a/pkg/admissioncontroller/webhook/auth/seed/authorizer.go +++ b/pkg/admissioncontroller/webhook/auth/seed/authorizer.go @@ -295,8 +295,13 @@ func (a *authorizer) authorizeLease(requestAuthorizer *authwebhook.RequestAuthor } // This is needed if the seed cluster is a garden cluster at the same time. - if attrs.GetName() == "gardenlet-leader-election" && - slices.Contains([]string{"create", "get", "list", "watch", "update"}, attrs.GetVerb()) { + // TODO(rfranzke): Remove this logic once we only support gardener-operator-based Gardener deployments (in this + // case, the seed authorizer would not handle gardenlet's leader-election Lease requests since they are not + // performed in the virtual/garden cluster but in the runtime cluster). + if attrs.GetName() == "gardenlet-leader-election" { + if ok, reason := authwebhook.CheckVerb(requestAuthorizer.Log, attrs, "create", "get", "list", "update", "watch"); !ok { + return auth.DecisionNoOpinion, reason, nil + } return auth.DecisionAllow, "", nil } diff --git a/pkg/admissioncontroller/webhook/auth/seed/authorizer_test.go b/pkg/admissioncontroller/webhook/auth/seed/authorizer_test.go index bb798068779..bf84bf290c8 100644 --- a/pkg/admissioncontroller/webhook/auth/seed/authorizer_test.go +++ b/pkg/admissioncontroller/webhook/auth/seed/authorizer_test.go @@ -2406,6 +2406,44 @@ var _ = Describe("Seed", func() { } }) + When("resource is gardenlet's leader election lease", func() { + BeforeEach(func() { + attrs.Name = "gardenlet-leader-election" + }) + + DescribeTable("should allow without consulting the graph for allowed verbs", + func(verb string) { + attrs.Verb = verb + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionAllow)) + Expect(reason).To(BeEmpty()) + }, + + Entry("create", "create"), + Entry("get", "get"), + Entry("list", "list"), + Entry("watch", "watch"), + Entry("update", "update"), + ) + + DescribeTable("should deny without consulting the graph for disallowed verbs", + func(verb string) { + attrs.Verb = verb + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("only the following verbs are allowed for this resource type: [create get list update watch]")) + }, + + Entry("patch", "patch"), + Entry("delete", "delete"), + Entry("deletecollection", "deletecollection"), + ) + }) + DescribeTable("should allow without consulting the graph because verb is create", func(verb string) { attrs.Verb = verb diff --git a/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go b/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go index feb9b1eddc7..d6348896ae0 100644 --- a/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go +++ b/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go @@ -12,6 +12,7 @@ import ( "github.com/go-logr/logr" certificatesv1 "k8s.io/api/certificates/v1" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,6 +63,7 @@ var ( eventCoreResource = corev1.Resource("events") eventResource = eventsv1.Resource("events") gardenletResource = seedmanagementv1alpha1.Resource("gardenlets") + leaseResource = coordinationv1.Resource("leases") projectResource = gardencorev1beta1.Resource("projects") secretResource = corev1.Resource("secrets") secretBindingResource = gardencorev1beta1.Resource("secretbindings") @@ -119,6 +121,12 @@ func (a *authorizer) Authorize(ctx context.Context, attrs auth.Attributes) (auth authwebhook.WithFieldSelectors(map[string]string{metav1.ObjectNameField: gardenletutils.ResourcePrefixSelfHostedShoot + requestAuthorizer.ToName}), ) + case leaseResource: + return requestAuthorizer.Check(graph.VertexTypeLease, attrs, + authwebhook.WithAllowedVerbs("get", "update", "patch", "list", "watch"), + authwebhook.WithAlwaysAllowedVerbs("create"), + ) + case secretResource: return a.authorizeSecret(ctx, requestAuthorizer, attrs) @@ -131,7 +139,8 @@ func (a *authorizer) Authorize(ctx context.Context, attrs auth.Attributes) (auth } return requestAuthorizer.Check(graph.VertexTypeShoot, attrs, - authwebhook.WithAllowedVerbs("get", "list", "watch"), + authwebhook.WithAllowedVerbs("update", "patch"), + authwebhook.WithAllowedSubresources("status"), ) default: diff --git a/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go b/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go index 8e912903d01..baad1267cca 100644 --- a/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go +++ b/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go @@ -13,6 +13,7 @@ import ( . "github.com/onsi/gomega" "go.uber.org/mock/gomock" certificatesv1 "k8s.io/api/certificates/v1" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -533,6 +534,86 @@ var _ = Describe("Shoot", func() { ) }) + Context("when requested for Leases", func() { + var ( + name, namespace string + attrs *auth.AttributesRecord + ) + + BeforeEach(func() { + name, namespace = "foo", "bar" + attrs = &auth.AttributesRecord{ + User: gardenletUser, + Name: name, + Namespace: namespace, + APIGroup: coordinationv1.SchemeGroupVersion.Group, + Resource: "leases", + ResourceRequest: true, + Verb: "get", + } + }) + + DescribeTable("should allow without consulting the graph because verb is create", + func(verb string) { + attrs.Verb = verb + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionAllow)) + Expect(reason).To(BeEmpty()) + }, + + Entry("create", "create"), + ) + + DescribeTable("should return correct result if path exists", + func(verb string) { + attrs.Verb = verb + + graph.EXPECT().HasPathFrom(graphutils.VertexTypeLease, namespace, name, graphutils.VertexTypeShoot, shootNamespace, shootName).Return(true) + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionAllow)) + Expect(reason).To(BeEmpty()) + + graph.EXPECT().HasPathFrom(graphutils.VertexTypeLease, namespace, name, graphutils.VertexTypeShoot, shootNamespace, shootName).Return(false) + decision, reason, err = authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("no relationship found")) + }, + + Entry("get", "get"), + Entry("list", "list"), + Entry("watch", "watch"), + Entry("update", "update"), + Entry("patch", "patch"), + ) + + DescribeTable("should have no opinion because verb is not allowed", + func(verb string) { + attrs.Verb = verb + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("only the following verbs are allowed for this resource type: [create get list patch update watch]")) + + }, + Entry("delete", "delete"), + Entry("deletecollection", "deletecollection"), + ) + + It("should have no opinion because no allowed subresource", func() { + attrs.Subresource = "foo" + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("only the following subresources are allowed for this resource type: []")) + }) + }) + Context("when requested for Secrets", func() { var ( name, namespace string @@ -666,16 +747,48 @@ var _ = Describe("Shoot", func() { decision, reason, err := authorizer.Authorize(ctx, attrs) Expect(err).NotTo(HaveOccurred()) Expect(decision).To(Equal(auth.DecisionNoOpinion)) - Expect(reason).To(ContainSubstring("only the following verbs are allowed for this resource type: [get list watch]")) + Expect(reason).To(ContainSubstring("only the following verbs are allowed for this resource type: [patch update]")) }, Entry("create", "create"), - Entry("update", "update"), - Entry("patch", "patch"), Entry("delete", "delete"), Entry("deletecollection", "deletecollection"), ) + + It("should have no opinion because no allowed subresource", func() { + attrs.Subresource = "foo" + attrs.Verb = "patch" + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("only the following subresources are allowed for this resource type: [status]")) + }) + + DescribeTable("should return correct result if path exists", + func(verb, subresource string) { + attrs.Verb = verb + attrs.Subresource = subresource + + graph.EXPECT().HasPathFrom(graphutils.VertexTypeShoot, namespace, name, graphutils.VertexTypeShoot, shootNamespace, shootName).Return(true) + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionAllow)) + Expect(reason).To(BeEmpty()) + + graph.EXPECT().HasPathFrom(graphutils.VertexTypeShoot, namespace, name, graphutils.VertexTypeShoot, shootNamespace, shootName).Return(false) + decision, reason, err = authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("no relationship found")) + }, + + Entry("patch w/o subresource", "patch", ""), + Entry("patch w/ status subresource", "patch", "status"), + Entry("update w/o subresource", "update", ""), + Entry("update w/ status subresource", "update", "status"), + ) }) }) diff --git a/pkg/apis/core/v1beta1/constants/types_constants.go b/pkg/apis/core/v1beta1/constants/types_constants.go index 3991c6e49e0..f5af73e1e92 100644 --- a/pkg/apis/core/v1beta1/constants/types_constants.go +++ b/pkg/apis/core/v1beta1/constants/types_constants.go @@ -322,6 +322,8 @@ const ( // ShootDisableIstioTLSTermination is a constant for an annotation on a Shoot stating that the Istio TLS termination // for its kube-apiserver shall be disabled. ShootDisableIstioTLSTermination = "shoot.gardener.cloud/disable-istio-tls-termination" + // ShootIsSelfHosted is a constant for a label on a Shoot indicating that it is self-hosted. + ShootIsSelfHosted = "shoot.gardener.cloud/self-hosted" // ShootAlphaControlPlaneScaleDownDisabled is a constant for an annotation on the Shoot resource stating that the // automatic scale-down shall be disabled for the etcd, kube-apiserver, kube-controller-manager. diff --git a/pkg/apis/core/v1beta1/types_seed.go b/pkg/apis/core/v1beta1/types_seed.go index 647b8964d9f..8768f9d3605 100644 --- a/pkg/apis/core/v1beta1/types_seed.go +++ b/pkg/apis/core/v1beta1/types_seed.go @@ -490,8 +490,6 @@ const ( SeedBackupBucketsReady ConditionType = "BackupBucketsReady" // SeedExtensionsReady is a constant for a condition type indicating that the extensions are ready. SeedExtensionsReady ConditionType = "ExtensionsReady" - // SeedGardenletReady is a constant for a condition type indicating that the Gardenlet is ready. - SeedGardenletReady ConditionType = "GardenletReady" // SeedSystemComponentsHealthy is a constant for a condition type indicating the system components health. SeedSystemComponentsHealthy ConditionType = "SeedSystemComponentsHealthy" // SeedEmergencyStopShootReconciliations is a constant for a condition type indicating disabled shoot reconciliations. diff --git a/pkg/apis/core/v1beta1/types_utils.go b/pkg/apis/core/v1beta1/types_utils.go index 99f43a58658..e3ceb9a18e6 100644 --- a/pkg/apis/core/v1beta1/types_utils.go +++ b/pkg/apis/core/v1beta1/types_utils.go @@ -62,4 +62,7 @@ const ( // ManagedResourceProgressingRolloutStuck is a constant for a reason in a condition that indicates // managed resource progressing condition is stuck in the true state for more than the threshold time. ManagedResourceProgressingRolloutStuck = "ProgressingRolloutStuck" + + // GardenletReady is a constant for a condition type indicating that the Gardenlet is ready. + GardenletReady ConditionType = "GardenletReady" ) diff --git a/pkg/apiserver/registry/core/shoot/strategy.go b/pkg/apiserver/registry/core/shoot/strategy.go index 9abdddfce49..a2e1a0ea42f 100644 --- a/pkg/apiserver/registry/core/shoot/strategy.go +++ b/pkg/apiserver/registry/core/shoot/strategy.go @@ -11,6 +11,7 @@ import ( "time" apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -190,6 +191,15 @@ func (shootStrategy) Canonicalize(obj runtime.Object) { shoot.Spec.Kubernetes.ClusterAutoscaler.MaxEmptyBulkDelete = nil } gardenerutils.MaintainSeedNameLabels(shoot, shoot.Spec.SeedName, shoot.Status.SeedName) + maintainIsSelfHostedLabel(shoot) +} + +func maintainIsSelfHostedLabel(shoot *core.Shoot) { + if gardencorehelper.IsShootSelfHosted(shoot.Spec.Provider.Workers) { + metav1.SetMetaDataLabel(&shoot.ObjectMeta, v1beta1constants.ShootIsSelfHosted, "true") + } else { + delete(shoot.Labels, v1beta1constants.ShootIsSelfHosted) + } } func (shootStrategy) AllowCreateOnUpdate() bool { diff --git a/pkg/apiserver/registry/core/shoot/strategy_test.go b/pkg/apiserver/registry/core/shoot/strategy_test.go index 03bad3a609b..2593781f88a 100644 --- a/pkg/apiserver/registry/core/shoot/strategy_test.go +++ b/pkg/apiserver/registry/core/shoot/strategy_test.go @@ -667,6 +667,26 @@ var _ = Describe("Strategy", func() { Expect(shoot.Spec.Kubernetes.ClusterAutoscaler.MaxEmptyBulkDelete).To(BeNil()) }) }) + + Context("self-hosted shoots", func() { + It("should correctly add the self-hosted label", func() { + shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, core.Worker{ControlPlane: &core.WorkerControlPlane{}}) + + strategy.Canonicalize(shoot) + + Expect(shoot.Labels).To(Equal(map[string]string{ + "shoot.gardener.cloud/self-hosted": "true", + })) + }) + + It("should correctly remove the self-hosted label", func() { + metav1.SetMetaDataLabel(&shoot.ObjectMeta, "shoot.gardener.cloud/self-hosted", "true") + + strategy.Canonicalize(shoot) + + Expect(shoot.Labels).To(BeEmpty()) + }) + }) }) Context("BindingStrategy", func() { diff --git a/pkg/component/garden/backupentry/backupentry.go b/pkg/component/garden/backupentry/backupentry.go index 24cbd393cc7..4cd8600e649 100644 --- a/pkg/component/garden/backupentry/backupentry.go +++ b/pkg/component/garden/backupentry/backupentry.go @@ -52,7 +52,7 @@ type Values struct { } // New creates a new instance of DeployWaiter for a BackupEntry. It takes a garden client and returns a deployer for a -// // core.gardener.cloud/v1beta1.BackupEntry resource in the garden cluster. +// core.gardener.cloud/v1beta1.BackupEntry resource in the garden cluster. func New( log logr.Logger, client client.Client, diff --git a/pkg/controllermanager/controller/add.go b/pkg/controllermanager/controller/add.go index 70b862b4158..d4fad5a7fd9 100644 --- a/pkg/controllermanager/controller/add.go +++ b/pkg/controllermanager/controller/add.go @@ -11,6 +11,7 @@ import ( kubernetesclientset "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/manager" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/controllermanager/controller/bastion" "github.com/gardener/gardener/pkg/controllermanager/controller/certificatesigningrequest" @@ -20,6 +21,7 @@ import ( "github.com/gardener/gardener/pkg/controllermanager/controller/credentialsbinding" "github.com/gardener/gardener/pkg/controllermanager/controller/event" "github.com/gardener/gardener/pkg/controllermanager/controller/exposureclass" + "github.com/gardener/gardener/pkg/controllermanager/controller/gardenletlifecycle" "github.com/gardener/gardener/pkg/controllermanager/controller/managedseedset" "github.com/gardener/gardener/pkg/controllermanager/controller/namespacedcloudprofile" "github.com/gardener/gardener/pkg/controllermanager/controller/project" @@ -92,6 +94,13 @@ func AddToManager(ctx context.Context, mgr manager.Manager, cfg *controllermanag return fmt.Errorf("failed adding ExposureClass controller: %w", err) } + if err := (&gardenletlifecycle.Reconciler{ + Config: *cfg.Controllers.Seed, + LeaseNamespace: gardencorev1beta1.GardenerSeedLeaseNamespace, + }).AddToManager(mgr); err != nil { + return fmt.Errorf("failed adding gardenlet lifecycle reconciler: %w", err) + } + if err := (&managedseedset.Reconciler{ Config: *cfg.Controllers.ManagedSeedSet, }).AddToManager(ctx, mgr); err != nil { diff --git a/pkg/controllermanager/controller/exposureclass/exposureclass_suite_test.go b/pkg/controllermanager/controller/exposureclass/exposureclass_suite_test.go index 575a972e5d6..9ebaf9bbb94 100644 --- a/pkg/controllermanager/controller/exposureclass/exposureclass_suite_test.go +++ b/pkg/controllermanager/controller/exposureclass/exposureclass_suite_test.go @@ -11,7 +11,7 @@ import ( . "github.com/onsi/gomega" ) -func TestControllerRegistration(t *testing.T) { +func TestExposureClass(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "ControllerManager Controller ExposureClass Suite") } diff --git a/pkg/controllermanager/controller/gardenletlifecycle/add.go b/pkg/controllermanager/controller/gardenletlifecycle/add.go new file mode 100644 index 00000000000..dc24cd6bd6d --- /dev/null +++ b/pkg/controllermanager/controller/gardenletlifecycle/add.go @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package gardenletlifecycle + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/clock" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" +) + +// ControllerName is the name of this controller. +const ControllerName = "gardenlet-lifecycle" + +// Request contains the namespace/name of the object as well as information whether it is a self-hosted Shoot. +type Request struct { + reconcile.Request + + IsSelfHostedShoot bool +} + +// AddToManager adds Reconciler to the given manager. +func (r *Reconciler) AddToManager(mgr manager.Manager) error { + if r.Client == nil { + r.Client = mgr.GetClient() + } + if r.Clock == nil { + r.Clock = clock.RealClock{} + } + + shootIsSelfHostedPredicate, err := predicate.LabelSelectorPredicate(metav1.LabelSelector{MatchLabels: map[string]string{v1beta1constants.ShootIsSelfHosted: "true"}}) + if err != nil { + return fmt.Errorf("failed computing shoot-is-self-hosted predicate: %w", err) + } + + return builder. + TypedControllerManagedBy[Request](mgr). + Named(ControllerName). + Watches(&gardencorev1beta1.Seed{}, r.EventHandler(), builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create))). + Watches(&gardencorev1beta1.Shoot{}, r.EventHandler(), builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create), shootIsSelfHostedPredicate)). + WithOptions(controller.TypedOptions[Request]{ + MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), + ReconciliationTimeout: r.Config.SyncPeriod.Duration, + }). + Complete(r) +} + +// EventHandler returns a handler for events. +func (r *Reconciler) EventHandler() handler.TypedEventHandler[client.Object, Request] { + return &handler.TypedFuncs[client.Object, Request]{ + CreateFunc: func(_ context.Context, e event.TypedCreateEvent[client.Object], q workqueue.TypedRateLimitingInterface[Request]) { + if e.Object != nil { + _, isSelfHostedShoot := e.Object.(*gardencorev1beta1.Shoot) + q.Add(Request{ + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: e.Object.GetName(), Namespace: e.Object.GetNamespace()}}, + IsSelfHostedShoot: isSelfHostedShoot, + }) + } + }, + } +} diff --git a/pkg/controllermanager/controller/gardenletlifecycle/add_test.go b/pkg/controllermanager/controller/gardenletlifecycle/add_test.go new file mode 100644 index 00000000000..947740c2c8e --- /dev/null +++ b/pkg/controllermanager/controller/gardenletlifecycle/add_test.go @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package gardenletlifecycle_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + "go.uber.org/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + . "github.com/gardener/gardener/pkg/controllermanager/controller/gardenletlifecycle" + mockworkqueue "github.com/gardener/gardener/third_party/mock/client-go/util/workqueue" +) + +var _ = Describe("Add", func() { + var reconciler *Reconciler + + BeforeEach(func() { + reconciler = &Reconciler{} + }) + + Describe("#EventHandler", func() { + var ( + ctx = context.Background() + hdlr handler.TypedEventHandler[client.Object, Request] + queue *mockworkqueue.MockTypedRateLimitingInterface[Request] + ) + + BeforeEach(func() { + hdlr = reconciler.EventHandler() + queue = mockworkqueue.NewMockTypedRateLimitingInterface[Request](gomock.NewController(GinkgoT())) + }) + + It("should correctly enqueue Seeds", func() { + obj := &gardencorev1beta1.Seed{ObjectMeta: metav1.ObjectMeta{Name: "seed"}} + + queue.EXPECT().Add(Request{ + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "seed"}}, + IsSelfHostedShoot: false, + }) + + hdlr.Create(ctx, event.CreateEvent{Object: obj}, queue) + }) + + It("should correctly enqueue Shoots", func() { + obj := &gardencorev1beta1.Shoot{ObjectMeta: metav1.ObjectMeta{Name: "shoot", Namespace: "shoot-namespace"}} + + queue.EXPECT().Add(Request{ + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "shoot", Namespace: "shoot-namespace"}}, + IsSelfHostedShoot: true, + }) + + hdlr.Create(ctx, event.CreateEvent{Object: obj}, queue) + }) + + It("should not enqueue the object for Update events", func() { + hdlr.Update(ctx, event.UpdateEvent{}, queue) + }) + + It("should not enqueue the object for Delete events", func() { + hdlr.Delete(ctx, event.DeleteEvent{}, queue) + }) + + It("should not enqueue the object for Generic events", func() { + hdlr.Generic(ctx, event.GenericEvent{}, queue) + }) + }) +}) diff --git a/pkg/controllermanager/controller/gardenletlifecycle/gardenletlifecycle_suite_test.go b/pkg/controllermanager/controller/gardenletlifecycle/gardenletlifecycle_suite_test.go new file mode 100644 index 00000000000..6a9d82905ae --- /dev/null +++ b/pkg/controllermanager/controller/gardenletlifecycle/gardenletlifecycle_suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package gardenletlifecycle_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestGardenletLifecycle(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ControllerManager Controller GardenletLifecycle Suite") +} diff --git a/pkg/controllermanager/controller/seed/lifecycle/reconciler.go b/pkg/controllermanager/controller/gardenletlifecycle/reconciler.go similarity index 54% rename from pkg/controllermanager/controller/seed/lifecycle/reconciler.go rename to pkg/controllermanager/controller/gardenletlifecycle/reconciler.go index 6e0778f5ed9..4475f53e581 100644 --- a/pkg/controllermanager/controller/seed/lifecycle/reconciler.go +++ b/pkg/controllermanager/controller/gardenletlifecycle/reconciler.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package lifecycle +package gardenletlifecycle import ( "context" @@ -23,12 +23,14 @@ import ( controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/utils/flow" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) -// Reconciler reconciles Seeds and checks whether the responsible gardenlet is regularly sending heartbeats. If not, it -// sets the GardenletReady condition of the Seed to Unknown after some grace period passed. If the gardenlet still did -// not send heartbeats and another grace period passed then also all shoot conditions and constraints are set to Unknown. +// Reconciler reconciles Seeds or self-hosted Shoots and checks whether the responsible gardenlet is regularly sending +// heartbeats. If not, it sets the GardenletReady condition to Unknown after some grace period passed. If the gardenlet +// still did not send heartbeats and another grace period passed then also all (other) Shoot conditions and constraints +// are set to Unknown. type Reconciler struct { Client client.Client Config controllermanagerconfigv1alpha1.SeedControllerConfiguration @@ -36,28 +38,30 @@ type Reconciler struct { LeaseNamespace string } -// Reconcile reconciles Seeds and checks whether the responsible gardenlet is regularly sending heartbeats. If not, it -// sets the GardenletReady condition of the Seed to Unknown after some grace period passed. If the gardenlet still did -// not send heartbeats and another grace period passed then also all shoot conditions and constraints are set to Unknown. -func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { +// Reconcile reconciles Seeds or self-hosted Shoots and checks whether the responsible gardenlet is regularly sending +// heartbeats. If not, it sets the GardenletReady condition to Unknown after some grace period passed. If the gardenlet +// still did not send heartbeats and another grace period passed then also all (other) Shoot conditions and constraints +// are set to Unknown. +func (r *Reconciler) Reconcile(ctx context.Context, req Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - seed := &gardencorev1beta1.Seed{} - if err := r.Client.Get(ctx, req.NamespacedName, seed); err != nil { + obj := newObj(req) + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { if apierrors.IsNotFound(err) { log.V(1).Info("Object is gone, stop reconciling") return reconcile.Result{}, nil } return reconcile.Result{}, fmt.Errorf("error retrieving object from store: %w", err) } + log = log.WithValues("object", client.ObjectKeyFromObject(obj), "isSelfHostedShoot", req.IsSelfHostedShoot) - // New seeds don't have conditions - gardenlet never reported anything yet. Wait for grace period. - if len(seed.Status.Conditions) == 0 { + // New objects don't have conditions - gardenlet never reported anything yet. Wait for grace period. + if len(conditions(obj)) == 0 { return reconcile.Result{RequeueAfter: r.Config.SyncPeriod.Duration}, nil } lease := &coordinationv1.Lease{} - if err := r.Client.Get(ctx, client.ObjectKey{Namespace: r.LeaseNamespace, Name: seed.Name}, lease); client.IgnoreNotFound(err) != nil { + if err := r.Client.Get(ctx, r.leaseKey(req), lease); client.IgnoreNotFound(err) != nil { return reconcile.Result{}, err } @@ -69,28 +73,28 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco log.Info("Lease was not renewed in time", "renewTime", lease.Spec.RenewTime.UTC(), "now", r.Clock.Now().UTC(), - "seedMonitorPeriod", r.Config.MonitorPeriod.Duration, + "monitorPeriod", r.Config.MonitorPeriod.Duration, ) } - log.Info("Setting Seed status to 'Unknown' as gardenlet stopped reporting seed status") + log.Info("Setting GardenletReady condition status to 'Unknown' as gardenlet stopped updating its Lease") - bldr, err := v1beta1helper.NewConditionBuilder(gardencorev1beta1.SeedGardenletReady) + bldr, err := v1beta1helper.NewConditionBuilder(gardencorev1beta1.GardenletReady) if err != nil { return reconcile.Result{}, err } - conditionGardenletReady := v1beta1helper.GetCondition(seed.Status.Conditions, gardencorev1beta1.SeedGardenletReady) + conditionGardenletReady := v1beta1helper.GetCondition(conditions(obj), gardencorev1beta1.GardenletReady) if conditionGardenletReady != nil { bldr.WithOldCondition(*conditionGardenletReady) } bldr.WithStatus(gardencorev1beta1.ConditionUnknown) - bldr.WithReason("SeedStatusUnknown") - bldr.WithMessage("Gardenlet stopped posting seed status.") + bldr.WithReason("StatusUnknown") + bldr.WithMessage("Gardenlet stopped posting status updates.") if newCondition, update := bldr.WithClock(r.Clock).Build(); update { - seed.Status.Conditions = v1beta1helper.MergeConditions(seed.Status.Conditions, newCondition) - if err := r.Client.Status().Update(ctx, seed); err != nil { + setConditions(obj, v1beta1helper.MergeConditions(conditions(obj), newCondition)) + if err := r.Client.Status().Update(ctx, obj); err != nil { return reconcile.Result{}, err } conditionGardenletReady = &newCondition @@ -98,27 +102,29 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco // If the gardenlet's client certificate is expired and the seed belongs to a `ManagedSeed` then we reconcile it in // order to re-bootstrap the gardenlet. - if seed.Status.ClientCertificateExpirationTimestamp != nil && seed.Status.ClientCertificateExpirationTimestamp.UTC().Before(r.Clock.Now().UTC()) { - managedSeed, err := kubernetesutils.GetManagedSeedByName(ctx, r.Client, seed.Name) - if err != nil { - return reconcile.Result{}, err - } + if !req.IsSelfHostedShoot { + if seed := obj.(*gardencorev1beta1.Seed); seed.Status.ClientCertificateExpirationTimestamp != nil && seed.Status.ClientCertificateExpirationTimestamp.UTC().Before(r.Clock.Now().UTC()) { + managedSeed, err := kubernetesutils.GetManagedSeedByName(ctx, r.Client, seed.Name) + if err != nil { + return reconcile.Result{}, err + } - if managedSeed != nil { - log.Info("Triggering ManagedSeed reconciliation since gardenlet client certificate is expired", "managedSeed", client.ObjectKeyFromObject(managedSeed)) + if managedSeed != nil { + log.Info("Triggering ManagedSeed reconciliation since gardenlet client certificate is expired", "managedSeed", client.ObjectKeyFromObject(managedSeed)) - patch := client.MergeFrom(managedSeed.DeepCopy()) - metav1.SetMetaDataAnnotation(&managedSeed.ObjectMeta, v1beta1constants.GardenerOperation, v1beta1constants.GardenerOperationReconcile) - if err := r.Client.Patch(ctx, managedSeed, patch); err != nil { - return reconcile.Result{}, err + patch := client.MergeFrom(managedSeed.DeepCopy()) + metav1.SetMetaDataAnnotation(&managedSeed.ObjectMeta, v1beta1constants.GardenerOperation, v1beta1constants.GardenerOperationReconcile) + if err := r.Client.Patch(ctx, managedSeed, patch); err != nil { + return reconcile.Result{}, err + } } } } - // If the `GardenletReady` condition is `Unknown` for at least the configured `shootMonitorPeriod` then we will mark the conditions - // and constraints for all the shoots that belong to this seed as `Unknown`. The reason is that the gardenlet didn't send a heartbeat - // anymore, hence, it most likely didn't check the shoot status. This means that the current shoot status might not reflect the truth - // anymore. We are indicating this by marking it as `Unknown`. + // If the `GardenletReady` condition is `Unknown` for at least the configured `shootMonitorPeriod` then we will mark + // the conditions and constraints for affected Shoots as `Unknown`. The reason is that the gardenlet didn't send a + // heartbeat anymore, hence, it most likely didn't check the shoot status. This means that the current shoot status + // might not reflect the truth anymore. We are indicating this by marking it as `Unknown`. if conditionGardenletReady != nil && conditionGardenletReady.LastTransitionTime.UTC().Add(r.Config.ShootMonitorPeriod.Duration).After(r.Clock.Now().UTC()) { return reconcile.Result{RequeueAfter: r.Config.SyncPeriod.Duration}, nil } @@ -128,21 +134,24 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco gardenletOfflineSince = conditionGardenletReady.LastTransitionTime.UTC() } - log.Info("Gardenlet has not sent heartbeats for at least the configured shoot monitor period, setting shoot conditions and constraints to 'Unknown' for all shoots on this seed", + log.Info("Gardenlet has not sent heartbeats for at least the configured shoot monitor period, setting shoot conditions and constraints to 'Unknown' for all affected Shoots", "gardenletOfflineSince", gardenletOfflineSince, "now", r.Clock.Now().UTC(), "shootMonitorPeriod", r.Config.ShootMonitorPeriod.Duration, ) shootList := &gardencorev1beta1.ShootList{} - if err := r.Client.List(ctx, shootList, client.MatchingFields{core.ShootStatusSeedName: seed.Name}); err != nil { - return reconcile.Result{}, err + if req.IsSelfHostedShoot { + shootList.Items = append(shootList.Items, *obj.(*gardencorev1beta1.Shoot)) + } else { + if err := r.Client.List(ctx, shootList, client.MatchingFields{core.ShootStatusSeedName: req.Name}); err != nil { + return reconcile.Result{}, err + } } var fns []flow.TaskFn - for _, s := range shootList.Items { - shoot := s + for _, shoot := range shootList.Items { fns = append(fns, func(ctx context.Context) error { return setShootStatusToUnknown(ctx, r.Clock, r.Client, &shoot) }) @@ -194,3 +203,39 @@ func conditionMapToConditions(m map[gardencorev1beta1.ConditionType]gardencorev1 return output } + +func newObj(req Request) client.Object { + if req.IsSelfHostedShoot { + return &gardencorev1beta1.Shoot{ObjectMeta: metav1.ObjectMeta{Name: req.Name, Namespace: req.Namespace}} + } + return &gardencorev1beta1.Seed{ObjectMeta: metav1.ObjectMeta{Name: req.Name}} +} + +func conditions(o client.Object) []gardencorev1beta1.Condition { + switch obj := o.(type) { + case *gardencorev1beta1.Seed: + return obj.Status.Conditions + case *gardencorev1beta1.Shoot: + return obj.Status.Conditions + default: + panic("unexpected object") + } +} + +func setConditions(o client.Object, conditions []gardencorev1beta1.Condition) { + switch obj := o.(type) { + case *gardencorev1beta1.Seed: + obj.Status.Conditions = conditions + case *gardencorev1beta1.Shoot: + obj.Status.Conditions = conditions + default: + panic("unexpected object") + } +} + +func (r *Reconciler) leaseKey(req Request) client.ObjectKey { + if req.IsSelfHostedShoot { + return client.ObjectKey{Namespace: req.Namespace, Name: gardenlet.ResourcePrefixSelfHostedShoot + req.Name} + } + return client.ObjectKey{Namespace: r.LeaseNamespace, Name: req.Name} +} diff --git a/pkg/controllermanager/controller/managedseedset/add_test.go b/pkg/controllermanager/controller/managedseedset/add_test.go index be4cba2aa3b..6668ea34190 100644 --- a/pkg/controllermanager/controller/managedseedset/add_test.go +++ b/pkg/controllermanager/controller/managedseedset/add_test.go @@ -470,7 +470,7 @@ var _ = Describe("Add", func() { It("should return true for update and false for create and delete event when Seed Ready status changes", func() { newSeed.Status.Conditions = []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedBackupBucketsReady, Status: gardencorev1beta1.ConditionTrue}, } @@ -527,12 +527,12 @@ var _ = Describe("Add", func() { It("should return true for create, update and delete event when pending replica has SeedNotReady status and Seed is ready", func() { oldSeed.Status.Conditions = []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedBackupBucketsReady, Status: gardencorev1beta1.ConditionTrue}, } newSeed.Status.Conditions = []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedBackupBucketsReady, Status: gardencorev1beta1.ConditionTrue}, } diff --git a/pkg/controllermanager/controller/managedseedset/replica.go b/pkg/controllermanager/controller/managedseedset/replica.go index 9942762b5ba..5166190e4dc 100644 --- a/pkg/controllermanager/controller/managedseedset/replica.go +++ b/pkg/controllermanager/controller/managedseedset/replica.go @@ -314,7 +314,7 @@ func managedSeedRegistered(managedSeed *seedmanagementv1alpha1.ManagedSeed) bool } func seedReady(seed *gardencorev1beta1.Seed) bool { - conditionGardenletReady := v1beta1helper.GetCondition(seed.Status.Conditions, gardencorev1beta1.SeedGardenletReady) + conditionGardenletReady := v1beta1helper.GetCondition(seed.Status.Conditions, gardencorev1beta1.GardenletReady) conditionBackupBucketsReady := v1beta1helper.GetCondition(seed.Status.Conditions, gardencorev1beta1.SeedBackupBucketsReady) conditionSystemComponentsHealthy := v1beta1helper.GetCondition(seed.Status.Conditions, gardencorev1beta1.SeedSystemComponentsHealthy) return seed.Generation == seed.Status.ObservedGeneration && seed.DeletionTimestamp == nil && diff --git a/pkg/controllermanager/controller/managedseedset/replica_test.go b/pkg/controllermanager/controller/managedseedset/replica_test.go index 70e92d54369..e731154dbd9 100644 --- a/pkg/controllermanager/controller/managedseedset/replica_test.go +++ b/pkg/controllermanager/controller/managedseedset/replica_test.go @@ -160,7 +160,7 @@ var _ = Describe("Replica", func() { var conditions []gardencorev1beta1.Condition if gardenletReady { conditions = append(conditions, gardencorev1beta1.Condition{ - Type: gardencorev1beta1.SeedGardenletReady, + Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue, }) } diff --git a/pkg/controllermanager/controller/seed/add.go b/pkg/controllermanager/controller/seed/add.go index 003f64a64d0..3a8d99ff200 100644 --- a/pkg/controllermanager/controller/seed/add.go +++ b/pkg/controllermanager/controller/seed/add.go @@ -13,7 +13,6 @@ import ( controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/controllermanager/controller/seed/backupbucketscheck" "github.com/gardener/gardener/pkg/controllermanager/controller/seed/extensionscheck" - "github.com/gardener/gardener/pkg/controllermanager/controller/seed/lifecycle" "github.com/gardener/gardener/pkg/controllermanager/controller/seed/reference" "github.com/gardener/gardener/pkg/controllermanager/controller/seed/secrets" ) @@ -32,12 +31,6 @@ func AddToManager(mgr manager.Manager, cfg controllermanagerconfigv1alpha1.Contr return fmt.Errorf("failed adding extensions check reconciler: %w", err) } - if err := (&lifecycle.Reconciler{ - Config: *cfg.Controllers.Seed, - }).AddToManager(mgr); err != nil { - return fmt.Errorf("failed adding lifecycle reconciler: %w", err) - } - if err := (&secrets.Reconciler{}).AddToManager(mgr); err != nil { return fmt.Errorf("failed adding secrets reconciler: %w", err) } diff --git a/pkg/controllermanager/controller/seed/lifecycle/add.go b/pkg/controllermanager/controller/seed/lifecycle/add.go deleted file mode 100644 index 6e093a9067d..00000000000 --- a/pkg/controllermanager/controller/seed/lifecycle/add.go +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package lifecycle - -import ( - "k8s.io/utils/clock" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/manager" - - gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" -) - -// ControllerName is the name of this controller. -const ControllerName = "seed-lifecycle" - -// AddToManager adds Reconciler to the given manager. -func (r *Reconciler) AddToManager(mgr manager.Manager) error { - if r.Client == nil { - r.Client = mgr.GetClient() - } - if r.Clock == nil { - r.Clock = clock.RealClock{} - } - if r.LeaseNamespace == "" { - r.LeaseNamespace = gardencorev1beta1.GardenerSeedLeaseNamespace - } - - return builder. - ControllerManagedBy(mgr). - Named(ControllerName). - For(&gardencorev1beta1.Seed{}, builder.WithPredicates(predicateutils.ForEventTypes(predicateutils.Create))). - WithOptions(controller.Options{ - MaxConcurrentReconciles: ptr.Deref(r.Config.ConcurrentSyncs, 0), - ReconciliationTimeout: r.Config.SyncPeriod.Duration, - }). - Complete(r) -} diff --git a/pkg/controllermanager/controller/shoot/conditions/add_test.go b/pkg/controllermanager/controller/shoot/conditions/add_test.go index 5a8f094db76..978b24ce7a3 100644 --- a/pkg/controllermanager/controller/shoot/conditions/add_test.go +++ b/pkg/controllermanager/controller/shoot/conditions/add_test.go @@ -63,11 +63,11 @@ var _ = Describe("Add", func() { oldSeed, newSeed *gardencorev1beta1.Seed gardenletReady = []gardencorev1beta1.Condition{{ - Type: gardencorev1beta1.SeedGardenletReady, + Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue, }} gardenletNotReady = []gardencorev1beta1.Condition{{ - Type: gardencorev1beta1.SeedGardenletReady, + Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionFalse, }} ) diff --git a/pkg/gardenlet/controller/add.go b/pkg/gardenlet/controller/add.go index be3da54bb35..d26ce197f65 100644 --- a/pkg/gardenlet/controller/add.go +++ b/pkg/gardenlet/controller/add.go @@ -30,6 +30,7 @@ import ( "github.com/gardener/gardener/pkg/gardenlet/controller/networkpolicy" "github.com/gardener/gardener/pkg/gardenlet/controller/seed" "github.com/gardener/gardener/pkg/gardenlet/controller/shoot" + "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/lease" "github.com/gardener/gardener/pkg/gardenlet/controller/tokenrequestor/workloadidentity" "github.com/gardener/gardener/pkg/gardenlet/controller/vpaevictionrequirements" "github.com/gardener/gardener/pkg/healthz" @@ -88,6 +89,11 @@ func AddToManager( return fmt.Errorf("failed adding Gardenlet controller: %w", err) } + // TODO(rfranzke): Remove this once all shoot reconcilers are added via `shoot.AddToManager`. + if err := lease.AddToManager(mgr, gardenCluster, seedClientSet.RESTClient(), healthManager, nil); err != nil { + return fmt.Errorf("failed adding shoot-lease reconciler: %w", err) + } + return nil } @@ -136,7 +142,7 @@ func AddToManager( return fmt.Errorf("failed adding Seed controller: %w", err) } - if err := shoot.AddToManager(ctx, mgr, gardenCluster, seedCluster, seedClientSet, shootClientMap, *cfg, identity, gardenClusterIdentity); err != nil { + if err := shoot.AddToManager(ctx, mgr, gardenCluster, seedCluster, seedClientSet, shootClientMap, *cfg, identity, gardenClusterIdentity, healthManager); err != nil { return fmt.Errorf("failed adding Shoot controller: %w", err) } diff --git a/pkg/gardenlet/controller/lease/add.go b/pkg/gardenlet/controller/lease/add.go new file mode 100644 index 00000000000..cebb1c05fc4 --- /dev/null +++ b/pkg/gardenlet/controller/lease/add.go @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package lease + +import ( + "time" + + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/clock" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" +) + +// AddToManager adds Reconciler to the given manager. +func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster cluster.Cluster, controllerNamePrefix string, predicates ...predicate.Predicate) error { + if r.GardenClient == nil { + r.GardenClient = gardenCluster.GetClient() + } + if r.Clock == nil { + r.Clock = clock.RealClock{} + } + + return builder. + ControllerManagedBy(mgr). + Named(controllerNamePrefix + "-lease"). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 1, + RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](time.Millisecond, 2*time.Second), + ReconciliationTimeout: time.Duration(r.LeaseResyncSeconds) * time.Second, + }). + WatchesRawSource(source.Kind[client.Object]( + gardenCluster.GetCache(), + r.NewObjectFunc(), + &handler.EnqueueRequestForObject{}, + append([]predicate.Predicate{predicateutils.ForEventTypes(predicateutils.Create)}, predicates...)..., + )). + Complete(r) +} diff --git a/pkg/gardenlet/controller/seed/lease/lease_suite_test.go b/pkg/gardenlet/controller/lease/lease_suite_test.go similarity index 100% rename from pkg/gardenlet/controller/seed/lease/lease_suite_test.go rename to pkg/gardenlet/controller/lease/lease_suite_test.go diff --git a/pkg/gardenlet/controller/lease/reconciler.go b/pkg/gardenlet/controller/lease/reconciler.go new file mode 100644 index 00000000000..d1d559b790d --- /dev/null +++ b/pkg/gardenlet/controller/lease/reconciler.go @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package lease + +import ( + "context" + "fmt" + "net/http" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "k8s.io/utils/clock" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/healthz" +) + +// Reconciler reconciles resources and updates a corresponding heartbeat Lease object in the garden cluster when the +// connection to the runtime cluster succeeds. +type Reconciler struct { + GardenClient client.Client + RuntimeRESTClient rest.Interface + + NewObjectFunc func() client.Object + GetObjectConditions func(client.Object) []gardencorev1beta1.Condition + SetObjectConditions func(client.Object, []gardencorev1beta1.Condition) + + LeaseResyncSeconds int32 + LeaseNamePrefix string + LeaseNamespace *string + Clock clock.Clock + HealthManager healthz.Manager +} + +// Reconcile reconciles resources and updates a corresponding heartbeat Lease object in the garden cluster when the +// connection to the runtime cluster succeeds. +func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log := logf.FromContext(ctx) + + obj := r.NewObjectFunc() + if err := r.GardenClient.Get(ctx, request.NamespacedName, obj); err != nil { + if apierrors.IsNotFound(err) { + log.V(1).Info("Object is gone, stop reconciling") + return reconcile.Result{}, nil + } + return reconcile.Result{}, fmt.Errorf("error retrieving object from store: %w", err) + } + + if err := CheckConnection(ctx, r.RuntimeRESTClient); err != nil { + r.HealthManager.Set(false) + return reconcile.Result{}, fmt.Errorf("cannot establish connection with runtime cluster: %w", err) + } + + if err := r.renewLease(ctx, obj); err != nil { + r.HealthManager.Set(false) + return reconcile.Result{}, err + } + + r.HealthManager.Set(true) + return reconcile.Result{RequeueAfter: time.Duration(r.LeaseResyncSeconds) * time.Second}, r.maintainGardenletReadyCondition(ctx, obj) +} + +// CheckConnection is a function which checks the connection to the runtime cluster. +// Exposed for testing. +var CheckConnection = func(ctx context.Context, client rest.Interface) error { + result := client.Get().AbsPath("/healthz").Do(ctx) + if result.Error() != nil { + return fmt.Errorf("failed to execute call to Kubernetes API Server: %v", result.Error()) + } + + var statusCode int + result.StatusCode(&statusCode) + if statusCode != http.StatusOK { + return fmt.Errorf("API Server returned unexpected status code: %d", statusCode) + } + + return nil +} + +func (r *Reconciler) renewLease(ctx context.Context, obj client.Object) error { + lease := &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.LeaseNamePrefix + obj.GetName(), + Namespace: ptr.Deref(r.LeaseNamespace, obj.GetNamespace()), + }, + } + + gvk, err := apiutil.GVKForObject(obj, r.GardenClient.Scheme()) + if err != nil { + return fmt.Errorf("failed determining group/version/kind for object: %w", err) + } + + _, err = controllerutils.CreateOrGetAndMergePatch(ctx, r.GardenClient, lease, func() error { + lease.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: gvk.GroupVersion().String(), + Kind: gvk.Kind, + Name: obj.GetName(), + UID: obj.GetUID(), + }} + lease.Spec.HolderIdentity = ptr.To(obj.GetName()) + lease.Spec.LeaseDurationSeconds = &r.LeaseResyncSeconds + lease.Spec.RenewTime = &metav1.MicroTime{Time: r.Clock.Now()} + return nil + }) + return err +} + +func (r *Reconciler) maintainGardenletReadyCondition(ctx context.Context, obj client.Object) error { + bldr, err := helper.NewConditionBuilder(gardencorev1beta1.GardenletReady) + if err != nil { + return err + } + + if oldCondition := helper.GetCondition(r.GetObjectConditions(obj), gardencorev1beta1.GardenletReady); oldCondition != nil { + bldr.WithOldCondition(*oldCondition) + } + bldr.WithStatus(gardencorev1beta1.ConditionTrue) + bldr.WithReason("GardenletReady") + bldr.WithMessage("Gardenlet is posting ready status.") + bldr.WithClock(r.Clock) + + newCondition, needsUpdate := bldr.Build() + if !needsUpdate { + return nil + } + + patch := client.StrategicMergeFrom(obj.DeepCopyObject().(client.Object)) + r.SetObjectConditions(obj, helper.MergeConditions(r.GetObjectConditions(obj), newCondition)) + return r.GardenClient.Status().Patch(ctx, obj, patch) +} diff --git a/pkg/gardenlet/controller/seed/lease/reconciler_test.go b/pkg/gardenlet/controller/lease/reconciler_test.go similarity index 88% rename from pkg/gardenlet/controller/seed/lease/reconciler_test.go rename to pkg/gardenlet/controller/lease/reconciler_test.go index 82579e97d2e..0716a0fcda3 100644 --- a/pkg/gardenlet/controller/seed/lease/reconciler_test.go +++ b/pkg/gardenlet/controller/lease/reconciler_test.go @@ -29,28 +29,26 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" "github.com/gardener/gardener/pkg/client/kubernetes" - gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" - . "github.com/gardener/gardener/pkg/gardenlet/controller/seed/lease" + . "github.com/gardener/gardener/pkg/gardenlet/controller/lease" "github.com/gardener/gardener/pkg/healthz" . "github.com/gardener/gardener/pkg/utils/test/matchers" ) var _ = Describe("LeaseReconciler", func() { var ( - ctx context.Context - clock clock.Clock - gardenClient client.Client - seedRESTClient *fakerestclient.RESTClient - healthManager healthz.Manager + ctx context.Context + clock clock.Clock + gardenClient client.Client + runtimeRESTClient *fakerestclient.RESTClient + healthManager healthz.Manager seed *gardencorev1beta1.Seed expectedCondition *gardencorev1beta1.Condition expectedLease *coordinationv1.Lease namespace = "gardener-system-seed-lease" - request reconcile.Request - reconciler *Reconciler - controllerConfig gardenletconfigv1alpha1.SeedControllerConfiguration + request reconcile.Request + reconciler *Reconciler ) BeforeEach(func() { @@ -85,18 +83,13 @@ var _ = Describe("LeaseReconciler", func() { } gardenClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).WithObjects(seed).WithStatusSubresource(&gardencorev1beta1.Seed{}).Build() - seedRESTClient = &fakerestclient.RESTClient{ + runtimeRESTClient = &fakerestclient.RESTClient{ NegotiatedSerializer: serializer.NewCodecFactory(kubernetes.GardenScheme).WithoutConversion(), Resp: &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("")), }, } - - controllerConfig = gardenletconfigv1alpha1.SeedControllerConfiguration{ - LeaseResyncSeconds: ptr.To[int32](2), - LeaseResyncMissThreshold: ptr.To[int32](10), - } }) JustBeforeEach(func() { @@ -104,12 +97,19 @@ var _ = Describe("LeaseReconciler", func() { Expect(healthManager.Start(ctx)).To(Succeed()) reconciler = &Reconciler{ - GardenClient: gardenClient, - SeedRESTClient: seedRESTClient, - Config: controllerConfig, - Clock: clock, - HealthManager: healthManager, - LeaseNamespace: namespace, + GardenClient: gardenClient, + RuntimeRESTClient: runtimeRESTClient, + NewObjectFunc: func() client.Object { return &gardencorev1beta1.Seed{} }, + GetObjectConditions: func(obj client.Object) []gardencorev1beta1.Condition { + return obj.(*gardencorev1beta1.Seed).Status.Conditions + }, + SetObjectConditions: func(obj client.Object, conditions []gardencorev1beta1.Condition) { + obj.(*gardencorev1beta1.Seed).Status.Conditions = conditions + }, + LeaseResyncSeconds: 2, + LeaseNamespace: &namespace, + Clock: clock, + HealthManager: healthManager, } }) @@ -153,7 +153,7 @@ var _ = Describe("LeaseReconciler", func() { expectedCondition = gardenletReadyCondition(clock) expectedLease.Spec.LeaseDurationSeconds = ptr.To[int32](3) - reconciler.Config.LeaseResyncSeconds = ptr.To[int32](3) + reconciler.LeaseResyncSeconds = 3 request = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(seed)} Expect(reconciler.Reconcile(ctx, request)).To(Equal(reconcile.Result{RequeueAfter: 3 * time.Second})) @@ -161,7 +161,7 @@ var _ = Describe("LeaseReconciler", func() { }) It("should fail if connection to Seed fails", func() { - seedRESTClient.Resp.StatusCode = http.StatusInternalServerError + runtimeRESTClient.Resp.StatusCode = http.StatusInternalServerError expectedLease = nil expectedCondition = nil diff --git a/pkg/gardenlet/controller/seed/add.go b/pkg/gardenlet/controller/seed/add.go index 8e872e04ae6..63c29ef2000 100644 --- a/pkg/gardenlet/controller/seed/add.go +++ b/pkg/gardenlet/controller/seed/add.go @@ -8,6 +8,7 @@ import ( "fmt" "os" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -50,12 +51,7 @@ func AddToManager( return fmt.Errorf("failed adding care reconciler: %w", err) } - if err := (&lease.Reconciler{ - SeedRESTClient: seedClientSet.RESTClient(), - Config: *cfg.Controllers.Seed, - HealthManager: healthManager, - SeedName: cfg.SeedConfig.Name, - }).AddToManager(mgr, gardenCluster); err != nil { + if err := lease.AddToManager(mgr, gardenCluster, seedClientSet.RESTClient(), *cfg.Controllers.Seed, healthManager, cfg.SeedConfig.Name, nil, ptr.To(gardencorev1beta1.GardenerSeedLeaseNamespace)); err != nil { return fmt.Errorf("failed adding lease reconciler: %w", err) } diff --git a/pkg/gardenlet/controller/seed/lease/add.go b/pkg/gardenlet/controller/seed/lease/add.go index f29ebd7682c..928edc19c06 100644 --- a/pkg/gardenlet/controller/seed/lease/add.go +++ b/pkg/gardenlet/controller/seed/lease/add.go @@ -5,52 +5,44 @@ package lease import ( - "time" - - "k8s.io/client-go/util/workqueue" + "k8s.io/client-go/rest" "k8s.io/utils/clock" - "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" + gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" + "github.com/gardener/gardener/pkg/gardenlet/controller/lease" + "github.com/gardener/gardener/pkg/healthz" ) -// ControllerName is the name of this controller. -const ControllerName = "seed-lease" +// AddToManager adds the seed-lease controller to the given manager. +func AddToManager( + mgr manager.Manager, + gardenCluster cluster.Cluster, + seedRESTClient rest.Interface, + config gardenletconfigv1alpha1.SeedControllerConfiguration, + healthManager healthz.Manager, + seedName string, + clock clock.Clock, + leaseNamespace *string, +) error { + return (&lease.Reconciler{ + RuntimeRESTClient: seedRESTClient, + HealthManager: healthManager, + Clock: clock, -// AddToManager adds Reconciler to the given manager. -func (r *Reconciler) AddToManager(mgr manager.Manager, gardenCluster cluster.Cluster) error { - if r.GardenClient == nil { - r.GardenClient = gardenCluster.GetClient() - } - if r.Clock == nil { - r.Clock = clock.RealClock{} - } - if r.LeaseNamespace == "" { - r.LeaseNamespace = gardencorev1beta1.GardenerSeedLeaseNamespace - } + NewObjectFunc: func() client.Object { return &gardencorev1beta1.Seed{} }, + GetObjectConditions: func(obj client.Object) []gardencorev1beta1.Condition { + return obj.(*gardencorev1beta1.Seed).Status.Conditions + }, + SetObjectConditions: func(obj client.Object, conditions []gardencorev1beta1.Condition) { + obj.(*gardencorev1beta1.Seed).Status.Conditions = conditions + }, - return builder. - ControllerManagedBy(mgr). - Named(ControllerName). - WithOptions(controller.Options{ - MaxConcurrentReconciles: 1, - RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](time.Millisecond, 2*time.Second), - ReconciliationTimeout: time.Duration(*r.Config.LeaseResyncSeconds) * time.Second, - }). - WatchesRawSource( - source.Kind[client.Object](gardenCluster.GetCache(), - &gardencorev1beta1.Seed{}, - &handler.EnqueueRequestForObject{}, - predicateutils.HasName(r.SeedName), - predicateutils.ForEventTypes(predicateutils.Create)), - ). - Complete(r) + LeaseNamespace: leaseNamespace, + LeaseResyncSeconds: *config.LeaseResyncSeconds, + }).AddToManager(mgr, gardenCluster, "seed", predicateutils.HasName(seedName)) } diff --git a/pkg/gardenlet/controller/seed/lease/reconciler.go b/pkg/gardenlet/controller/seed/lease/reconciler.go deleted file mode 100644 index 6ed700c6063..00000000000 --- a/pkg/gardenlet/controller/seed/lease/reconciler.go +++ /dev/null @@ -1,132 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package lease - -import ( - "context" - "fmt" - "net/http" - "time" - - coordinationv1 "k8s.io/api/coordination/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/rest" - "k8s.io/utils/clock" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" - "github.com/gardener/gardener/pkg/controllerutils" - gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/healthz" -) - -// Reconciler reconciles Seed resources and updates the heartbeat Lease object in the garden cluster when the connection -// to the seed cluster succeeds. -type Reconciler struct { - GardenClient client.Client - SeedRESTClient rest.Interface - Config gardenletconfigv1alpha1.SeedControllerConfiguration - Clock clock.Clock - HealthManager healthz.Manager - LeaseNamespace string - SeedName string -} - -// Reconcile reconciles Seed resources and updates the heartbeat Lease object in the garden cluster when the connection -// to the seed cluster succeeds. -func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - log := logf.FromContext(ctx) - - seed := &gardencorev1beta1.Seed{} - if err := r.GardenClient.Get(ctx, request.NamespacedName, seed); err != nil { - if apierrors.IsNotFound(err) { - log.V(1).Info("Object is gone, stop reconciling") - return reconcile.Result{}, nil - } - return reconcile.Result{}, fmt.Errorf("error retrieving object from store: %w", err) - } - - if err := CheckSeedConnection(ctx, r.SeedRESTClient); err != nil { - r.HealthManager.Set(false) - return reconcile.Result{}, fmt.Errorf("cannot establish connection with Seed: %w", err) - } - - if err := r.renewLeaseForSeed(ctx, seed); err != nil { - r.HealthManager.Set(false) - return reconcile.Result{}, err - } - - r.HealthManager.Set(true) - return reconcile.Result{RequeueAfter: time.Duration(*r.Config.LeaseResyncSeconds) * time.Second}, r.maintainGardenletReadyCondition(ctx, seed) -} - -// CheckSeedConnection is a function which checks the connection to the seed. -// Exposed for testing. -var CheckSeedConnection = func(ctx context.Context, client rest.Interface) error { - result := client.Get().AbsPath("/healthz").Do(ctx) - if result.Error() != nil { - return fmt.Errorf("failed to execute call to Kubernetes API Server: %v", result.Error()) - } - - var statusCode int - result.StatusCode(&statusCode) - if statusCode != http.StatusOK { - return fmt.Errorf("API Server returned unexpected status code: %d", statusCode) - } - - return nil -} - -func (r *Reconciler) renewLeaseForSeed(ctx context.Context, seed *gardencorev1beta1.Seed) error { - lease := &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: seed.Name, - Namespace: r.LeaseNamespace, - }, - } - - _, err := controllerutils.CreateOrGetAndMergePatch(ctx, r.GardenClient, lease, func() error { - lease.OwnerReferences = []metav1.OwnerReference{{ - APIVersion: gardencorev1beta1.SchemeGroupVersion.String(), - Kind: "Seed", - Name: seed.GetName(), - UID: seed.GetUID(), - }} - lease.Spec.HolderIdentity = ptr.To(seed.Name) - lease.Spec.LeaseDurationSeconds = r.Config.LeaseResyncSeconds - lease.Spec.RenewTime = &metav1.MicroTime{Time: r.Clock.Now()} - return nil - }) - return err -} - -func (r *Reconciler) maintainGardenletReadyCondition(ctx context.Context, seed *gardencorev1beta1.Seed) error { - bldr, err := helper.NewConditionBuilder(gardencorev1beta1.SeedGardenletReady) - if err != nil { - return err - } - - if oldCondition := helper.GetCondition(seed.Status.Conditions, gardencorev1beta1.SeedGardenletReady); oldCondition != nil { - bldr.WithOldCondition(*oldCondition) - } - bldr.WithStatus(gardencorev1beta1.ConditionTrue) - bldr.WithReason("GardenletReady") - bldr.WithMessage("Gardenlet is posting ready status.") - bldr.WithClock(r.Clock) - - newCondition, needsUpdate := bldr.Build() - if !needsUpdate { - return nil - } - - patch := client.StrategicMergeFrom(seed.DeepCopy()) - seed.Status.Conditions = helper.MergeConditions(seed.Status.Conditions, newCondition) - return r.GardenClient.Status().Patch(ctx, seed, patch) -} diff --git a/pkg/gardenlet/controller/seed/seed/reconciler.go b/pkg/gardenlet/controller/seed/seed/reconciler.go index ccb94189b35..caca10fe39b 100644 --- a/pkg/gardenlet/controller/seed/seed/reconciler.go +++ b/pkg/gardenlet/controller/seed/seed/reconciler.go @@ -221,7 +221,7 @@ func (r *Reconciler) updateStatusOperationSuccess(ctx context.Context, seed *gar switch cond.Type { case gardencorev1beta1.SeedBackupBucketsReady, gardencorev1beta1.SeedExtensionsReady, - gardencorev1beta1.SeedGardenletReady, + gardencorev1beta1.GardenletReady, gardencorev1beta1.SeedSystemComponentsHealthy: if cond.Status != gardencorev1beta1.ConditionFalse { seed.Status.Conditions[i].Status = gardencorev1beta1.ConditionProgressing diff --git a/pkg/gardenlet/controller/shoot/add.go b/pkg/gardenlet/controller/shoot/add.go index ab0b22b9708..5d4e9ea6079 100644 --- a/pkg/gardenlet/controller/shoot/add.go +++ b/pkg/gardenlet/controller/shoot/add.go @@ -21,9 +21,12 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes/clientmap" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/care" + "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/lease" "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/shoot" "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/state" "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/status" + "github.com/gardener/gardener/pkg/healthz" + "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" ) // AddToManager adds all Shoot controllers to the given manager. @@ -37,6 +40,7 @@ func AddToManager( cfg gardenletconfigv1alpha1.GardenletConfiguration, identity *gardencorev1beta1.Gardener, gardenClusterIdentity string, + healthManager healthz.Manager, ) error { var responsibleForUnmanagedSeed bool if err := gardenCluster.GetAPIReader().Get(ctx, client.ObjectKey{Name: cfg.SeedConfig.Name, Namespace: v1beta1constants.GardenNamespace}, &seedmanagementv1alpha1.ManagedSeed{}); err != nil { @@ -90,5 +94,11 @@ func AddToManager( } } + if gardenlet.IsResponsibleForSelfHostedShoot() { + if err := lease.AddToManager(mgr, gardenCluster, seedClientSet.RESTClient(), healthManager, nil); err != nil { + return fmt.Errorf("failed adding lease reconciler: %w", err) + } + } + return nil } diff --git a/pkg/gardenlet/controller/shoot/lease/add.go b/pkg/gardenlet/controller/shoot/lease/add.go new file mode 100644 index 00000000000..253a7236a71 --- /dev/null +++ b/pkg/gardenlet/controller/shoot/lease/add.go @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package lease + +import ( + "k8s.io/client-go/rest" + "k8s.io/utils/clock" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/gardenlet/controller/lease" + "github.com/gardener/gardener/pkg/healthz" + "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" +) + +// AddToManager adds the shoot-lease controller to the given manager. +func AddToManager(mgr manager.Manager, gardenCluster cluster.Cluster, shootRESTClient rest.Interface, healthManager healthz.Manager, clock clock.Clock) error { + return (&lease.Reconciler{ + RuntimeRESTClient: shootRESTClient, + HealthManager: healthManager, + Clock: clock, + + NewObjectFunc: func() client.Object { return &gardencorev1beta1.Shoot{} }, + GetObjectConditions: func(obj client.Object) []gardencorev1beta1.Condition { + return obj.(*gardencorev1beta1.Shoot).Status.Conditions + }, + SetObjectConditions: func(obj client.Object, conditions []gardencorev1beta1.Condition) { + obj.(*gardencorev1beta1.Shoot).Status.Conditions = conditions + }, + + LeaseNamePrefix: gardenlet.ResourcePrefixSelfHostedShoot, + LeaseResyncSeconds: 2, + }).AddToManager(mgr, gardenCluster, "shoot") +} diff --git a/pkg/scheduler/controller/shoot/reconciler.go b/pkg/scheduler/controller/shoot/reconciler.go index 38470648b86..a0dcba7fd1b 100644 --- a/pkg/scheduler/controller/shoot/reconciler.go +++ b/pkg/scheduler/controller/shoot/reconciler.go @@ -634,7 +634,7 @@ func verifySeedReadiness(seed *gardencorev1beta1.Seed) bool { return false } - if cond := v1beta1helper.GetCondition(seed.Status.Conditions, gardencorev1beta1.SeedGardenletReady); cond == nil || cond.Status != gardencorev1beta1.ConditionTrue { + if cond := v1beta1helper.GetCondition(seed.Status.Conditions, gardencorev1beta1.GardenletReady); cond == nil || cond.Status != gardencorev1beta1.ConditionTrue { return false } diff --git a/pkg/scheduler/controller/shoot/reconciler_test.go b/pkg/scheduler/controller/shoot/reconciler_test.go index fbec6dea541..e3edfaff2de 100644 --- a/pkg/scheduler/controller/shoot/reconciler_test.go +++ b/pkg/scheduler/controller/shoot/reconciler_test.go @@ -81,7 +81,7 @@ var _ = Describe("Scheduler_Control", func() { Status: gardencorev1beta1.SeedStatus{ Conditions: []gardencorev1beta1.Condition{ { - Type: gardencorev1beta1.SeedGardenletReady, + Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue, }, }, @@ -871,7 +871,7 @@ var _ = Describe("Scheduler_Control", func() { It("should fail because it cannot find a seed cluster due to gardenlet not ready", func() { seed.Status.Conditions = []gardencorev1beta1.Condition{ { - Type: gardencorev1beta1.SeedGardenletReady, + Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionFalse, }, } @@ -1139,7 +1139,7 @@ var _ = DescribeTable("condition is false", }, Status: gardencorev1beta1.SeedStatus{ Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedBackupBucketsReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedExtensionsReady, Status: gardencorev1beta1.ConditionTrue}, }, @@ -1161,8 +1161,8 @@ var _ = DescribeTable("condition is false", Expect(verifySeedReadiness(seed)).To(expected) }, - Entry("SeedGardenletReady is missing", gardencorev1beta1.SeedGardenletReady, true, true, BeFalse()), - Entry("SeedGardenletReady is false", gardencorev1beta1.SeedGardenletReady, false, true, BeFalse()), + Entry("GardenletReady is missing", gardencorev1beta1.GardenletReady, true, true, BeFalse()), + Entry("GardenletReady is false", gardencorev1beta1.GardenletReady, false, true, BeFalse()), Entry("SeedBackupBucketsReady is missing", gardencorev1beta1.SeedBackupBucketsReady, true, true, BeFalse()), Entry("SeedBackupBucketsReady is missing but no backup specified", gardencorev1beta1.SeedBackupBucketsReady, true, false, BeTrue()), Entry("SeedBackupBucketsReady is false", gardencorev1beta1.SeedBackupBucketsReady, false, true, BeFalse()), diff --git a/pkg/utils/graph/eventhandler_seed.go b/pkg/utils/graph/eventhandler_seed.go index 60148f0a02c..9c4a95e5b9f 100644 --- a/pkg/utils/graph/eventhandler_seed.go +++ b/pkg/utils/graph/eventhandler_seed.go @@ -52,7 +52,7 @@ func (g *graph) setupSeedWatch(ctx context.Context, informer cache.Informer) err g.handleSeedCreateOrUpdate(newSeed) } - newGardenletReadyCondition := v1beta1helper.GetCondition(newSeed.Status.Conditions, gardencorev1beta1.SeedGardenletReady) + newGardenletReadyCondition := v1beta1helper.GetCondition(newSeed.Status.Conditions, gardencorev1beta1.GardenletReady) // When the GardenletReady condition transitions to 'Unknown' then the client certificate might be expired. // Hence, check if seed belongs to a ManagedSeed and reconcile it to potentially allow re-bootstrapping it. diff --git a/pkg/utils/graph/eventhandler_shoot.go b/pkg/utils/graph/eventhandler_shoot.go index a102356bb84..67615cb1749 100644 --- a/pkg/utils/graph/eventhandler_shoot.go +++ b/pkg/utils/graph/eventhandler_shoot.go @@ -19,6 +19,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" ) func (g *graph) setupShootWatch(ctx context.Context, informer cache.Informer) error { @@ -106,7 +107,9 @@ func (g *graph) HandleShootCreateOrUpdate(ctx context.Context, shoot *gardencore g.deleteAllIncomingEdges(VertexTypeSecretBinding, VertexTypeShoot, shoot.Namespace, shoot.Name) g.deleteAllIncomingEdges(VertexTypeCredentialsBinding, VertexTypeShoot, shoot.Namespace, shoot.Name) g.deleteAllIncomingEdges(VertexTypeShootState, VertexTypeShoot, shoot.Namespace, shoot.Name) - if !g.forSelfHostedShoots { + if g.forSelfHostedShoots { + g.deleteAllIncomingEdges(VertexTypeLease, VertexTypeShoot, shoot.Namespace, shoot.Name) + } else { g.deleteAllOutgoingEdges(VertexTypeShoot, shoot.Namespace, shoot.Name, VertexTypeSeed) } @@ -237,6 +240,11 @@ func (g *graph) HandleShootCreateOrUpdate(ctx context.Context, shoot *gardencore } } } + + if g.forSelfHostedShoots { + leaseVertex := g.getOrCreateVertex(VertexTypeLease, shoot.Namespace, gardenlet.ResourcePrefixSelfHostedShoot+shoot.Name) + g.addEdge(leaseVertex, shootVertex) + } } func (g *graph) handleShootDelete(shoot *gardencorev1beta1.Shoot) { diff --git a/pkg/utils/graph/graph_test.go b/pkg/utils/graph/graph_test.go index 3a928c11416..a9a2bbee202 100644 --- a/pkg/utils/graph/graph_test.go +++ b/pkg/utils/graph/graph_test.go @@ -2688,8 +2688,8 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d It("should behave as expected for gardencorev1beta1.Shoot", func() { By("Add") fakeInformerShoot.Add(shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(22)) - Expect(graph.graph.Edges().Len()).To(Equal(21)) + Expect(graph.graph.Nodes().Len()).To(Equal(23)) + Expect(graph.graph.Edges().Len()).To(Equal(22)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", *shoot1.Spec.CloudProfileName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeNamespacedCloudProfile, shoot1.Namespace, shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) @@ -2712,6 +2712,7 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Add (with namespaced cloud profile)") shoot1Copy := shoot1.DeepCopy() @@ -2721,8 +2722,8 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Name: "namespaced-profile-1", } fakeInformerShoot.Add(shoot1Copy) - Expect(graph.graph.Nodes().Len()).To(Equal(22)) - Expect(graph.graph.Edges().Len()).To(Equal(21)) + Expect(graph.graph.Nodes().Len()).To(Equal(23)) + Expect(graph.graph.Edges().Len()).To(Equal(22)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", *shoot1.Spec.CloudProfileName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) Expect(graph.HasPathFrom(VertexTypeNamespacedCloudProfile, shoot1.Namespace, shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) @@ -2746,13 +2747,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Add (secret binding is nil)") shoot1Copy = shoot1.DeepCopy() shoot1Copy.Spec.SecretBindingName = nil fakeInformerShoot.Add(shoot1Copy) - Expect(graph.graph.Nodes().Len()).To(Equal(21)) - Expect(graph.graph.Edges().Len()).To(Equal(20)) + Expect(graph.graph.Nodes().Len()).To(Equal(22)) + Expect(graph.graph.Edges().Len()).To(Equal(21)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", *shoot1.Spec.CloudProfileName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCredentialsBinding, shoot1.Namespace, *shoot1.Spec.CredentialsBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -2773,13 +2775,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Add (credentials binding is nil)") shoot1Copy = shoot1.DeepCopy() shoot1Copy.Spec.CredentialsBindingName = nil fakeInformerShoot.Add(shoot1Copy) - Expect(graph.graph.Nodes().Len()).To(Equal(21)) - Expect(graph.graph.Edges().Len()).To(Equal(20)) + Expect(graph.graph.Nodes().Len()).To(Equal(22)) + Expect(graph.graph.Edges().Len()).To(Equal(21)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", *shoot1.Spec.CloudProfileName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -2800,13 +2803,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (cloud profile name)") shoot1Copy = shoot1.DeepCopy() shoot1.Spec.CloudProfile = &gardencorev1beta1.CloudProfileReference{Name: "foo", Kind: "CloudProfile"} fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(22)) - Expect(graph.graph.Edges().Len()).To(Equal(21)) + Expect(graph.graph.Nodes().Len()).To(Equal(23)) + Expect(graph.graph.Edges().Len()).To(Equal(22)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", *shoot1Copy.Spec.CloudProfileName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -2830,13 +2834,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (namespaced cloud profile)") shoot1Copy = shoot1.DeepCopy() shoot1Copy.Spec.CloudProfile = &gardencorev1beta1.CloudProfileReference{Name: "namespaced-profile", Kind: "NamespacedCloudProfile"} fakeInformerShoot.Update(shoot1, shoot1Copy) - Expect(graph.graph.Nodes().Len()).To(Equal(22)) - Expect(graph.graph.Edges().Len()).To(Equal(21)) + Expect(graph.graph.Nodes().Len()).To(Equal(23)) + Expect(graph.graph.Edges().Len()).To(Equal(22)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", *shoot1Copy.Spec.CloudProfileName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1Copy.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) @@ -2861,13 +2866,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (secret binding name)") shoot1Copy = shoot1.DeepCopy() shoot1.Spec.SecretBindingName = ptr.To("bar") fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(22)) - Expect(graph.graph.Edges().Len()).To(Equal(21)) + Expect(graph.graph.Nodes().Len()).To(Equal(23)) + Expect(graph.graph.Edges().Len()).To(Equal(22)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1Copy.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) @@ -2890,13 +2896,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (credentials binding name)") shoot1Copy = shoot1.DeepCopy() shoot1.Spec.CredentialsBindingName = ptr.To("bar") fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(22)) - Expect(graph.graph.Edges().Len()).To(Equal(21)) + Expect(graph.graph.Nodes().Len()).To(Equal(23)) + Expect(graph.graph.Edges().Len()).To(Equal(22)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -2919,13 +2926,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (audit policy config map name)") shoot1Copy = shoot1.DeepCopy() shoot1.Spec.Kubernetes.KubeAPIServer.AuditConfig = nil fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(21)) - Expect(graph.graph.Edges().Len()).To(Equal(20)) + Expect(graph.graph.Nodes().Len()).To(Equal(22)) + Expect(graph.graph.Edges().Len()).To(Equal(21)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -2947,13 +2955,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (structured authentication config map name)") shoot1Copy = shoot1.DeepCopy() shoot1.Spec.Kubernetes.KubeAPIServer.StructuredAuthentication = nil fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(20)) - Expect(graph.graph.Edges().Len()).To(Equal(19)) + Expect(graph.graph.Nodes().Len()).To(Equal(21)) + Expect(graph.graph.Edges().Len()).To(Equal(20)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -2975,13 +2984,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (structured authorization kubeconfig secrets)") shoot1Copy = shoot1.DeepCopy() shoot1.Spec.Kubernetes.KubeAPIServer.StructuredAuthorization.Kubeconfigs = nil fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(19)) - Expect(graph.graph.Edges().Len()).To(Equal(18)) + Expect(graph.graph.Nodes().Len()).To(Equal(20)) + Expect(graph.graph.Edges().Len()).To(Equal(19)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -3003,13 +3013,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (structured authorization config map name and kubeconfig secrets)") shoot1Copy = shoot1.DeepCopy() shoot1.Spec.Kubernetes.KubeAPIServer.StructuredAuthorization = nil fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(18)) - Expect(graph.graph.Edges().Len()).To(Equal(17)) + Expect(graph.graph.Nodes().Len()).To(Equal(19)) + Expect(graph.graph.Edges().Len()).To(Equal(18)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -3031,13 +3042,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (dns provider secrets)") shoot1Copy = shoot1.DeepCopy() shoot1.Spec.DNS = nil fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(16)) - Expect(graph.graph.Edges().Len()).To(Equal(15)) + Expect(graph.graph.Nodes().Len()).To(Equal(17)) + Expect(graph.graph.Edges().Len()).To(Equal(16)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -3059,13 +3071,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Update (resources)") shoot1Copy = shoot1.DeepCopy() shoot1.Spec.Resources = nil fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(14)) - Expect(graph.graph.Edges().Len()).To(Equal(13)) + Expect(graph.graph.Nodes().Len()).To(Equal(15)) + Expect(graph.graph.Edges().Len()).To(Equal(14)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -3087,13 +3100,14 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Remove managed issuer annotation") shoot1Copy = shoot1.DeepCopy() shoot1.Annotations = map[string]string{} fakeInformerShoot.Update(shoot1Copy, shoot1) - Expect(graph.graph.Nodes().Len()).To(Equal(13)) - Expect(graph.graph.Edges().Len()).To(Equal(12)) + Expect(graph.graph.Nodes().Len()).To(Equal(14)) + Expect(graph.graph.Edges().Len()).To(Equal(13)) Expect(graph.HasPathFrom(VertexTypeNamespace, "", shoot1.Namespace, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeCloudProfile, "", shoot1.Spec.CloudProfile.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeSecretBinding, shoot1.Namespace, *shoot1.Spec.SecretBindingName, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) @@ -3115,6 +3129,7 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeTrue()) By("Delete") fakeInformerShoot.Delete(shoot1) @@ -3142,6 +3157,7 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCACluster, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) Expect(graph.HasPathFrom(VertexTypeConfigMap, shoot1.Namespace, shoot1ConfigMapNameCAKubelet, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) Expect(graph.HasPathFrom(VertexTypeShootState, shoot1.Namespace, shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) + Expect(graph.HasPathFrom(VertexTypeLease, shoot1.Namespace, "self-hosted-shoot-"+shoot1.Name, VertexTypeShoot, shoot1.Namespace, shoot1.Name)).To(BeFalse()) }) }) diff --git a/pkg/utils/kubernetes/health/seed.go b/pkg/utils/kubernetes/health/seed.go index 72565ea20ec..48c3e707479 100644 --- a/pkg/utils/kubernetes/health/seed.go +++ b/pkg/utils/kubernetes/health/seed.go @@ -15,7 +15,7 @@ import ( var ( trueSeedConditionTypes = []gardencorev1beta1.ConditionType{ - gardencorev1beta1.SeedGardenletReady, + gardencorev1beta1.GardenletReady, gardencorev1beta1.SeedSystemComponentsHealthy, } ) diff --git a/pkg/utils/kubernetes/health/seed_test.go b/pkg/utils/kubernetes/health/seed_test.go index 3be908c19e6..54493abca01 100644 --- a/pkg/utils/kubernetes/health/seed_test.go +++ b/pkg/utils/kubernetes/health/seed_test.go @@ -24,7 +24,7 @@ var _ = Describe("Seed", func() { Status: gardencorev1beta1.SeedStatus{ Gardener: &gardencorev1beta1.Gardener{}, Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, }, }, @@ -33,7 +33,7 @@ var _ = Describe("Seed", func() { Status: gardencorev1beta1.SeedStatus{ Gardener: &gardencorev1beta1.Gardener{ID: "thegardener"}, Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, }, }, @@ -41,7 +41,7 @@ var _ = Describe("Seed", func() { Entry("unhealthy available condition (gardenlet ready)", &gardencorev1beta1.Seed{ Status: gardencorev1beta1.SeedStatus{ Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionFalse}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionFalse}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, }, }, @@ -49,7 +49,7 @@ var _ = Describe("Seed", func() { Entry("unhealthy available condition (seed system components healthy)", &gardencorev1beta1.Seed{ Status: gardencorev1beta1.SeedStatus{ Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionFalse}, }, }, @@ -59,7 +59,7 @@ var _ = Describe("Seed", func() { Status: gardencorev1beta1.SeedStatus{ Gardener: &gardencorev1beta1.Gardener{ID: "thegardener"}, Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, }, }, @@ -70,7 +70,7 @@ var _ = Describe("Seed", func() { }, Status: gardencorev1beta1.SeedStatus{ Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, }, }, @@ -93,7 +93,7 @@ var _ = Describe("Seed", func() { Status: gardencorev1beta1.SeedStatus{ Gardener: &gardencorev1beta1.Gardener{Version: "1.12.8"}, Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, }, }, @@ -102,7 +102,7 @@ var _ = Describe("Seed", func() { Status: gardencorev1beta1.SeedStatus{ Gardener: &gardencorev1beta1.Gardener{Version: "1.12.8"}, Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, }, }, @@ -111,7 +111,7 @@ var _ = Describe("Seed", func() { Status: gardencorev1beta1.SeedStatus{ Gardener: &gardencorev1beta1.Gardener{Version: "1.12.8"}, Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionFalse}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionFalse}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, }, }, @@ -120,7 +120,7 @@ var _ = Describe("Seed", func() { Status: gardencorev1beta1.SeedStatus{ Gardener: &gardencorev1beta1.Gardener{Version: "1.12.8"}, Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionFalse}, }, }, @@ -129,7 +129,7 @@ var _ = Describe("Seed", func() { Status: gardencorev1beta1.SeedStatus{ Gardener: &gardencorev1beta1.Gardener{Version: "1.12.8"}, Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionFalse}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionFalse}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionFalse}, }, }, diff --git a/plugin/pkg/global/customverbauthorizer/admission.go b/plugin/pkg/global/customverbauthorizer/admission.go index 13325f63aa2..e488f3b9158 100644 --- a/plugin/pkg/global/customverbauthorizer/admission.go +++ b/plugin/pkg/global/customverbauthorizer/admission.go @@ -256,7 +256,7 @@ func (c *CustomVerbAuthorizer) admitShoots(ctx context.Context, a admission.Attr } } - if mustCheckShootAutonomy(oldObj, obj) { + if mustCheckIfShootIsSelfHosted(oldObj, obj) { return c.authorize(ctx, a, CustomVerbShootMarkSelfHosted, "modify .spec.provider.workers[].controlPlane") } @@ -421,6 +421,6 @@ func mustCheckLimits(oldLimits, limits *core.Limits, parentCloudProfile *v1beta1 ptr.Deref(limits.MaxNodesTotal, 0) > ptr.Deref(parentCloudProfile.Spec.Limits.MaxNodesTotal, 0) } -func mustCheckShootAutonomy(oldShoot, shoot *core.Shoot) bool { +func mustCheckIfShootIsSelfHosted(oldShoot, shoot *core.Shoot) bool { return !apiequality.Semantic.DeepEqual(helper.ControlPlaneWorkerPoolForShoot(oldShoot.Spec.Provider.Workers), helper.ControlPlaneWorkerPoolForShoot(shoot.Spec.Provider.Workers)) } diff --git a/skaffold-gardenadm.yaml b/skaffold-gardenadm.yaml index 63fecc64b89..c945f0640ed 100644 --- a/skaffold-gardenadm.yaml +++ b/skaffold-gardenadm.yaml @@ -889,6 +889,7 @@ build: - pkg/gardenlet/controller/controllerinstallation/required - pkg/gardenlet/controller/controllerinstallation/utils - pkg/gardenlet/controller/gardenlet + - pkg/gardenlet/controller/lease - pkg/gardenlet/controller/managedseed - pkg/gardenlet/controller/networkpolicy - pkg/gardenlet/controller/seed @@ -897,6 +898,7 @@ build: - pkg/gardenlet/controller/seed/seed - pkg/gardenlet/controller/shoot - pkg/gardenlet/controller/shoot/care + - pkg/gardenlet/controller/shoot/lease - pkg/gardenlet/controller/shoot/shoot - pkg/gardenlet/controller/shoot/shoot/helper - pkg/gardenlet/controller/shoot/state diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml index 234857b7643..4a100372750 100644 --- a/skaffold-operator.yaml +++ b/skaffold-operator.yaml @@ -730,6 +730,7 @@ build: - pkg/controllermanager/controller/credentialsbinding - pkg/controllermanager/controller/event - pkg/controllermanager/controller/exposureclass + - pkg/controllermanager/controller/gardenletlifecycle - pkg/controllermanager/controller/managedseedset - pkg/controllermanager/controller/namespacedcloudprofile - pkg/controllermanager/controller/project @@ -741,7 +742,6 @@ build: - pkg/controllermanager/controller/seed - pkg/controllermanager/controller/seed/backupbucketscheck - pkg/controllermanager/controller/seed/extensionscheck - - pkg/controllermanager/controller/seed/lifecycle - pkg/controllermanager/controller/seed/reference - pkg/controllermanager/controller/seed/secrets - pkg/controllermanager/controller/seed/utils diff --git a/skaffold-seed.yaml b/skaffold-seed.yaml index 86d8e27d321..9f60d0474b4 100644 --- a/skaffold-seed.yaml +++ b/skaffold-seed.yaml @@ -309,6 +309,7 @@ build: - pkg/gardenlet/controller/controllerinstallation/required - pkg/gardenlet/controller/controllerinstallation/utils - pkg/gardenlet/controller/gardenlet + - pkg/gardenlet/controller/lease - pkg/gardenlet/controller/managedseed - pkg/gardenlet/controller/networkpolicy - pkg/gardenlet/controller/seed @@ -317,6 +318,7 @@ build: - pkg/gardenlet/controller/seed/seed - pkg/gardenlet/controller/shoot - pkg/gardenlet/controller/shoot/care + - pkg/gardenlet/controller/shoot/lease - pkg/gardenlet/controller/shoot/shoot - pkg/gardenlet/controller/shoot/shoot/helper - pkg/gardenlet/controller/shoot/state diff --git a/skaffold.yaml b/skaffold.yaml index 1d8c51412d5..4ffe908bb22 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -305,6 +305,7 @@ build: - pkg/controllermanager/controller/credentialsbinding - pkg/controllermanager/controller/event - pkg/controllermanager/controller/exposureclass + - pkg/controllermanager/controller/gardenletlifecycle - pkg/controllermanager/controller/managedseedset - pkg/controllermanager/controller/namespacedcloudprofile - pkg/controllermanager/controller/project @@ -316,7 +317,6 @@ build: - pkg/controllermanager/controller/seed - pkg/controllermanager/controller/seed/backupbucketscheck - pkg/controllermanager/controller/seed/extensionscheck - - pkg/controllermanager/controller/seed/lifecycle - pkg/controllermanager/controller/seed/reference - pkg/controllermanager/controller/seed/secrets - pkg/controllermanager/controller/seed/utils @@ -1225,6 +1225,7 @@ build: - pkg/gardenlet/controller/controllerinstallation/required - pkg/gardenlet/controller/controllerinstallation/utils - pkg/gardenlet/controller/gardenlet + - pkg/gardenlet/controller/lease - pkg/gardenlet/controller/managedseed - pkg/gardenlet/controller/networkpolicy - pkg/gardenlet/controller/seed @@ -1233,6 +1234,7 @@ build: - pkg/gardenlet/controller/seed/seed - pkg/gardenlet/controller/shoot - pkg/gardenlet/controller/shoot/care + - pkg/gardenlet/controller/shoot/lease - pkg/gardenlet/controller/shoot/shoot - pkg/gardenlet/controller/shoot/shoot/helper - pkg/gardenlet/controller/shoot/state diff --git a/test/framework/k8s_utils_test.go b/test/framework/k8s_utils_test.go index 084a6c5c779..f4e8329f182 100644 --- a/test/framework/k8s_utils_test.go +++ b/test/framework/k8s_utils_test.go @@ -317,7 +317,7 @@ func appendShootConditionsToShoot(shoot *gardencorev1beta1.Shoot) { func appendSeedConditionsToShoot(shoot *gardencorev1beta1.Shoot) { shoot.Status.Conditions = append(shoot.Status.Conditions, []gardencorev1beta1.Condition{ { - Type: gardencorev1beta1.SeedGardenletReady, + Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue, }, { diff --git a/test/integration/controllermanager/seed/lifecycle/lifecycle_suite_test.go b/test/integration/controllermanager/gardenletlifecycle/lifecycle_suite_test.go similarity index 89% rename from test/integration/controllermanager/seed/lifecycle/lifecycle_suite_test.go rename to test/integration/controllermanager/gardenletlifecycle/lifecycle_suite_test.go index f354fbdfde2..6dd036d0ad6 100644 --- a/test/integration/controllermanager/seed/lifecycle/lifecycle_suite_test.go +++ b/test/integration/controllermanager/gardenletlifecycle/lifecycle_suite_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package lifecycle_test +package gardenletlifecycle_test import ( "context" @@ -15,7 +15,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/rest" testclock "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" @@ -30,15 +29,15 @@ import ( "github.com/gardener/gardener/pkg/api/indexer" "github.com/gardener/gardener/pkg/client/kubernetes" controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" - "github.com/gardener/gardener/pkg/controllermanager/controller/seed/lifecycle" + "github.com/gardener/gardener/pkg/controllermanager/controller/gardenletlifecycle" "github.com/gardener/gardener/pkg/logger" - "github.com/gardener/gardener/pkg/utils" + . "github.com/gardener/gardener/pkg/utils/test/matchers" gardenerenvtest "github.com/gardener/gardener/test/envtest" ) -func TestLifecycle(t *testing.T) { +func TestGardenletLifecycle(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Test Integration ControllerManager Seed Lifecycle Suite") + RunSpecs(t, "Test Integration ControllerManager GardenletLifecycle Suite") } const ( @@ -90,13 +89,17 @@ var _ = BeforeSuite(func() { By("Create test Namespace") testNamespace = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - // create dedicated namespace for each test run, so that we can run multiple tests concurrently for stress tests - GenerateName: testID + "-", + GenerateName: "gardener-system-seed-lease-", }, } Expect(testClient.Create(ctx, testNamespace)).To(Succeed()) log.Info("Created Namespace for test", "namespaceName", testNamespace.Name) - testRunID = testNamespace.Name + "-" + utils.ComputeSHA256Hex([]byte(uuid.NewUUID()))[:8] + testRunID = testNamespace.Name + + DeferCleanup(func() { + By("Delete test Namespace") + Expect(testClient.Delete(ctx, testNamespace)).To(Or(Succeed(), BeNotFoundError())) + }) By("Setup manager") mgr, err := manager.New(restConfig, manager.Options{ @@ -117,7 +120,7 @@ var _ = BeforeSuite(func() { By("Register controller") fakeClock = testclock.NewFakeClock(time.Now()) - Expect((&lifecycle.Reconciler{ + Expect((&gardenletlifecycle.Reconciler{ Config: controllermanagerconfigv1alpha1.SeedControllerConfiguration{ MonitorPeriod: &metav1.Duration{Duration: seedMonitorPeriod}, ShootMonitorPeriod: &metav1.Duration{Duration: shootMonitorPeriod}, diff --git a/test/integration/controllermanager/seed/lifecycle/lifecycle_test.go b/test/integration/controllermanager/gardenletlifecycle/lifecycle_test.go similarity index 98% rename from test/integration/controllermanager/seed/lifecycle/lifecycle_test.go rename to test/integration/controllermanager/gardenletlifecycle/lifecycle_test.go index f180c398e23..2dc7b45d7a2 100644 --- a/test/integration/controllermanager/seed/lifecycle/lifecycle_test.go +++ b/test/integration/controllermanager/gardenletlifecycle/lifecycle_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package lifecycle_test +package gardenletlifecycle_test import ( "time" @@ -20,7 +20,7 @@ import ( . "github.com/gardener/gardener/pkg/utils/test/matchers" ) -var _ = Describe("Seed Lifecycle controller tests", func() { +var _ = Describe("Gardenlet Lifecycle controller tests", func() { var ( seed *gardencorev1beta1.Seed lease *coordinationv1.Lease @@ -168,7 +168,7 @@ var _ = Describe("Seed Lifecycle controller tests", func() { By("Add GardenletReady condition to Seed") patch := client.MergeFrom(seed.DeepCopy()) seed.Status.Conditions = []gardencorev1beta1.Condition{{ - Type: gardencorev1beta1.SeedGardenletReady, + Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue, LastTransitionTime: metav1.Time{Time: fakeClock.Now().Add(-24 * time.Hour)}, }} diff --git a/test/integration/controllermanager/managedseedset/managedseedset_test.go b/test/integration/controllermanager/managedseedset/managedseedset_test.go index a8faa98289b..87a9d9ce1a1 100644 --- a/test/integration/controllermanager/managedseedset/managedseedset_test.go +++ b/test/integration/controllermanager/managedseedset/managedseedset_test.go @@ -90,7 +90,7 @@ var _ = Describe("ManagedSeedSet controller test", func() { seed.Status = gardencorev1beta1.SeedStatus{ ObservedGeneration: seed.GetGeneration(), Conditions: []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedBackupBucketsReady, Status: gardencorev1beta1.ConditionTrue}, }, diff --git a/test/integration/controllermanager/shoot/conditions/conditions_test.go b/test/integration/controllermanager/shoot/conditions/conditions_test.go index 3db3e58aeed..f82bd180e54 100644 --- a/test/integration/controllermanager/shoot/conditions/conditions_test.go +++ b/test/integration/controllermanager/shoot/conditions/conditions_test.go @@ -130,7 +130,7 @@ var _ = Describe("Shoot Conditions controller tests", func() { }).Should(And( Not(ContainCondition(OfType(gardencorev1beta1.SeedBackupBucketsReady))), Not(ContainCondition(OfType(gardencorev1beta1.SeedExtensionsReady))), - Not(ContainCondition(OfType(gardencorev1beta1.SeedGardenletReady))), + Not(ContainCondition(OfType(gardencorev1beta1.GardenletReady))), Not(ContainCondition(OfType(gardencorev1beta1.SeedSystemComponentsHealthy))), )) }) @@ -150,7 +150,7 @@ var _ = Describe("Shoot Conditions controller tests", func() { }).Should(And( Not(ContainCondition(OfType(gardencorev1beta1.SeedBackupBucketsReady))), Not(ContainCondition(OfType(gardencorev1beta1.SeedExtensionsReady))), - Not(ContainCondition(OfType(gardencorev1beta1.SeedGardenletReady))), + Not(ContainCondition(OfType(gardencorev1beta1.GardenletReady))), Not(ContainCondition(OfType(gardencorev1beta1.SeedSystemComponentsHealthy))), )) }) @@ -182,7 +182,7 @@ var _ = Describe("Shoot Conditions controller tests", func() { conditions := []gardencorev1beta1.Condition{ {Type: gardencorev1beta1.SeedBackupBucketsReady, Status: gardencorev1beta1.ConditionProgressing}, {Type: gardencorev1beta1.SeedExtensionsReady, Status: gardencorev1beta1.ConditionProgressing}, - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionProgressing}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionProgressing}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionProgressing}, {Type: gardencorev1beta1.ConditionType("custom"), Status: gardencorev1beta1.ConditionProgressing}, } @@ -199,7 +199,7 @@ var _ = Describe("Shoot Conditions controller tests", func() { }).Should(And( ContainCondition(OfType(gardencorev1beta1.SeedBackupBucketsReady)), ContainCondition(OfType(gardencorev1beta1.SeedExtensionsReady)), - ContainCondition(OfType(gardencorev1beta1.SeedGardenletReady)), + ContainCondition(OfType(gardencorev1beta1.GardenletReady)), ContainCondition(OfType(gardencorev1beta1.SeedSystemComponentsHealthy)), ContainCondition(OfType("custom")), )) @@ -211,7 +211,7 @@ var _ = Describe("Shoot Conditions controller tests", func() { }).Should(And( ContainCondition(OfType(gardencorev1beta1.SeedBackupBucketsReady)), ContainCondition(OfType(gardencorev1beta1.SeedExtensionsReady)), - ContainCondition(OfType(gardencorev1beta1.SeedGardenletReady)), + ContainCondition(OfType(gardencorev1beta1.GardenletReady)), ContainCondition(OfType(gardencorev1beta1.SeedSystemComponentsHealthy)), ContainCondition(OfType("custom")), )) @@ -235,7 +235,7 @@ var _ = Describe("Shoot Conditions controller tests", func() { }).ShouldNot(And( ContainCondition(OfType(gardencorev1beta1.SeedBackupBucketsReady)), ContainCondition(OfType(gardencorev1beta1.SeedExtensionsReady)), - ContainCondition(OfType(gardencorev1beta1.SeedGardenletReady)), + ContainCondition(OfType(gardencorev1beta1.GardenletReady)), ContainCondition(OfType(gardencorev1beta1.SeedSystemComponentsHealthy)), ContainCondition(OfType("custom")), )) diff --git a/test/integration/controllermanager/shoot/migration/migration_test.go b/test/integration/controllermanager/shoot/migration/migration_test.go index 8323a77e4e0..316126be7c9 100644 --- a/test/integration/controllermanager/shoot/migration/migration_test.go +++ b/test/integration/controllermanager/shoot/migration/migration_test.go @@ -240,7 +240,7 @@ var _ = Describe("Shoot Migration controller tests", Ordered, func() { destinationSeed.Status.Gardener = sourceSeed.Status.Gardener destinationSeed.Status.ObservedGeneration = destinationSeed.Generation destinationSeed.Status.Conditions = []gardencorev1beta1.Condition{ - {Type: gardencorev1beta1.SeedGardenletReady, Status: gardencorev1beta1.ConditionTrue}, + {Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue}, {Type: gardencorev1beta1.SeedSystemComponentsHealthy, Status: gardencorev1beta1.ConditionTrue}, } Expect(testClient.Status().Update(ctx, destinationSeed)).To(Succeed()) diff --git a/test/integration/gardenlet/seed/lease/lease_suite_test.go b/test/integration/gardenlet/seed/lease/lease_suite_test.go index 1689ccb7cb4..b2f4c2159c8 100644 --- a/test/integration/gardenlet/seed/lease/lease_suite_test.go +++ b/test/integration/gardenlet/seed/lease/lease_suite_test.go @@ -172,16 +172,9 @@ var _ = BeforeSuite(func() { fakeClock = testclock.NewFakeClock(time.Now()) healthManager = healthz.NewDefaultHealthz() - Expect((&lease.Reconciler{ - SeedRESTClient: kubernetesClient.RESTClient(), - Config: gardenletconfigv1alpha1.SeedControllerConfiguration{ - LeaseResyncSeconds: ptr.To[int32](1), - }, - Clock: fakeClock, - HealthManager: healthManager, - LeaseNamespace: testNamespace.Name, - SeedName: seed.Name, - }).AddToManager(mgr, mgr)).To(Succeed()) + Expect(lease.AddToManager(mgr, mgr, kubernetesClient.RESTClient(), gardenletconfigv1alpha1.SeedControllerConfiguration{ + LeaseResyncSeconds: ptr.To[int32](1), + }, healthManager, seed.Name, fakeClock, &testNamespace.Name)).To(Succeed()) By("Start manager") mgrContext, mgrCancel := context.WithCancel(ctx) diff --git a/test/integration/gardenlet/seed/lease/lease_test.go b/test/integration/gardenlet/seed/lease/lease_test.go index dc5bc9d4482..b7a35e1c922 100644 --- a/test/integration/gardenlet/seed/lease/lease_test.go +++ b/test/integration/gardenlet/seed/lease/lease_test.go @@ -18,7 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - leasereconciler "github.com/gardener/gardener/pkg/gardenlet/controller/seed/lease" + leasecontroller "github.com/gardener/gardener/pkg/gardenlet/controller/lease" "github.com/gardener/gardener/pkg/utils/test" . "github.com/gardener/gardener/pkg/utils/test/matchers" ) @@ -26,85 +26,99 @@ import ( var _ = Describe("Seed lease controller tests", func() { var lease *coordinationv1.Lease - BeforeEach(func() { + BeforeEach(OncePerOrdered, func() { fakeClock.SetTime(time.Now()) lease = &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{Name: seed.Name, Namespace: testNamespace.Name}} }) - It("should maintain the Lease object and set the internal health status to true", func() { - By("Ensure Lease got maintained") - Eventually(func(g Gomega) { - g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) - g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Seed", Name: seed.Name, UID: seed.UID})) - g.Expect(lease.Spec.RenewTime.Sub(fakeClock.Now())).To(BeNumerically("<=", 0)) - g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(1)))) - g.Expect(lease.Spec.HolderIdentity).To(Equal(&seed.Name)) - g.Expect(healthManager.Get()).To(BeTrue()) - }).Should(Succeed()) - - By("Ensure GardenletReady condition was maintained") - Eventually(func(g Gomega) { - g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(seed), seed)).To(Succeed()) - g.Expect(seed.Status.Conditions).To(ContainCondition(OfType(gardencorev1beta1.SeedGardenletReady), WithStatus(gardencorev1beta1.ConditionTrue))) - }).Should(Succeed()) - - By("Step clock") - fakeClock.Step(time.Hour) - - By("Ensure health status is true") - Consistently(func() bool { - return healthManager.Get() - }).Should(BeTrue()) - - By("Ensure Lease got updated") - Eventually(func(g Gomega) { - g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) - g.Expect(lease.Spec.RenewTime.Sub(fakeClock.Now())).To(BeNumerically("<=", 0)) - g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Seed", Name: seed.Name, UID: seed.UID})) - g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(1)))) - g.Expect(lease.Spec.HolderIdentity).To(Equal(&seed.Name)) - }).Should(Succeed()) + Describe("maintain the Lease object and set the internal health status to true", Ordered, func() { + It("should ensured Lease gets maintained", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) + g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Seed", Name: seed.Name, UID: seed.UID})) + g.Expect(lease.Spec.RenewTime.Sub(fakeClock.Now())).To(BeNumerically("<=", 0)) + g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(1)))) + g.Expect(lease.Spec.HolderIdentity).To(Equal(&seed.Name)) + g.Expect(healthManager.Get()).To(BeTrue()) + }).Should(Succeed()) + }) + + It("should ensure GardenletReady condition gets maintained", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(seed), seed)).To(Succeed()) + g.Expect(seed.Status.Conditions).To(ContainCondition(OfType(gardencorev1beta1.GardenletReady), WithStatus(gardencorev1beta1.ConditionTrue))) + }).Should(Succeed()) + }) + + It("should step the fake clock", func() { + fakeClock.Step(time.Hour) + }) + + It("should ensure health status is set to true", func() { + Consistently(func() bool { + return healthManager.Get() + }).Should(BeTrue()) + }) + + It("should ensure Lease gets updated", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) + g.Expect(lease.Spec.RenewTime.Sub(fakeClock.Now())).To(BeNumerically("<=", 0)) + g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Seed", Name: seed.Name, UID: seed.UID})) + g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(1)))) + g.Expect(lease.Spec.HolderIdentity).To(Equal(&seed.Name)) + }).Should(Succeed()) + }) }) - It("should not update the Lease object and set the internal health status to false", func() { + Describe("do not update the Lease object and set the internal health status to false", Ordered, func() { var fakeError error - DeferCleanup(test.WithVar(&leasereconciler.CheckSeedConnection, func(context.Context, rest.Interface) error { return fakeError })) - - By("Ensure Lease got maintained") - Eventually(func(g Gomega) { - g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) - g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Seed", Name: seed.Name, UID: seed.UID})) - g.Expect(lease.Spec.RenewTime.Sub(fakeClock.Now())).To(BeNumerically("<=", 0)) - g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(1)))) - g.Expect(lease.Spec.HolderIdentity).To(Equal(&seed.Name)) - g.Expect(healthManager.Get()).To(BeTrue()) - }).Should(Succeed()) - - By("Ensure GardenletReady condition was maintained") - Eventually(func(g Gomega) { - g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(seed), seed)).To(Succeed()) - g.Expect(seed.Status.Conditions).To(ContainCondition(OfType(gardencorev1beta1.SeedGardenletReady), WithStatus(gardencorev1beta1.ConditionTrue))) - }).Should(Succeed()) - - By("Ensure seed connection fails") - fakeError = errors.New("fake") - - By("Ensure health status was set to false") - Eventually(func() bool { - return healthManager.Get() - }).Should(BeFalse()) - - By("Step clock") - fakeClock.Step(time.Hour) - - By("Ensure Lease did not get updated") - Consistently(func(g Gomega) { - g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) - g.Expect(fakeClock.Now().Sub(lease.Spec.RenewTime.Time)).To(BeNumerically(">=", time.Hour)) - g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Seed", Name: seed.Name, UID: seed.UID})) - g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(1)))) - g.Expect(lease.Spec.HolderIdentity).To(Equal(&seed.Name)) - }).Should(Succeed()) + + BeforeEach(OncePerOrdered, func() { + DeferCleanup(test.WithVar(&leasecontroller.CheckConnection, func(context.Context, rest.Interface) error { return fakeError })) + }) + + It("should ensure Lease gets maintained", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) + g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Seed", Name: seed.Name, UID: seed.UID})) + g.Expect(lease.Spec.RenewTime.Sub(fakeClock.Now())).To(BeNumerically("<=", 0)) + g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(1)))) + g.Expect(lease.Spec.HolderIdentity).To(Equal(&seed.Name)) + g.Expect(healthManager.Get()).To(BeTrue()) + }).Should(Succeed()) + }) + + It("should ensure GardenletReady condition gets maintained", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(seed), seed)).To(Succeed()) + g.Expect(seed.Status.Conditions).To(ContainCondition(OfType(gardencorev1beta1.GardenletReady), WithStatus(gardencorev1beta1.ConditionTrue))) + }).Should(Succeed()) + }) + + It("should make the seed connection fail", func() { + fakeError = errors.New("fake") + }) + + It("should ensure health status gets set to false", func() { + Eventually(func() bool { + return healthManager.Get() + }).Should(BeFalse()) + }) + + It("should step the fake clock", func() { + fakeClock.Step(time.Hour) + }) + + It("should ensure the Lease object is not updated and the internal health status remains false", func() { + Consistently(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) + g.Expect(fakeClock.Now().Sub(lease.Spec.RenewTime.Time)).To(BeNumerically(">=", time.Hour)) + g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Seed", Name: seed.Name, UID: seed.UID})) + g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(1)))) + g.Expect(lease.Spec.HolderIdentity).To(Equal(&seed.Name)) + }).Should(Succeed()) + }) }) }) diff --git a/test/integration/gardenlet/shoot/lease/lease_suite_test.go b/test/integration/gardenlet/shoot/lease/lease_suite_test.go new file mode 100644 index 00000000000..cf1843c6191 --- /dev/null +++ b/test/integration/gardenlet/shoot/lease/lease_suite_test.go @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package lease_test + +import ( + "context" + "testing" + "time" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubernetesclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + testclock "k8s.io/utils/clock/testing" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + controllerconfig "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/lease" + "github.com/gardener/gardener/pkg/healthz" + "github.com/gardener/gardener/pkg/logger" + . "github.com/gardener/gardener/pkg/utils/test/matchers" + gardenerenvtest "github.com/gardener/gardener/test/envtest" +) + +func TestLease(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Test Integration Gardenlet Shoot Lease Suite") +} + +const testID = "shoot-lease-controller-test" + +var ( + ctx = context.Background() + log logr.Logger + + restConfig *rest.Config + testEnv *gardenerenvtest.GardenerTestEnvironment + testClient client.Client + + testNamespace *corev1.Namespace + testRunID string + + shoot *gardencorev1beta1.Shoot + fakeClock *testclock.FakeClock + healthManager healthz.Manager +) + +var _ = BeforeSuite(func() { + logf.SetLogger(logger.MustNewZapLogger(logger.DebugLevel, logger.FormatJSON, zap.WriteTo(GinkgoWriter))) + log = logf.Log.WithName(testID) + + By("Start test environment") + testEnv = &gardenerenvtest.GardenerTestEnvironment{ + Environment: &envtest.Environment{}, + GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,ExtensionValidator,ShootDNS,ShootQuotaValidator,ShootTolerationRestriction,ShootValidator,ShootMutator"}, + }, + } + + var err error + restConfig, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(restConfig).NotTo(BeNil()) + + DeferCleanup(func() { + By("Stop test environment") + Expect(testEnv.Stop()).To(Succeed()) + }) + + testSchemeBuilder := runtime.NewSchemeBuilder( + kubernetes.AddGardenSchemeToScheme, + resourcesv1alpha1.AddToScheme, + ) + testScheme := runtime.NewScheme() + Expect(testSchemeBuilder.AddToScheme(testScheme)).To(Succeed()) + + By("Create test client") + testClient, err = client.New(restConfig, client.Options{Scheme: testScheme}) + Expect(err).NotTo(HaveOccurred()) + + By("Create test Namespace") + testNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "garden-", + }, + } + Expect(testClient.Create(ctx, testNamespace)).To(Succeed()) + log.Info("Created Namespace for test", "namespaceName", testNamespace.Name) + testRunID = testNamespace.Name + + DeferCleanup(func() { + By("Delete test Namespace") + Expect(testClient.Delete(ctx, testNamespace)).To(Or(Succeed(), BeNotFoundError())) + }) + + By("Create Shoot") + shoot = &gardencorev1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "shoot-", + Namespace: testNamespace.Name, + Labels: map[string]string{testID: testRunID}, + }, + Spec: gardencorev1beta1.ShootSpec{ + CredentialsBindingName: ptr.To("my-provider-account"), + CloudProfile: &gardencorev1beta1.CloudProfileReference{Name: "test-cloudprofile", Kind: "CloudProfile"}, + Region: "foo-region", + Provider: gardencorev1beta1.Provider{ + Type: "test", + Workers: []gardencorev1beta1.Worker{ + { + Name: "cpu-worker", + Minimum: 2, + Maximum: 2, + Machine: gardencorev1beta1.Machine{ + Type: "large", + }, + }, + }, + }, + Kubernetes: gardencorev1beta1.Kubernetes{ + Version: "1.31.1", + }, + Networking: &gardencorev1beta1.Networking{ + Type: ptr.To("foo-networking"), + }, + }, + } + Expect(testClient.Create(ctx, shoot)).To(Succeed()) + log.Info("Created Shoot for test", "shoot", client.ObjectKeyFromObject(shoot)) + + DeferCleanup(func() { + By("Delete Shoot") + Expect(testClient.Delete(ctx, shoot)).To(Or(Succeed(), BeNotFoundError())) + }) + + By("Setup manager") + mgr, err := manager.New(restConfig, manager.Options{ + Scheme: testScheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespace.Name: {}}, + }, + Controller: controllerconfig.Controller{ + SkipNameValidation: ptr.To(true), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Create Kubernetes clientset") + kubernetesClient, err := kubernetesclientset.NewForConfig(mgr.GetConfig()) + Expect(err).NotTo(HaveOccurred()) + + By("Register controller") + fakeClock = testclock.NewFakeClock(time.Now()) + healthManager = healthz.NewDefaultHealthz() + + Expect(lease.AddToManager(mgr, mgr, kubernetesClient.RESTClient(), healthManager, fakeClock)).To(Succeed()) + + By("Start manager") + mgrContext, mgrCancel := context.WithCancel(ctx) + + go func() { + defer GinkgoRecover() + Expect(mgr.Start(mgrContext)).To(Succeed()) + }() + + DeferCleanup(func() { + By("Stop manager") + mgrCancel() + }) +}) diff --git a/test/integration/gardenlet/shoot/lease/lease_test.go b/test/integration/gardenlet/shoot/lease/lease_test.go new file mode 100644 index 00000000000..fcaa7f22192 --- /dev/null +++ b/test/integration/gardenlet/shoot/lease/lease_test.go @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package lease_test + +import ( + "context" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + leasecontroller "github.com/gardener/gardener/pkg/gardenlet/controller/lease" + "github.com/gardener/gardener/pkg/utils/test" + . "github.com/gardener/gardener/pkg/utils/test/matchers" +) + +var _ = Describe("Shoot lease controller tests", func() { + var lease *coordinationv1.Lease + + BeforeEach(OncePerOrdered, func() { + fakeClock.SetTime(time.Now()) + + lease = &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{Name: "self-hosted-shoot-" + shoot.Name, Namespace: shoot.Namespace}} + }) + + Describe("maintain the Lease object and set the internal health status to true", Ordered, func() { + It("should ensured Lease gets maintained", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) + g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Shoot", Name: shoot.Name, UID: shoot.UID})) + g.Expect(lease.Spec.RenewTime.Sub(fakeClock.Now())).To(BeNumerically("<=", 0)) + g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(2)))) + g.Expect(lease.Spec.HolderIdentity).To(Equal(&shoot.Name)) + g.Expect(healthManager.Get()).To(BeTrue()) + }).Should(Succeed()) + }) + + It("should ensure GardenletReady condition gets maintained", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(shoot), shoot)).To(Succeed()) + g.Expect(shoot.Status.Conditions).To(ContainCondition(OfType(gardencorev1beta1.GardenletReady), WithStatus(gardencorev1beta1.ConditionTrue))) + }).Should(Succeed()) + }) + + It("should step the fake clock", func() { + fakeClock.Step(time.Hour) + }) + + It("should ensure health status is set to true", func() { + Consistently(func() bool { + return healthManager.Get() + }).Should(BeTrue()) + }) + + It("should ensure Lease gets updated", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) + g.Expect(lease.Spec.RenewTime.Sub(fakeClock.Now())).To(BeNumerically("<=", 0)) + g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Shoot", Name: shoot.Name, UID: shoot.UID})) + g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(2)))) + g.Expect(lease.Spec.HolderIdentity).To(Equal(&shoot.Name)) + }).Should(Succeed()) + }) + }) + + Describe("do not update the Lease object and set the internal health status to false", Ordered, func() { + var fakeError error + + BeforeEach(OncePerOrdered, func() { + DeferCleanup(test.WithVar(&leasecontroller.CheckConnection, func(context.Context, rest.Interface) error { return fakeError })) + }) + + It("should ensure Lease gets maintained", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) + g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Shoot", Name: shoot.Name, UID: shoot.UID})) + g.Expect(lease.Spec.RenewTime.Sub(fakeClock.Now())).To(BeNumerically("<=", 0)) + g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(2)))) + g.Expect(lease.Spec.HolderIdentity).To(Equal(&shoot.Name)) + g.Expect(healthManager.Get()).To(BeTrue()) + }).Should(Succeed()) + }) + + It("should ensure GardenletReady condition gets maintained", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(shoot), shoot)).To(Succeed()) + g.Expect(shoot.Status.Conditions).To(ContainCondition(OfType(gardencorev1beta1.GardenletReady), WithStatus(gardencorev1beta1.ConditionTrue))) + }).Should(Succeed()) + }) + + It("should make the seed connection fail", func() { + fakeError = errors.New("fake") + }) + + It("should ensure health status gets set to false", func() { + Eventually(func() bool { + return healthManager.Get() + }).Should(BeFalse()) + }) + + It("should step the fake clock", func() { + fakeClock.Step(time.Hour) + }) + + It("should ensure the Lease object is not updated and the internal health status remains false", func() { + Consistently(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(lease), lease)).To(Succeed()) + g.Expect(fakeClock.Now().Sub(lease.Spec.RenewTime.Time)).To(BeNumerically(">=", time.Hour)) + g.Expect(lease.OwnerReferences).To(ConsistOf(metav1.OwnerReference{APIVersion: "core.gardener.cloud/v1beta1", Kind: "Shoot", Name: shoot.Name, UID: shoot.UID})) + g.Expect(lease.Spec.LeaseDurationSeconds).To(PointTo(Equal(int32(2)))) + g.Expect(lease.Spec.HolderIdentity).To(Equal(&shoot.Name)) + }).Should(Succeed()) + }) + }) +}) diff --git a/test/integration/scheduler/shoot/shoot_test.go b/test/integration/scheduler/shoot/shoot_test.go index 6882e283802..c4525344f24 100644 --- a/test/integration/scheduler/shoot/shoot_test.go +++ b/test/integration/scheduler/shoot/shoot_test.go @@ -731,7 +731,7 @@ func createSeed(region string, zones []string, accessRestrictions []gardencorev1 }, Conditions: []gardencorev1beta1.Condition{ { - Type: gardencorev1beta1.SeedGardenletReady, + Type: gardencorev1beta1.GardenletReady, Status: gardencorev1beta1.ConditionTrue, }, }, From 58c114918a31aa99be01e257f9fec838204b3930 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Tue, 11 Nov 2025 10:43:50 +0100 Subject: [PATCH 078/176] Update helm to v3.19.1 (#13408) --- go.mod | 2 +- go.sum | 4 ++-- hack/tools.mk | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 81f96642331..56392687794 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( gomodules.xyz/jsonpatch/v2 v2.5.0 gonum.org/v1/gonum v0.16.0 google.golang.org/protobuf v1.36.10 - helm.sh/helm/v3 v3.18.6 + helm.sh/helm/v3 v3.19.1 istio.io/api v1.27.3 istio.io/client-go v1.27.2 k8s.io/api v0.34.1 diff --git a/go.sum b/go.sum index 723004333db..f119bd3bc96 100644 --- a/go.sum +++ b/go.sum @@ -1074,8 +1074,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -helm.sh/helm/v3 v3.18.6 h1:S/2CqcYnNfLckkHLI0VgQbxgcDaU3N4A/46E3n9wSNY= -helm.sh/helm/v3 v3.18.6/go.mod h1:L/dXDR2r539oPlFP1PJqKAC1CUgqHJDLkxKpDGrWnyg= +helm.sh/helm/v3 v3.19.1 h1:QVMzHbanyurO8oynx0drDOfG02XxSvrHqaFrf9yrMf0= +helm.sh/helm/v3 v3.19.1/go.mod h1:gX10tB5ErM+8fr7bglUUS/UfTOO8UUTYWIBH1IYNnpE= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/tools.mk b/hack/tools.mk index 2217843c385..02dd2c23ab7 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -65,7 +65,7 @@ GO_ADD_LICENSE_VERSION ?= v1.2.0 GOIMPORTSREVISER_VERSION ?= v3.10.0 GO_VULN_CHECK_VERSION ?= latest # renovate: datasource=github-releases depName=helm/helm -HELM_VERSION ?= v3.18.6 +HELM_VERSION ?= v3.19.1 # renovate: datasource=github-releases depName=kubernetes-sigs/kind KIND_VERSION ?= v0.30.0 # renovate: datasource=github-releases depName=kubernetes/kubernetes From 1ba4d935ed202b5da4cd9d4ad91f9e58b540ab2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20G=C3=B6tz?= <47362717+oliver-goetz@users.noreply.github.com> Date: Tue, 11 Nov 2025 15:02:51 +0100 Subject: [PATCH 079/176] Fix local setup for docker>=v29.0.0 (#13410) --- hack/kind-up.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/kind-up.sh b/hack/kind-up.sh index 2d8b9875c1c..4493b0604b2 100755 --- a/hack/kind-up.sh +++ b/hack/kind-up.sh @@ -84,7 +84,7 @@ setup_kind_network() { local network network_options network_ipam expected_network_ipam network="$(docker network inspect $existing_network_id | yq '.[]')" network_options="$(echo "$network" | yq '.EnableIPv6 + "," + .Options["com.docker.network.bridge.enable_ip_masquerade"]')" - network_ipam="$(echo "$network" | yq '.IPAM.Config' -o=json -I=0)" + network_ipam="$(echo "$network" | yq '.IPAM.Config' -o=json -I=0 | sed -E 's/"IPRange":"",//g')" expected_network_ipam='[{"Subnet":"172.18.0.0/16","Gateway":"172.18.0.1"},{"Subnet":"fd00:10::/64","Gateway":"fd00:10::1"}]' if [ "$network_options" = 'true,true' ] && [ "$network_ipam" = "$expected_network_ipam" ] ; then From 448264134c7e93a2a949626fe975299d0bba8eda Mon Sep 17 00:00:00 2001 From: Tomas Valasek <13797085+tvalasek@users.noreply.github.com> Date: Tue, 11 Nov 2025 16:07:53 +0100 Subject: [PATCH 080/176] Flux link fix (#13418) The original repository was archived by the owner on Apr 11, 2025. This is the new link --- extensions/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/README.md b/extensions/README.md index bc2986fbf96..9ff935d5505 100644 --- a/extensions/README.md +++ b/extensions/README.md @@ -65,7 +65,7 @@ The primary DNS provider manages `DNSRecord` resources (mandatory for Gardener r - [Shoot Certificate Service](https://github.com/gardener/gardener-extension-shoot-cert-service) - [Shoot DNS Service](https://github.com/gardener/gardener-extension-shoot-dns-service) - [Shoot Falco Service](https://github.com/gardener/gardener-extension-shoot-falco-service) -- [Shoot Flux Service](https://github.com/23technologies/gardener-extension-shoot-flux) +- [Shoot Flux Service](https://github.com/stackitcloud/gardener-extension-shoot-flux) - [Shoot Lakom Service](https://github.com/gardener/gardener-extension-shoot-lakom-service) - [Shoot Networking Filter](https://github.com/gardener/gardener-extension-shoot-networking-filter) - [Shoot Networking Problem Detector](https://github.com/gardener/gardener-extension-shoot-networking-problemdetector) From a9c146a86ba2901604daad81c4e1479721d1b8ca Mon Sep 17 00:00:00 2001 From: Indrit Fejza Date: Tue, 11 Nov 2025 19:05:52 +0100 Subject: [PATCH 081/176] Add VPN note to getting started locally guide (#13334) * Add VPN note to getting started locally guide Added a note about VPN issues affecting kubeconfig setup. * improve wording * improve help message * Update docs/deployment/getting_started_locally.md Co-authored-by: Marc Vornetran --------- Co-authored-by: Marc Vornetran --- docs/deployment/getting_started_locally.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/deployment/getting_started_locally.md b/docs/deployment/getting_started_locally.md index 82a1712b588..764b54978fa 100644 --- a/docs/deployment/getting_started_locally.md +++ b/docs/deployment/getting_started_locally.md @@ -329,6 +329,10 @@ For convenience a [helper script](../../hack/usage/generate-admin-kubeconf.sh) i ./hack/usage/generate-admin-kubeconf.sh > admin-kubeconf.yaml ``` +> [!NOTE] +> Keep in mind that using a VPN on your local machine could cause problems with the setup, and the shoot's kubeconfig could fail with connection issues. +> If you experience connection problems using the shoot's kubeconfig, try disabling the VPN first. + If you want to change the default namespace or shoot name, you can do so by passing different values as arguments. ```bash From 0405e73d2de53a08fa2ae337dd91c903f17e0307 Mon Sep 17 00:00:00 2001 From: Dominik Froehlich Date: Tue, 11 Nov 2025 19:05:58 +0100 Subject: [PATCH 082/176] fix: overlap validation of shoot defaults with seed networks (#13349) --- pkg/apis/core/validation/seed.go | 80 ++++++++++--- pkg/apis/core/validation/seed_test.go | 125 +++++++++++++++++---- pkg/utils/validation/cidr/disjoint.go | 58 ++++++---- pkg/utils/validation/cidr/disjoint_test.go | 8 +- 4 files changed, 208 insertions(+), 63 deletions(-) diff --git a/pkg/apis/core/validation/seed.go b/pkg/apis/core/validation/seed.go index 9495f9faf3c..8d4b824984e 100644 --- a/pkg/apis/core/validation/seed.go +++ b/pkg/apis/core/validation/seed.go @@ -288,46 +288,94 @@ func validateSeedNetworks(seedNetworks core.SeedNetworks, fldPath *field.Path, i return append(allErrs, errs...) } + // Four buckets: all cidrs, seed cidrs and shoot defaults V4 and V6 + // - all cidrs: must be valid CIDRs, must not overlap with reserved ranges + // - seed cidrs: must not overlap with each other + // - shoot defaults (V4): must not overlap with each other + // - shoot defaults (V6): must not overlap with each other, must not overlap with seed networks var ( - primaryIPFamily = helper.DeterminePrimaryIPFamily(seedNetworks.IPFamilies) - networks []cidrvalidation.CIDR - reservedSeedServiceRange = cidrvalidation.NewCIDR(v1beta1constants.ReservedKubeApiServerMappingRange, field.NewPath("")) + primaryIPFamily = helper.DeterminePrimaryIPFamily(seedNetworks.IPFamilies) + allCIDRs []cidrvalidation.CIDR + seedCIDRs []cidrvalidation.CIDR + shootDefaultsV4 []cidrvalidation.CIDR + shootDefaultsV6 []cidrvalidation.CIDR + pods cidrvalidation.CIDR + services cidrvalidation.CIDR + nodes cidrvalidation.CIDR + shootPods cidrvalidation.CIDR + shootServices cidrvalidation.CIDR + reservedKubeApiServerMappingRange = cidrvalidation.NewCIDR(v1beta1constants.ReservedKubeApiServerMappingRange, field.NewPath("")) ) if !inTemplate || len(seedNetworks.Pods) > 0 { - networks = append(networks, cidrvalidation.NewCIDR(seedNetworks.Pods, fldPath.Child("pods"))) + pods = cidrvalidation.NewCIDR(seedNetworks.Pods, fldPath.Child("pods")) + allCIDRs = append(allCIDRs, pods) + seedCIDRs = append(seedCIDRs, pods) } if !inTemplate || len(seedNetworks.Services) > 0 { - services := cidrvalidation.NewCIDR(seedNetworks.Services, fldPath.Child("services")) - networks = append(networks, services) + services = cidrvalidation.NewCIDR(seedNetworks.Services, fldPath.Child("services")) + allCIDRs = append(allCIDRs, services) + seedCIDRs = append(seedCIDRs, services) // Service range must not be larger than /8 for ipv4 if services.IsIPv4() { - maxSize, _ := reservedSeedServiceRange.GetIPNet().Mask.Size() + maxSize, _ := reservedKubeApiServerMappingRange.GetIPNet().Mask.Size() allErrs = append(allErrs, services.ValidateMaxSize(maxSize)...) } } if seedNetworks.Nodes != nil { - networks = append(networks, cidrvalidation.NewCIDR(*seedNetworks.Nodes, fldPath.Child("nodes"))) + nodes = cidrvalidation.NewCIDR(*seedNetworks.Nodes, fldPath.Child("nodes")) + allCIDRs = append(allCIDRs, nodes) + seedCIDRs = append(seedCIDRs, nodes) } if shootDefaults := seedNetworks.ShootDefaults; shootDefaults != nil { if shootDefaults.Pods != nil { - networks = append(networks, cidrvalidation.NewCIDR(*shootDefaults.Pods, fldPath.Child("shootDefaults", "pods"))) + shootPods = cidrvalidation.NewCIDR(*shootDefaults.Pods, fldPath.Child("shootDefaults", "pods")) + allCIDRs = append(allCIDRs, shootPods) + if shootPods.IsIPv4() { + shootDefaultsV4 = append(shootDefaultsV4, shootPods) + } else { + shootDefaultsV6 = append(shootDefaultsV6, shootPods) + } } if shootDefaults.Services != nil { - networks = append(networks, cidrvalidation.NewCIDR(*shootDefaults.Services, fldPath.Child("shootDefaults", "services"))) + shootServices = cidrvalidation.NewCIDR(*shootDefaults.Services, fldPath.Child("shootDefaults", "services")) + allCIDRs = append(allCIDRs, shootServices) + if shootServices.IsIPv4() { + shootDefaultsV4 = append(shootDefaultsV4, shootServices) + } else { + shootDefaultsV6 = append(shootDefaultsV6, shootServices) + } } } - allErrs = append(allErrs, cidrvalidation.ValidateCIDRParse(networks...)...) + // All CIDRs must be valid + allErrs = append(allErrs, cidrvalidation.ValidateCIDRParse(allCIDRs...)...) // Don't check IP family in dualstack case. if len(seedNetworks.IPFamilies) != 2 { - allErrs = append(allErrs, cidrvalidation.ValidateCIDRIPFamily(networks, string(primaryIPFamily))...) + allErrs = append(allErrs, cidrvalidation.ValidateCIDRIPFamily(allCIDRs, string(primaryIPFamily))...) + } + + // In case any IP ranges are incorrect, there is no benefit in checking for intersections. + if len(allErrs) > 0 { + return allErrs + } + + // Seed CIDRs must not overlap with each other + allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlap(seedCIDRs, false)...) + // Shoot default CIDRs must not overlap with each other + allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlap(shootDefaultsV4, false)...) + allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlap(shootDefaultsV6, false)...) + // Shoot default CIDRs (V6) must not overlap with seed CIDRs + for _, seedCIDR := range seedCIDRs { + allErrs = append(allErrs, seedCIDR.ValidateNotOverlap(shootDefaultsV6...)...) } - allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlap(networks, false)...) - allErrs = append(allErrs, reservedSeedServiceRange.ValidateNotOverlap(networks...)...) - vpnRangeV6 := cidrvalidation.NewCIDR(v1beta1constants.DefaultVPNRangeV6, field.NewPath("")) - allErrs = append(allErrs, vpnRangeV6.ValidateNotOverlap(networks...)...) + // All CIDRs must not overlap with reserved ranges + allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlapWithReservedRanges(pods, "pod")...) + allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlapWithReservedRanges(services, "service")...) + allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlapWithReservedRanges(nodes, "node")...) + allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlapWithReservedRanges(shootPods, "pod")...) + allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlapWithReservedRanges(shootServices, "service")...) return allErrs } diff --git a/pkg/apis/core/validation/seed_test.go b/pkg/apis/core/validation/seed_test.go index b8ca21de21e..6b7d4ab8d25 100644 --- a/pkg/apis/core/validation/seed_test.go +++ b/pkg/apis/core/validation/seed_test.go @@ -420,10 +420,10 @@ var _ = Describe("Seed Validation Tests", func() { Namespace: "garden", }, } - new := seed.DeepCopy() - new.Spec.DNS.Internal = nil + newSeed := seed.DeepCopy() + newSeed.Spec.DNS.Internal = nil - errorList := ValidateSeedUpdate(new, seed) + errorList := ValidateSeedUpdate(newSeed, seed) Expect(errorList).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.dns.internal"), @@ -652,10 +652,10 @@ var _ = Describe("Seed Validation Tests", func() { }, }, } - new := seed.DeepCopy() - new.Spec.DNS.Defaults = nil + newSeed := seed.DeepCopy() + newSeed.Spec.DNS.Defaults = nil - errorList := ValidateSeedUpdate(new, seed) + errorList := ValidateSeedUpdate(newSeed, seed) Expect(errorList).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.dns.defaults"), @@ -819,26 +819,27 @@ var _ = Describe("Seed Validation Tests", func() { })) }) - It("should forbid Seed with overlapping networks", func() { - shootDefaultPodCIDR := "10.0.1.128/28" // 10.0.1.128 -> 10.0.1.13 - shootDefaultServiceCIDR := "10.0.1.144/30" // 10.0.1.144 -> 10.0.1.17 + It("should forbid Seed with overlapping networks but allow overlap with shoot defaults", func() { + shootDefaultPodCIDR := "10.0.1.128/28" // 10.0.1.128 -> 10.0.1.143 + shootDefaultServiceCIDR := "10.0.1.144/30" // 10.0.1.144 -> 10.0.1.147 - nodesCIDR := "10.0.0.0/8" // 10.0.0.0 -> 10.255.255.25 + nodesCIDR := "10.0.0.0/8" // 10.0.0.0 -> 10.255.255.255 // Pods CIDR overlaps with Nodes network // Services CIDR overlaps with Nodes and Pods // Shoot default pod CIDR overlaps with services // Shoot default pod CIDR overlaps with shoot default pod CIDR seed.Spec.Networks = core.SeedNetworks{ Nodes: &nodesCIDR, // 10.0.0.0 -> 10.255.255.25 - Pods: "10.0.1.0/24", // 10.0.1.0 -> 10.0.1.25 - Services: "10.0.1.64/26", // 10.0.1.64 -> 10.0.1.17 + Pods: "10.0.1.0/24", // 10.0.1.0 -> 10.0.1.255 + Services: "10.0.1.64/26", // 10.0.1.64 -> 10.0.1.127 ShootDefaults: &core.ShootNetworks{ Pods: &shootDefaultPodCIDR, Services: &shootDefaultServiceCIDR, }, } - Expect(ValidateSeed(seed)).To(ConsistOfFields(Fields{ + errorList := ValidateSeed(seed) + Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.networks.nodes"), "BadValue": Equal("10.0.0.0/8"), @@ -849,36 +850,64 @@ var _ = Describe("Seed Validation Tests", func() { "BadValue": Equal("10.0.0.0/8"), "Detail": Equal("must not overlap with \"spec.networks.services\" (\"10.0.1.64/26\")"), }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.services"), + "Detail": Equal(`must not overlap with "spec.networks.pods" ("10.0.1.0/24")`), + })) + + Expect(errorList).To(Not(ContainElement(HaveFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.networks.shootDefaults.pods"), "Detail": Equal(`must not overlap with "spec.networks.nodes" ("10.0.0.0/8")`), - }, Fields{ + })))) + + Expect(errorList).To(Not(ContainElement(HaveFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.networks.shootDefaults.services"), "Detail": Equal(`must not overlap with "spec.networks.nodes" ("10.0.0.0/8")`), - }, Fields{ - "Type": Equal(field.ErrorTypeInvalid), - "Field": Equal("spec.networks.services"), - "Detail": Equal(`must not overlap with "spec.networks.pods" ("10.0.1.0/24")`), - }, Fields{ + })))) + + Expect(errorList).To(Not(ContainElement(HaveFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.networks.shootDefaults.pods"), "Detail": Equal(`must not overlap with "spec.networks.pods" ("10.0.1.0/24")`), - }, Fields{ + })))) + + Expect(errorList).To(Not(ContainElement(HaveFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.networks.shootDefaults.services"), "Detail": Equal(`must not overlap with "spec.networks.pods" ("10.0.1.0/24")`), - })) + })))) }) - It("should forbid Seed with overlap to reserved range", func() { + It("should forbid Seed with overlap to reserved ranges", func() { // Service CIDR overlaps with reserved range seed.Spec.Networks.Pods = "240.0.0.0/16" // 240.0.0.0 -> 240.0.255.255 + seed.Spec.Networks.Services = "243.1.0.0/16" + seed.Spec.Networks.Nodes = ptr.To("242.2.0.0/16") + seed.Spec.Networks.ShootDefaults.Pods = ptr.To("244.3.0.0/16") + seed.Spec.Networks.ShootDefaults.Services = ptr.To("244.4.0.0/16") Expect(ValidateSeed(seed)).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.networks.pods"), - "Detail": Equal(`must not overlap with "[]" ("240.0.0.0/8")`), + "Detail": Equal(`pod network intersects with reserved kube-apiserver mapping range (240.0.0.0/8)`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.services"), + "Detail": Equal(`service network intersects with reserved shoot service network mapping range (243.0.0.0/8)`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.nodes"), + "Detail": Equal(`node network intersects with reserved shoot node network mapping range (242.0.0.0/8)`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.shootDefaults.pods"), + "Detail": Equal(`pod network intersects with reserved shoot pod network mapping range (244.0.0.0/8)`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.shootDefaults.services"), + "Detail": Equal(`service network intersects with reserved shoot pod network mapping range (244.0.0.0/8)`), })) }) @@ -941,6 +970,56 @@ var _ = Describe("Seed Validation Tests", func() { })) }) + It("should forbid Seed with overlapping networks", func() { + seed.Spec.Networks.Nodes = ptr.To("2001:db8:11::/48") + seed.Spec.Networks.Pods = "2001:db8:11:1::/49" + seed.Spec.Networks.Services = "2001:db8:11:2::/49" + seed.Spec.Networks.ShootDefaults.Pods = ptr.To("2001:db8:11:a::/64") + seed.Spec.Networks.ShootDefaults.Services = ptr.To("2001:db8:11:b::/64") + + errorList := ValidateSeed(seed) + Expect(errorList).To(ConsistOfFields(Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.nodes"), + "BadValue": Equal("2001:db8:11::/48"), + "Detail": Equal(`must not overlap with "spec.networks.pods" ("2001:db8:11:1::/49")`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.nodes"), + "BadValue": Equal("2001:db8:11::/48"), + "Detail": Equal(`must not overlap with "spec.networks.services" ("2001:db8:11:2::/49")`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.services"), + "BadValue": Equal("2001:db8:11:2::/49"), + "Detail": Equal(`must not overlap with "spec.networks.pods" ("2001:db8:11:1::/49")`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.shootDefaults.pods"), + "Detail": Equal(`must not overlap with "spec.networks.pods" ("2001:db8:11:1::/49")`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.shootDefaults.pods"), + "Detail": Equal(`must not overlap with "spec.networks.nodes" ("2001:db8:11::/48")`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.shootDefaults.pods"), + "Detail": Equal(`must not overlap with "spec.networks.services" ("2001:db8:11:2::/49")`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.shootDefaults.services"), + "Detail": Equal(`must not overlap with "spec.networks.nodes" ("2001:db8:11::/48")`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.shootDefaults.services"), + "Detail": Equal(`must not overlap with "spec.networks.pods" ("2001:db8:11:1::/49")`), + }, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("spec.networks.shootDefaults.services"), + "Detail": Equal(`must not overlap with "spec.networks.services" ("2001:db8:11:2::/49")`), + })) + }) + It("should forbid IPv4 CIDRs with IPv6 IP family", func() { seed.Spec.Networks.Nodes = ptr.To("10.1.0.0/16") seed.Spec.Networks.Pods = "10.2.0.0/16" diff --git a/pkg/utils/validation/cidr/disjoint.go b/pkg/utils/validation/cidr/disjoint.go index e0eab72daca..477686935cc 100644 --- a/pkg/utils/validation/cidr/disjoint.go +++ b/pkg/utils/validation/cidr/disjoint.go @@ -60,33 +60,51 @@ func validateOverlapWithSeed(fldPath *field.Path, shootNetwork []string, network allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("shoot %s network intersects with seed node network", networkType))) } } + allErrs = append(allErrs, validateOverlapWithReservedRanges(fldPath, network, networkType)...) + } + if len(shootNetwork) == 0 && networkRequired { + allErrs = append(allErrs, field.Required(fldPath, networkType+"s is required")) + } - if NetworksIntersect(v1beta1constants.DefaultVPNRangeV6, network) { - allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("shoot %s network intersects with default vpn network (%s)", networkType, v1beta1constants.DefaultVPNRangeV6))) - } + return allErrs +} - if NetworksIntersect(v1beta1constants.ReservedKubeApiServerMappingRange, network) { - allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("shoot %s network intersects with reserved kube-apiserver mapping range (%s)", networkType, v1beta1constants.ReservedKubeApiServerMappingRange))) - } +// ValidateCIDROverlapWithReservedRanges validates that the given CIDR does not overlap with reserved ranges. +func ValidateCIDROverlapWithReservedRanges(cidr CIDR, networkType string) field.ErrorList { + allErrs := field.ErrorList{} - if NetworksIntersect(v1beta1constants.ReservedSeedPodNetworkMappedRange, network) { - allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("shoot %s network intersects with reserved seed pod network mapping range (%s)", networkType, v1beta1constants.ReservedSeedPodNetworkMappedRange))) - } + if cidr == nil { + return allErrs + } + allErrs = append(allErrs, validateOverlapWithReservedRanges(cidr.GetFieldPath(), cidr.GetCIDR(), networkType)...) + return allErrs +} - if NetworksIntersect(v1beta1constants.ReservedShootNodeNetworkMappedRange, network) { - allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("shoot %s network intersects with reserved shoot node network mapping range (%s)", networkType, v1beta1constants.ReservedShootNodeNetworkMappedRange))) - } +func validateOverlapWithReservedRanges(fldPath *field.Path, network, networkType string) field.ErrorList { + allErrs := field.ErrorList{} - if NetworksIntersect(v1beta1constants.ReservedShootServiceNetworkMappedRange, network) { - allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("shoot %s network intersects with reserved shoot service network mapping range (%s)", networkType, v1beta1constants.ReservedShootServiceNetworkMappedRange))) - } + if NetworksIntersect(v1beta1constants.DefaultVPNRangeV6, network) { + allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("%s network intersects with default vpn network (%s)", networkType, v1beta1constants.DefaultVPNRangeV6))) + } - if NetworksIntersect(v1beta1constants.ReservedShootPodNetworkMappedRange, network) { - allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("shoot %s network intersects with reserved shoot pod network mapping range (%s)", networkType, v1beta1constants.ReservedShootPodNetworkMappedRange))) - } + if NetworksIntersect(v1beta1constants.ReservedKubeApiServerMappingRange, network) { + allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("%s network intersects with reserved kube-apiserver mapping range (%s)", networkType, v1beta1constants.ReservedKubeApiServerMappingRange))) } - if len(shootNetwork) == 0 && networkRequired { - allErrs = append(allErrs, field.Required(fldPath, networkType+"s is required")) + + if NetworksIntersect(v1beta1constants.ReservedSeedPodNetworkMappedRange, network) { + allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("%s network intersects with reserved seed pod network mapping range (%s)", networkType, v1beta1constants.ReservedSeedPodNetworkMappedRange))) + } + + if NetworksIntersect(v1beta1constants.ReservedShootNodeNetworkMappedRange, network) { + allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("%s network intersects with reserved shoot node network mapping range (%s)", networkType, v1beta1constants.ReservedShootNodeNetworkMappedRange))) + } + + if NetworksIntersect(v1beta1constants.ReservedShootServiceNetworkMappedRange, network) { + allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("%s network intersects with reserved shoot service network mapping range (%s)", networkType, v1beta1constants.ReservedShootServiceNetworkMappedRange))) + } + + if NetworksIntersect(v1beta1constants.ReservedShootPodNetworkMappedRange, network) { + allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("%s network intersects with reserved shoot pod network mapping range (%s)", networkType, v1beta1constants.ReservedShootPodNetworkMappedRange))) } return allErrs diff --git a/pkg/utils/validation/cidr/disjoint_test.go b/pkg/utils/validation/cidr/disjoint_test.go index d2141fbf619..ad6e599451e 100644 --- a/pkg/utils/validation/cidr/disjoint_test.go +++ b/pkg/utils/validation/cidr/disjoint_test.go @@ -188,17 +188,17 @@ var _ = Describe("utils", func() { PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("[].nodes"), - "Detail": ContainSubstring("shoot node network intersects with reserved shoot service network mapping range (243.0.0.0/8)"), + "Detail": ContainSubstring("node network intersects with reserved shoot service network mapping range (243.0.0.0/8)"), })), PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("[].services"), - "Detail": ContainSubstring("shoot service network intersects with reserved shoot node network mapping range (242.0.0.0/8)"), + "Detail": ContainSubstring("service network intersects with reserved shoot node network mapping range (242.0.0.0/8)"), })), PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("[].pods"), - "Detail": ContainSubstring("shoot pod network intersects with reserved seed pod network mapping range (241.0.0.0/8)"), + "Detail": ContainSubstring("pod network intersects with reserved seed pod network mapping range (241.0.0.0/8)"), })), )) }) @@ -224,7 +224,7 @@ var _ = Describe("utils", func() { PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("[].pods"), - "Detail": ContainSubstring("shoot pod network intersects with reserved shoot pod network mapping range (244.0.0.0/8)"), + "Detail": ContainSubstring("pod network intersects with reserved shoot pod network mapping range (244.0.0.0/8)"), })), )) }) From 931b14a03a99f3b7a9d17c12a2e3c51679463314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20G=C3=B6tz?= <47362717+oliver-goetz@users.noreply.github.com> Date: Tue, 11 Nov 2025 19:06:07 +0100 Subject: [PATCH 083/176] Deactivate `UnauthenticatedHTTP2DOSMitigation` for kube-apiservers where `IstioTLSTermination` is active (#13405) * Deactivate `UnauthenticatedHTTP2DOSMitigation` feature gate for kube-apiservers where IstioTLSTermination is active * Tighten `overload.premature_reset_total_stream_count` for istio-ingressgateway More cautious mitigation of HTTP/2 "Rapid Reset" DoS Vulnerability than default. --- .../istio-ingress/templates/deployment.yaml | 4 + .../istio/test_charts/ingress_deployment.yaml | 4 + pkg/component/shared/kubeapiserver.go | 15 ++++ pkg/component/shared/kubeapiserver_test.go | 79 ++++++++++++------- .../operation/botanist/kubeapiserver.go | 1 + .../controller/garden/garden/components.go | 1 + 6 files changed, 74 insertions(+), 30 deletions(-) diff --git a/pkg/component/networking/istio/charts/istio/istio-ingress/templates/deployment.yaml b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/deployment.yaml index 94362bf9a1b..9d709f7b21b 100644 --- a/pkg/component/networking/istio/charts/istio/istio-ingress/templates/deployment.yaml +++ b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/deployment.yaml @@ -34,6 +34,10 @@ spec: protocolDetectionTimeout: 100ms runtimeValues: "overload.global_downstream_max_connections": "750000" + # Limits for mitigating HTTP/2 "Rapid Reset" DoS Vulnerability + # See also https://github.com/envoyproxy/envoy/security/advisories/GHSA-jhv4-f7mr-xx76 + "overload.premature_reset_total_stream_count": "100" + "overload.premature_reset_min_stream_lifetime_seconds": "1" spec: serviceAccountName: istio-ingressgateway-service-account securityContext: diff --git a/pkg/component/networking/istio/test_charts/ingress_deployment.yaml b/pkg/component/networking/istio/test_charts/ingress_deployment.yaml index 1aab7cb9ca9..ce6a360a539 100644 --- a/pkg/component/networking/istio/test_charts/ingress_deployment.yaml +++ b/pkg/component/networking/istio/test_charts/ingress_deployment.yaml @@ -33,6 +33,10 @@ spec: protocolDetectionTimeout: 100ms runtimeValues: "overload.global_downstream_max_connections": "750000" + # Limits for mitigating HTTP/2 "Rapid Reset" DoS Vulnerability + # See also https://github.com/envoyproxy/envoy/security/advisories/GHSA-jhv4-f7mr-xx76 + "overload.premature_reset_total_stream_count": "100" + "overload.premature_reset_min_stream_lifetime_seconds": "1" spec: serviceAccountName: istio-ingressgateway-service-account securityContext: diff --git a/pkg/component/shared/kubeapiserver.go b/pkg/component/shared/kubeapiserver.go index e28a9db70f1..d18ddb9ac74 100644 --- a/pkg/component/shared/kubeapiserver.go +++ b/pkg/component/shared/kubeapiserver.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer/json" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/features" admissionapiv1 "k8s.io/pod-security-admission/admission/api/v1" admissionapiv1alpha1 "k8s.io/pod-security-admission/admission/api/v1alpha1" admissionapiv1beta1 "k8s.io/pod-security-admission/admission/api/v1beta1" @@ -82,6 +83,7 @@ func NewKubeAPIServer( priorityClassName string, isWorkerless bool, runsAsStaticPod bool, + istioTLSTerminationEnabled bool, auditWebhookConfig *apiserver.AuditWebhook, authenticationWebhookConfig *kubeapiserver.AuthenticationWebhook, authorizationWebhookConfigs []kubeapiserver.AuthorizationWebhook, @@ -160,6 +162,19 @@ func NewKubeAPIServer( enableAnonymousAuthentication = apiServerConfig.EnableAnonymousAuthentication } + // UnauthenticatedHTTP2DOSMitigation feature gate must be disabled when IstioTLSTermination is enabled. + // The mitigation which is implemented in kube-apiserver does not affect the connection from client but only between + // istio ingress gateway and kube-apiserver. Therefore, it does not provide any protection but can cause quite some + // issues like dropped connections. It is not recommended to have this when using L7 load-balancing. + // More details: https://github.com/kubernetes/kubernetes/pull/121120 + // With IstioTLSTermination enabled, HTTP/2 "Rapid Reset" DoS Vulnerability is mitigated by Envoy measurements. + if istioTLSTerminationEnabled { + if featureGates == nil { + featureGates = make(map[string]bool) + } + featureGates[string(features.UnauthenticatedHTTP2DOSMitigation)] = false + } + enabledAdmissionPluginConfigs, err := convertToAdmissionPluginConfigs(ctx, resourceConfigClient, objectMeta.Namespace, enabledAdmissionPlugins) if err != nil { return nil, err diff --git a/pkg/component/shared/kubeapiserver_test.go b/pkg/component/shared/kubeapiserver_test.go index f48301b902e..46a3fcc3cb9 100644 --- a/pkg/component/shared/kubeapiserver_test.go +++ b/pkg/component/shared/kubeapiserver_test.go @@ -79,6 +79,7 @@ var _ = Describe("KubeAPIServer", func() { authenticationWebhookConfig *kubeapiserver.AuthenticationWebhook authorizationWebhookConfigs []kubeapiserver.AuthorizationWebhook resourcesToStoreInETCDEvents []schema.GroupResource + istioTLSTerminationEnabled bool runtimeClientSet kubernetes.Interface resourceConfigClient client.Client @@ -97,6 +98,7 @@ var _ = Describe("KubeAPIServer", func() { isWorkerless = false runsAsStaticPod = false auditWebhookConfig = nil + istioTLSTerminationEnabled = false authenticationWebhookConfig = &kubeapiserver.AuthenticationWebhook{Version: ptr.To("authn-version")} authorizationWebhookConfigs = []kubeapiserver.AuthorizationWebhook{{Name: "custom", Kubeconfig: []byte("bar"), WebhookConfiguration: apiserverv1beta1.WebhookConfiguration{FailurePolicy: "Fail"}}} resourcesToStoreInETCDEvents = []schema.GroupResource{{Resource: "foo", Group: "bar"}} @@ -116,7 +118,7 @@ var _ = Describe("KubeAPIServer", func() { Describe("AnonymousAuthenticationEnabled", func() { It("should not set the field by default", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().AnonymousAuthenticationEnabled).To(BeNil()) }) @@ -124,7 +126,7 @@ var _ = Describe("KubeAPIServer", func() { It("should set the field to true if explicitly enabled", func() { apiServerConfig = &gardencorev1beta1.KubeAPIServerConfig{EnableAnonymousAuthentication: ptr.To(true)} - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().AnonymousAuthenticationEnabled).To(PointTo(BeTrue())) }) @@ -132,7 +134,7 @@ var _ = Describe("KubeAPIServer", func() { Describe("APIAudiences", func() { It("should set the field to 'kubernetes' and 'gardener' by default", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().APIAudiences).To(ConsistOf("kubernetes", "gardener")) }) @@ -141,7 +143,7 @@ var _ = Describe("KubeAPIServer", func() { apiAudiences := []string{"foo", "bar"} apiServerConfig = &gardencorev1beta1.KubeAPIServerConfig{APIAudiences: apiAudiences} - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().APIAudiences).To(Equal(append(apiAudiences, "gardener"))) }) @@ -150,10 +152,27 @@ var _ = Describe("KubeAPIServer", func() { apiAudiences := []string{"foo", "bar", "gardener"} apiServerConfig = &gardencorev1beta1.KubeAPIServerConfig{APIAudiences: apiAudiences} - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().APIAudiences).To(Equal(apiAudiences)) }) + + It("should deactivate UnauthenticatedHTTP2DOSMitigation feature gate when IstioTLSTermination is active", func() { + istioTLSTerminationEnabled = true + + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + Expect(err).NotTo(HaveOccurred()) + Expect(kubeAPIServer.GetValues().FeatureGates).To(HaveKeyWithValue("UnauthenticatedHTTP2DOSMitigation", false)) + }) + + It("should deactivate UnauthenticatedHTTP2DOSMitigation feature gate when IstioTLSTermination is active and it is manually activated", func() { + istioTLSTerminationEnabled = true + apiServerConfig = &gardencorev1beta1.KubeAPIServerConfig{KubernetesConfig: gardencorev1beta1.KubernetesConfig{FeatureGates: map[string]bool{"UnauthenticatedHTTP2DOSMitigation": true}}} + + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + Expect(err).NotTo(HaveOccurred()) + Expect(kubeAPIServer.GetValues().FeatureGates).To(HaveKeyWithValue("UnauthenticatedHTTP2DOSMitigation", false)) + }) }) Describe("AdmissionPlugins", func() { @@ -166,7 +185,7 @@ var _ = Describe("KubeAPIServer", func() { func(configuredPlugins []gardencorev1beta1.AdmissionPlugin, expectedPlugins []apiserver.AdmissionPluginConfig, isWorkerless bool) { apiServerConfig.AdmissionPlugins = configuredPlugins - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().EnabledAdmissionPlugins).To(Equal(expectedPlugins)) }, @@ -235,7 +254,7 @@ var _ = Describe("KubeAPIServer", func() { var expectedDisabledPlugins []gardencorev1beta1.AdmissionPlugin AfterEach(func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().DisabledAdmissionPlugins).To(Equal(expectedDisabledPlugins)) }) @@ -310,7 +329,7 @@ var _ = Describe("KubeAPIServer", func() { codec = serializer.NewCodecFactory(runtimeScheme).CodecForVersions(ser, ser, versions, versions) configData = nil - kubeAPIServer, err = NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err = NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) }) Context("When the config is nil", func() { @@ -522,7 +541,7 @@ exemptions: prepTest() } - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).To(errMatcher) if kubeAPIServer != nil { Expect(kubeAPIServer.GetValues().Audit).To(Equal(expectedConfig)) @@ -675,7 +694,7 @@ exemptions: prepTest() } - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).To(errMatcher) if kubeAPIServer != nil { Expect(kubeAPIServer.GetValues().AuthenticationConfiguration).To(Equal(expectedConfig)) @@ -793,7 +812,7 @@ authorizers: prepTest() } - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).To(errMatcher) if kubeAPIServer != nil { Expect(kubeAPIServer.GetValues().AuthorizationWebhooks).To(Equal(expectedWebhooks)) @@ -947,7 +966,7 @@ authorizers: Describe("DefaultNotReadyTolerationSeconds and DefaultUnreachableTolerationSeconds", func() { It("should not set the fields", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().DefaultNotReadyTolerationSeconds).To(BeNil()) Expect(kubeAPIServer.GetValues().DefaultUnreachableTolerationSeconds).To(BeNil()) @@ -959,7 +978,7 @@ authorizers: DefaultUnreachableTolerationSeconds: ptr.To[int64](130), } - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().DefaultNotReadyTolerationSeconds).To(PointTo(Equal(int64(120)))) Expect(kubeAPIServer.GetValues().DefaultUnreachableTolerationSeconds).To(PointTo(Equal(int64(130)))) @@ -968,7 +987,7 @@ authorizers: Describe("EventTTL", func() { It("should not set the event ttl field", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().EventTTL).To(BeNil()) }) @@ -980,7 +999,7 @@ authorizers: EventTTL: eventTTL, } - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().EventTTL).To(Equal(eventTTL)) }) @@ -988,7 +1007,7 @@ authorizers: Describe("FeatureGates", func() { It("should set the field to nil by default", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().FeatureGates).To(BeNil()) }) @@ -1002,7 +1021,7 @@ authorizers: }, } - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().FeatureGates).To(Equal(featureGates)) }) @@ -1015,7 +1034,7 @@ authorizers: prepTest() } - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().OIDC).To(Equal(expectedConfig)) }, @@ -1041,7 +1060,7 @@ authorizers: Describe("Requests", func() { It("should set the field to nil by default", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().Requests).To(BeNil()) }) @@ -1053,7 +1072,7 @@ authorizers: } apiServerConfig = &gardencorev1beta1.KubeAPIServerConfig{Requests: requests} - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().Requests).To(Equal(requests)) }) @@ -1061,7 +1080,7 @@ authorizers: Describe("RuntimeConfig", func() { It("should set the field to nil by default", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().RuntimeConfig).To(BeNil()) }) @@ -1070,7 +1089,7 @@ authorizers: runtimeConfig := map[string]bool{"foo": true, "bar": false} apiServerConfig = &gardencorev1beta1.KubeAPIServerConfig{RuntimeConfig: runtimeConfig} - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().RuntimeConfig).To(Equal(runtimeConfig)) }) @@ -1080,7 +1099,7 @@ authorizers: It("should set the field to the configured values", func() { vpnConfig = kubeapiserver.VPNConfig{Enabled: true} - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().VPN).To(Equal(vpnConfig)) }) @@ -1088,7 +1107,7 @@ authorizers: Describe("WatchCacheSizes", func() { It("should set the field to nil by default", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().WatchCacheSizes).To(BeNil()) }) @@ -1100,7 +1119,7 @@ authorizers: } apiServerConfig = &gardencorev1beta1.KubeAPIServerConfig{WatchCacheSizes: watchCacheSizes} - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().WatchCacheSizes).To(Equal(watchCacheSizes)) }) @@ -1108,7 +1127,7 @@ authorizers: Describe("PriorityClassName", func() { It("should set the field properly", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().PriorityClassName).To(Equal(priorityClassName)) }) @@ -1116,7 +1135,7 @@ authorizers: Describe("IsWorkerless", func() { It("should set the field properly", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().IsWorkerless).To(Equal(isWorkerless)) }) @@ -1124,7 +1143,7 @@ authorizers: Describe("AuthenticationWebhook", func() { It("should set the field properly", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().AuthenticationWebhook).To(Equal(authenticationWebhookConfig)) }) @@ -1132,7 +1151,7 @@ authorizers: Describe("AuthorizationWebhooks", func() { It("should set the field properly", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().AuthorizationWebhooks).To(Equal(authorizationWebhookConfigs)) }) @@ -1140,7 +1159,7 @@ authorizers: Describe("ResourcesToStoreInETCDEvents", func() { It("should set the field properly", func() { - kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) + kubeAPIServer, err := NewKubeAPIServer(ctx, runtimeClientSet, resourceConfigClient, namespace, objectMeta, runtimeVersion, targetVersion, sm, namePrefix, apiServerConfig, autoscalingConfig, vpnConfig, priorityClassName, isWorkerless, runsAsStaticPod, istioTLSTerminationEnabled, auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, resourcesToStoreInETCDEvents) Expect(err).NotTo(HaveOccurred()) Expect(kubeAPIServer.GetValues().ResourcesToStoreInETCDEvents).To(Equal(resourcesToStoreInETCDEvents)) }) diff --git a/pkg/gardenlet/operation/botanist/kubeapiserver.go b/pkg/gardenlet/operation/botanist/kubeapiserver.go index 6f7b01a43f6..a338810d9dc 100644 --- a/pkg/gardenlet/operation/botanist/kubeapiserver.go +++ b/pkg/gardenlet/operation/botanist/kubeapiserver.go @@ -65,6 +65,7 @@ func (b *Botanist) DefaultKubeAPIServer(ctx context.Context) (kubeapiserver.Inte v1beta1constants.PriorityClassNameShootControlPlane500, b.Shoot.IsWorkerless, b.Shoot.RunsControlPlane(), + b.ShootUsesIstioTLSTermination(), nil, nil, nil, diff --git a/pkg/operator/controller/garden/garden/components.go b/pkg/operator/controller/garden/garden/components.go index 8d63407e946..a4f1fd71e9d 100644 --- a/pkg/operator/controller/garden/garden/components.go +++ b/pkg/operator/controller/garden/garden/components.go @@ -740,6 +740,7 @@ func (r *Reconciler) newKubeAPIServer( v1beta1constants.PriorityClassNameGardenSystem500, true, false, + features.DefaultFeatureGate.Enabled(features.IstioTLSTermination), auditWebhookConfig, authenticationWebhookConfig, authorizationWebhookConfigs, From 58cf7e471bf13bed2b7bb807d908a50511566a8b Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Tue, 11 Nov 2025 19:06:14 +0100 Subject: [PATCH 084/176] Update module github.com/docker/cli to v29 (#13407) * Update module github.com/docker/cli to v29 * Run `make tidy` --------- Co-authored-by: acumino --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 56392687794..9e3ce22e8e6 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/containerd/errdefs v1.0.0 github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 - github.com/docker/cli v28.5.2+incompatible + github.com/docker/cli v29.0.0+incompatible github.com/fluent/fluent-operator/v3 v3.5.0 github.com/gardener/cert-management v0.19.0 github.com/gardener/dependency-watchdog v1.6.0 diff --git a/go.sum b/go.sum index f119bd3bc96..65bb34422df 100644 --- a/go.sum +++ b/go.sum @@ -188,8 +188,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/cli v28.5.2+incompatible h1:XmG99IHcBmIAoC1PPg9eLBZPlTrNUAijsHLm8PjhBlg= -github.com/docker/cli v28.5.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.0.0+incompatible h1:KgsN2RUFMNM8wChxryicn4p46BdQWpXOA1XLGBGPGAw= +github.com/docker/cli v29.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= From 2f17eb54ddd20be1a88ea576171124e0b1aa1a57 Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Tue, 11 Nov 2025 20:54:56 +0100 Subject: [PATCH 085/176] Add IPv6 support for `fluent-bit` (#12003) Enhance `fluent-bit`'s `ConfigMap` to listen on `::`, so both IPv6 and IPv4 are bound. The change is needed for IronCore, having a single-node IPv6 setup --- pkg/component/observability/logging/fluentbit/fluentbit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/component/observability/logging/fluentbit/fluentbit.go b/pkg/component/observability/logging/fluentbit/fluentbit.go index 00d86826d76..4298a497462 100644 --- a/pkg/component/observability/logging/fluentbit/fluentbit.go +++ b/pkg/component/observability/logging/fluentbit/fluentbit.go @@ -509,7 +509,7 @@ func (f *fluentBit) getClusterFluentBitConfig() *fluentbitv1alpha2.ClusterFluent LogLevel: "error", ParsersFile: "parsers.conf", HttpServer: ptr.To(true), - HttpListen: "0.0.0.0", + HttpListen: "::", HttpPort: ptr.To[int32](2020), }, InputSelector: metav1.LabelSelector{ From e6907ca810aadf812cdc6e731f29def2197dfdb3 Mon Sep 17 00:00:00 2001 From: Shafeeque E S Date: Wed, 12 Nov 2025 01:25:04 +0530 Subject: [PATCH 086/176] Use `kubernetes.io/metadata.name` label instead of `gardener.cloud/purpose` in GRM webhook namespace selectors (#13398) --- .../gardener/resourcemanager/resource_manager.go | 2 +- .../resourcemanager/resource_manager_test.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/component/gardener/resourcemanager/resource_manager.go b/pkg/component/gardener/resourcemanager/resource_manager.go index b288b84b835..34cae0dd327 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager.go +++ b/pkg/component/gardener/resourcemanager/resource_manager.go @@ -1990,7 +1990,7 @@ func (r *resourceManager) buildWebhookNamespaceSelector() *metav1.LabelSelector return &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{{ - Key: v1beta1constants.GardenerPurpose, + Key: corev1.LabelMetadataName, Operator: operator, Values: []string{metav1.NamespaceSystem, "kubernetes-dashboard"}, }}, diff --git a/pkg/component/gardener/resourcemanager/resource_manager_test.go b/pkg/component/gardener/resourcemanager/resource_manager_test.go index dcb3e8f16b8..05f18397f2b 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager_test.go +++ b/pkg/component/gardener/resourcemanager/resource_manager_test.go @@ -925,13 +925,13 @@ var _ = Describe("ResourceManager", func() { switch responsibilityMode { case ForSource: namespaceSelectorMatchExpressions = []metav1.LabelSelectorRequirement{{ - Key: "gardener.cloud/purpose", + Key: "kubernetes.io/metadata.name", Operator: metav1.LabelSelectorOpNotIn, Values: []string{"kube-system", "kubernetes-dashboard"}, }} case ForTarget: namespaceSelectorMatchExpressions = []metav1.LabelSelectorRequirement{{ - Key: "gardener.cloud/purpose", + Key: "kubernetes.io/metadata.name", Operator: metav1.LabelSelectorOpIn, Values: []string{"kube-system", "kubernetes-dashboard"}, }} @@ -1323,7 +1323,7 @@ webhooks: name: projected-token-mount.resources.gardener.cloud namespaceSelector: matchExpressions: - - key: gardener.cloud/purpose + - key: kubernetes.io/metadata.name operator: In values: - kube-system @@ -1357,7 +1357,7 @@ webhooks: name: high-availability-config.resources.gardener.cloud namespaceSelector: matchExpressions: - - key: gardener.cloud/purpose + - key: kubernetes.io/metadata.name operator: In values: - kube-system @@ -1513,7 +1513,7 @@ webhooks: name: system-components-config.resources.gardener.cloud namespaceSelector: matchExpressions: - - key: gardener.cloud/purpose + - key: kubernetes.io/metadata.name operator: In values: - kube-system @@ -1547,7 +1547,7 @@ webhooks: name: pod-topology-spread-constraints.resources.gardener.cloud namespaceSelector: matchExpressions: - - key: gardener.cloud/purpose + - key: kubernetes.io/metadata.name operator: In values: - kube-system From 347df6ed91c9c3b532ab37c3021b7805f162a7b3 Mon Sep 17 00:00:00 2001 From: Johannes Scheerer Date: Tue, 11 Nov 2025 20:55:11 +0100 Subject: [PATCH 087/176] Do not set `Cache{Una,A}uthorizedRequests` fields in authorization (#13406) configuration of `kube-apiserver`. These fields are automatically defaulted, but they are not understood by older kubernetes releases as they are not present. Therefore, we need to unset them. Co-authored-by: Aleksandar Savchev --- pkg/component/kubernetes/apiserver/authorization.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/component/kubernetes/apiserver/authorization.go b/pkg/component/kubernetes/apiserver/authorization.go index 44561d61045..33460af7ef5 100644 --- a/pkg/component/kubernetes/apiserver/authorization.go +++ b/pkg/component/kubernetes/apiserver/authorization.go @@ -56,6 +56,12 @@ func (k *kubeAPIServer) reconcileConfigMapAuthorizationConfig(ctx context.Contex } for _, webhook := range k.values.AuthorizationWebhooks { + // For Kubernetes versions < 1.34 the Cache{Una,A}uthorizedRequests fields do not exist. + if versionutils.ConstraintK8sLess134.Check(k.values.Version) { + webhook.CacheAuthorizedRequests = nil + webhook.CacheUnauthorizedRequests = nil + } + config := apiserverv1beta1.AuthorizerConfiguration{ Type: "Webhook", Name: webhook.Name, From 6a6ac663c07f960aaa59a98432457c83e02e3b77 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 12 Nov 2025 00:16:51 +0100 Subject: [PATCH 088/176] Update module golang.org/x/crypto to v0.44.0 (#13422) --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 9e3ce22e8e6..a7ddcf13b4c 100644 --- a/go.mod +++ b/go.mod @@ -52,9 +52,9 @@ require ( go.uber.org/zap v1.27.0 go.yaml.in/yaml/v2 v2.4.3 go.yaml.in/yaml/v4 v4.0.0-rc.2 - golang.org/x/crypto v0.43.0 + golang.org/x/crypto v0.44.0 golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 - golang.org/x/text v0.30.0 + golang.org/x/text v0.31.0 golang.org/x/time v0.14.0 golang.org/x/tools v0.38.0 gomodules.xyz/jsonpatch/v2 v2.5.0 @@ -264,10 +264,10 @@ require ( golang.org/x/mod v0.29.0 // indirect golang.org/x/net v0.46.0 // indirect golang.org/x/oauth2 v0.32.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect - golang.org/x/term v0.36.0 // indirect + golang.org/x/term v0.37.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect google.golang.org/grpc v1.76.0 // indirect diff --git a/go.sum b/go.sum index 65bb34422df..ce1f18c37da 100644 --- a/go.sum +++ b/go.sum @@ -853,8 +853,8 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= -golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -916,8 +916,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -941,18 +941,18 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU= golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= -golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= -golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 7fb185dc55a38b32f1fd4e43048f2bce99acb5ea Mon Sep 17 00:00:00 2001 From: Sebastian Stauch Date: Wed, 12 Nov 2025 09:38:53 +0100 Subject: [PATCH 089/176] Revert "Update dependency kyverno/kyverno to v1.16.0 (#13391)" (#13419) This reverts commit bc3bc162eabac5ff7e4c407abef5a62711cd6b24. --- example/provider-extensions/kyverno/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/provider-extensions/kyverno/kustomization.yaml b/example/provider-extensions/kyverno/kustomization.yaml index fc5276abbce..ad73a020620 100644 --- a/example/provider-extensions/kyverno/kustomization.yaml +++ b/example/provider-extensions/kyverno/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- https://github.com/kyverno/kyverno/releases/download/v1.16.0/install.yaml +- https://github.com/kyverno/kyverno/releases/download/v1.15.2/install.yaml - kyverno-poddisruptionbudget.yaml - kyverno-rbac.yaml From 8227c0838aa2e6c1e7db0f0c0b306e05eb6d7257 Mon Sep 17 00:00:00 2001 From: Ashwani Kumar Date: Wed, 12 Nov 2025 15:46:53 +0530 Subject: [PATCH 090/176] Added set of flags for CA nodeGroupBackoff (#13403) * Added set of flags for nodGroupBackoff * Addressed review comments Co-authored-by: Aaron Francis Fernandes <79958509+aaronfern@users.noreply.github.com> --------- Co-authored-by: Aaron Francis Fernandes <79958509+aaronfern@users.noreply.github.com> --- docs/api-reference/core.md | 42 + example/90-shoot.yaml | 3 + pkg/apis/core/types_shoot.go | 6 + pkg/apis/core/v1beta1/defaults_shoot.go | 9 + pkg/apis/core/v1beta1/defaults_shoot_test.go | 73 +- pkg/apis/core/v1beta1/generated.pb.go | 2008 +++++++++-------- pkg/apis/core/v1beta1/generated.proto | 12 + pkg/apis/core/v1beta1/types_shoot.go | 9 + .../core/v1beta1/zz_generated.conversion.go | 6 + .../core/v1beta1/zz_generated.deepcopy.go | 15 + pkg/apis/core/validation/shoot.go | 4 + pkg/apis/core/validation/shoot_test.go | 30 + pkg/apis/core/zz_generated.deepcopy.go | 15 + pkg/apiserver/openapi/openapi_generated.go | 18 + .../clusterautoscaler/cluster_autoscaler.go | 3 + .../cluster_autoscaler_test.go | 77 +- 16 files changed, 1350 insertions(+), 980 deletions(-) diff --git a/docs/api-reference/core.md b/docs/api-reference/core.md index b670753d337..a3562357d69 100644 --- a/docs/api-reference/core.md +++ b/docs/api-reference/core.md @@ -4071,6 +4071,48 @@ int32 Default: 1

+ + +initialNodeGroupBackoffDuration
+ + +Kubernetes meta/v1.Duration + + + + +(Optional) +

InitialNodeGroupBackoffDuration is the duration of first backoff after a new node failed to start (default: 5m).

+ + + + +maxNodeGroupBackoffDuration
+ + +Kubernetes meta/v1.Duration + + + + +(Optional) +

MaxNodeGroupBackoffDuration is the maximum backoff duration for a NodeGroup after new nodes failed to start (default: 30m).

+ + + + +nodeGroupBackoffResetTimeout
+ + +Kubernetes meta/v1.Duration + + + + +(Optional) +

NodeGroupBackoffResetTimeout is the time after last failed scale-up when the backoff duration is reset (default: 3h).

+ +

ClusterAutoscalerOptions diff --git a/example/90-shoot.yaml b/example/90-shoot.yaml index db868ebb9bb..f7dedaf4294 100644 --- a/example/90-shoot.yaml +++ b/example/90-shoot.yaml @@ -323,6 +323,9 @@ spec: # expander: "priority,least-waste" # see: https://github.com/gardener/autoscaler/blob/machine-controller-manager-provider/cluster-autoscaler/FAQ.md#what-are-expanders # maxGracefulTerminationSeconds: 600 # maxNodeProvisionTime: 20m + # initialNodeGroupBackoffDuration: 5m + # maxNodeGroupBackoffDuration: 30m + # nodeGroupBackoffResetTimeout: 3h # scaleDownUtilizationThreshold: 0.5 # scaleDownUnneededTime: 30m # scaleDownDelayAfterAdd: 1h diff --git a/pkg/apis/core/types_shoot.go b/pkg/apis/core/types_shoot.go index f456c544cf4..9fc077acd0e 100644 --- a/pkg/apis/core/types_shoot.go +++ b/pkg/apis/core/types_shoot.go @@ -548,6 +548,12 @@ type ClusterAutoscaler struct { Expander *ExpanderMode // MaxNodeProvisionTime defines how long CA waits for node to be provisioned (default: 20 mins). MaxNodeProvisionTime *metav1.Duration + // InitialNodeGroupBackoffDuration is the duration of first backoff after a new node failed to start (default: 5m). + InitialNodeGroupBackoffDuration *metav1.Duration + // MaxNodeGroupBackoffDuration is the maximum backoff duration for a NodeGroup after new nodes failed to start (default: 30m). + MaxNodeGroupBackoffDuration *metav1.Duration + // NodeGroupBackoffResetTimeout is the time after last failed scale-up when the backoff duration is reset (default: 3h). + NodeGroupBackoffResetTimeout *metav1.Duration // MaxGracefulTerminationSeconds is the number of seconds CA waits for pod termination when trying to scale down a node (default: 600). MaxGracefulTerminationSeconds *int32 // StartupTaints specifies a list of taint keys to ignore in node templates when considering to scale a node group. diff --git a/pkg/apis/core/v1beta1/defaults_shoot.go b/pkg/apis/core/v1beta1/defaults_shoot.go index cbf818a7d55..711dafad550 100644 --- a/pkg/apis/core/v1beta1/defaults_shoot.go +++ b/pkg/apis/core/v1beta1/defaults_shoot.go @@ -375,6 +375,15 @@ func SetDefaults_ClusterAutoscaler(obj *ClusterAutoscaler) { if obj.MaxDrainParallelism == nil { obj.MaxDrainParallelism = ptr.To[int32](1) } + if obj.InitialNodeGroupBackoffDuration == nil { + obj.InitialNodeGroupBackoffDuration = &metav1.Duration{Duration: 5 * time.Minute} + } + if obj.MaxNodeGroupBackoffDuration == nil { + obj.MaxNodeGroupBackoffDuration = &metav1.Duration{Duration: 30 * time.Minute} + } + if obj.NodeGroupBackoffResetTimeout == nil { + obj.NodeGroupBackoffResetTimeout = &metav1.Duration{Duration: 3 * time.Hour} + } } // SetDefaults_NginxIngress sets default values for NginxIngress objects. diff --git a/pkg/apis/core/v1beta1/defaults_shoot_test.go b/pkg/apis/core/v1beta1/defaults_shoot_test.go index 80ef47bc868..ffab783a347 100644 --- a/pkg/apis/core/v1beta1/defaults_shoot_test.go +++ b/pkg/apis/core/v1beta1/defaults_shoot_test.go @@ -944,25 +944,31 @@ var _ = Describe("Shoot defaulting", func() { Expect(obj.Spec.Kubernetes.ClusterAutoscaler.NewPodScaleUpDelay).To(PointTo(Equal(metav1.Duration{Duration: 0}))) Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxScaleDownParallelism).To(PointTo(Equal(int32(10)))) Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxDrainParallelism).To(PointTo(Equal(int32(1)))) + Expect(obj.Spec.Kubernetes.ClusterAutoscaler.InitialNodeGroupBackoffDuration).To(PointTo(Equal(metav1.Duration{Duration: 5 * time.Minute}))) + Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxNodeGroupBackoffDuration).To(PointTo(Equal(metav1.Duration{Duration: 30 * time.Minute}))) + Expect(obj.Spec.Kubernetes.ClusterAutoscaler.NodeGroupBackoffResetTimeout).To(PointTo(Equal(metav1.Duration{Duration: 3 * time.Hour}))) }) It("should not overwrite the already set values for ClusterAutoscaler field", func() { obj.Spec.Kubernetes.ClusterAutoscaler = &ClusterAutoscaler{ - ScaleDownDelayAfterAdd: &metav1.Duration{Duration: 1 * time.Hour}, - ScaleDownDelayAfterDelete: &metav1.Duration{Duration: 2 * time.Hour}, - ScaleDownDelayAfterFailure: &metav1.Duration{Duration: 3 * time.Hour}, - ScaleDownUnneededTime: &metav1.Duration{Duration: 4 * time.Hour}, - ScaleDownUtilizationThreshold: ptr.To(0.8), - ScanInterval: &metav1.Duration{Duration: 5 * time.Hour}, - Expander: &expanderRandom, - MaxNodeProvisionTime: &metav1.Duration{Duration: 6 * time.Hour}, - MaxGracefulTerminationSeconds: ptr.To(int32(60 * 60 * 24)), - IgnoreDaemonsetsUtilization: ptr.To(true), - Verbosity: ptr.To[int32](4), - NewPodScaleUpDelay: &metav1.Duration{Duration: 1}, - MaxEmptyBulkDelete: ptr.To[int32](20), - MaxScaleDownParallelism: ptr.To[int32](15), - MaxDrainParallelism: ptr.To[int32](5), + ScaleDownDelayAfterAdd: &metav1.Duration{Duration: 1 * time.Hour}, + ScaleDownDelayAfterDelete: &metav1.Duration{Duration: 2 * time.Hour}, + ScaleDownDelayAfterFailure: &metav1.Duration{Duration: 3 * time.Hour}, + ScaleDownUnneededTime: &metav1.Duration{Duration: 4 * time.Hour}, + ScaleDownUtilizationThreshold: ptr.To(0.8), + ScanInterval: &metav1.Duration{Duration: 5 * time.Hour}, + Expander: &expanderRandom, + MaxNodeProvisionTime: &metav1.Duration{Duration: 6 * time.Hour}, + MaxGracefulTerminationSeconds: ptr.To(int32(60 * 60 * 24)), + IgnoreDaemonsetsUtilization: ptr.To(true), + Verbosity: ptr.To[int32](4), + NewPodScaleUpDelay: &metav1.Duration{Duration: 1}, + MaxEmptyBulkDelete: ptr.To[int32](20), + MaxScaleDownParallelism: ptr.To[int32](15), + MaxDrainParallelism: ptr.To[int32](5), + InitialNodeGroupBackoffDuration: &metav1.Duration{Duration: 10 * time.Minute}, + MaxNodeGroupBackoffDuration: &metav1.Duration{Duration: 20 * time.Minute}, + NodeGroupBackoffResetTimeout: &metav1.Duration{Duration: 1 * time.Hour}, } SetObjectDefaults_Shoot(obj) @@ -982,24 +988,30 @@ var _ = Describe("Shoot defaulting", func() { Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxEmptyBulkDelete).To(PointTo(Equal(int32(20)))) Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxScaleDownParallelism).To(PointTo(Equal(int32(15)))) Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxDrainParallelism).To(PointTo(Equal(int32(5)))) + Expect(obj.Spec.Kubernetes.ClusterAutoscaler.InitialNodeGroupBackoffDuration).To(PointTo(Equal(metav1.Duration{Duration: 10 * time.Minute}))) + Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxNodeGroupBackoffDuration).To(PointTo(Equal(metav1.Duration{Duration: 20 * time.Minute}))) + Expect(obj.Spec.Kubernetes.ClusterAutoscaler.NodeGroupBackoffResetTimeout).To(PointTo(Equal(metav1.Duration{Duration: 1 * time.Hour}))) }) It("should sync MaxScaleDownParallelism value with MaxEmptyBulkDelete when the latter is set and the former is not", func() { obj.Spec.Kubernetes.ClusterAutoscaler = &ClusterAutoscaler{ - ScaleDownDelayAfterAdd: &metav1.Duration{Duration: 1 * time.Hour}, - ScaleDownDelayAfterDelete: &metav1.Duration{Duration: 2 * time.Hour}, - ScaleDownDelayAfterFailure: &metav1.Duration{Duration: 3 * time.Hour}, - ScaleDownUnneededTime: &metav1.Duration{Duration: 4 * time.Hour}, - ScaleDownUtilizationThreshold: ptr.To(0.8), - ScanInterval: &metav1.Duration{Duration: 5 * time.Hour}, - Expander: &expanderRandom, - MaxNodeProvisionTime: &metav1.Duration{Duration: 6 * time.Hour}, - MaxGracefulTerminationSeconds: ptr.To(int32(60 * 60 * 24)), - IgnoreDaemonsetsUtilization: ptr.To(true), - Verbosity: ptr.To[int32](4), - NewPodScaleUpDelay: &metav1.Duration{Duration: 1}, - MaxEmptyBulkDelete: ptr.To[int32](17), - MaxDrainParallelism: ptr.To[int32](5), + ScaleDownDelayAfterAdd: &metav1.Duration{Duration: 1 * time.Hour}, + ScaleDownDelayAfterDelete: &metav1.Duration{Duration: 2 * time.Hour}, + ScaleDownDelayAfterFailure: &metav1.Duration{Duration: 3 * time.Hour}, + ScaleDownUnneededTime: &metav1.Duration{Duration: 4 * time.Hour}, + ScaleDownUtilizationThreshold: ptr.To(0.8), + ScanInterval: &metav1.Duration{Duration: 5 * time.Hour}, + Expander: &expanderRandom, + MaxNodeProvisionTime: &metav1.Duration{Duration: 6 * time.Hour}, + MaxGracefulTerminationSeconds: ptr.To(int32(60 * 60 * 24)), + IgnoreDaemonsetsUtilization: ptr.To(true), + Verbosity: ptr.To[int32](4), + NewPodScaleUpDelay: &metav1.Duration{Duration: 1}, + MaxEmptyBulkDelete: ptr.To[int32](17), + MaxDrainParallelism: ptr.To[int32](5), + InitialNodeGroupBackoffDuration: &metav1.Duration{Duration: 10 * time.Minute}, + MaxNodeGroupBackoffDuration: &metav1.Duration{Duration: 20 * time.Minute}, + NodeGroupBackoffResetTimeout: &metav1.Duration{Duration: 1 * time.Hour}, } SetObjectDefaults_Shoot(obj) @@ -1019,6 +1031,9 @@ var _ = Describe("Shoot defaulting", func() { Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxEmptyBulkDelete).To(PointTo(Equal(int32(17)))) Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxScaleDownParallelism).To(PointTo(Equal(int32(17)))) Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxDrainParallelism).To(PointTo(Equal(int32(5)))) + Expect(obj.Spec.Kubernetes.ClusterAutoscaler.InitialNodeGroupBackoffDuration).To(PointTo(Equal(metav1.Duration{Duration: 10 * time.Minute}))) + Expect(obj.Spec.Kubernetes.ClusterAutoscaler.MaxNodeGroupBackoffDuration).To(PointTo(Equal(metav1.Duration{Duration: 20 * time.Minute}))) + Expect(obj.Spec.Kubernetes.ClusterAutoscaler.NodeGroupBackoffResetTimeout).To(PointTo(Equal(metav1.Duration{Duration: 1 * time.Hour}))) }) }) diff --git a/pkg/apis/core/v1beta1/generated.pb.go b/pkg/apis/core/v1beta1/generated.pb.go index 3da202d663a..fcf4807b78b 100644 --- a/pkg/apis/core/v1beta1/generated.pb.go +++ b/pkg/apis/core/v1beta1/generated.pb.go @@ -5969,927 +5969,932 @@ func init() { } var fileDescriptor_ca37af0df9a5bbd2 = []byte{ - // 14705 bytes of a gzipped FileDescriptorProto + // 14794 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x6b, 0x70, 0x24, 0xd9, 0x55, 0x20, 0xec, 0x2c, 0xbd, 0x8f, 0x1e, 0x2d, 0xdd, 0x56, 0x77, 0xab, 0xd5, 0x8f, 0x6a, 0xe7, - 0x8c, 0x4d, 0x0f, 0x63, 0xab, 0x99, 0xf1, 0xd8, 0xe3, 0x19, 0x33, 0x0f, 0xa9, 0x4a, 0xea, 0x2e, - 0xb7, 0xa4, 0xae, 0xb9, 0x25, 0xcd, 0x8c, 0x67, 0x60, 0x4c, 0x2a, 0xeb, 0xaa, 0x94, 0xd3, 0x59, - 0x99, 0x35, 0x99, 0x59, 0x6a, 0xa9, 0xc7, 0xc6, 0x0f, 0x30, 0x9f, 0x6d, 0x0c, 0x1f, 0xf0, 0x01, - 0xfe, 0x6c, 0xf3, 0x05, 0x83, 0x09, 0xcc, 0xb7, 0xcb, 0xae, 0xd9, 0x80, 0x60, 0x23, 0x80, 0x80, - 0x5d, 0x88, 0x60, 0x31, 0x04, 0x26, 0x08, 0x0c, 0xbb, 0x66, 0x01, 0x81, 0xb5, 0x5e, 0xd8, 0x88, - 0x7d, 0xc6, 0x12, 0xb1, 0x44, 0xf4, 0x12, 0xec, 0xc6, 0x7d, 0x64, 0xe6, 0xcd, 0x57, 0xa9, 0x94, - 0x25, 0xc9, 0x33, 0x8b, 0x7f, 0x49, 0x75, 0xcf, 0xbd, 0xe7, 0xdc, 0x57, 0x9e, 0x7b, 0xee, 0xb9, - 0xe7, 0x01, 0x0b, 0x0d, 0xc3, 0xdb, 0x6a, 0x6f, 0xcc, 0xe9, 0x76, 0xf3, 0x5a, 0x43, 0x73, 0xea, - 0xc4, 0x22, 0x4e, 0xf8, 0x4f, 0xeb, 0x76, 0xe3, 0x9a, 0xd6, 0x32, 0xdc, 0x6b, 0xba, 0xed, 0x90, - 0x6b, 0xdb, 0x0f, 0x6d, 0x10, 0x4f, 0x7b, 0xe8, 0x5a, 0x83, 0xc2, 0x34, 0x8f, 0xd4, 0xe7, 0x5a, - 0x8e, 0xed, 0xd9, 0xe8, 0xe1, 0x10, 0xc7, 0x9c, 0xdf, 0x34, 0xfc, 0xa7, 0x75, 0xbb, 0x31, 0x47, - 0x71, 0xcc, 0x51, 0x1c, 0x73, 0x02, 0xc7, 0xec, 0xdb, 0x65, 0xba, 0x76, 0xc3, 0xbe, 0xc6, 0x50, - 0x6d, 0xb4, 0x37, 0xd9, 0x2f, 0xf6, 0x83, 0xfd, 0xc7, 0x49, 0xcc, 0x3e, 0x70, 0xfb, 0xdd, 0xee, - 0x9c, 0x61, 0xd3, 0xce, 0x5c, 0xd3, 0xda, 0x9e, 0xed, 0xea, 0x9a, 0x69, 0x58, 0x8d, 0x6b, 0xdb, - 0x89, 0xde, 0xcc, 0xaa, 0x52, 0x55, 0xd1, 0xed, 0x8e, 0x75, 0x9c, 0x0d, 0x4d, 0x4f, 0xab, 0x73, - 0x23, 0xac, 0x43, 0x76, 0x3c, 0x62, 0xb9, 0x86, 0x6d, 0xb9, 0x6f, 0xa7, 0x23, 0x21, 0xce, 0xb6, - 0x3c, 0x37, 0x91, 0x0a, 0x69, 0x98, 0x1e, 0x09, 0x31, 0x35, 0x35, 0x7d, 0xcb, 0xb0, 0x88, 0xb3, - 0xeb, 0x37, 0xbf, 0xe6, 0x10, 0xd7, 0x6e, 0x3b, 0x3a, 0x39, 0x54, 0x2b, 0xf7, 0x5a, 0x93, 0x78, - 0x5a, 0x1a, 0xad, 0x6b, 0x59, 0xad, 0x9c, 0xb6, 0xe5, 0x19, 0xcd, 0x24, 0x99, 0x77, 0x1d, 0xd4, - 0xc0, 0xd5, 0xb7, 0x48, 0x53, 0x4b, 0xb4, 0x7b, 0x47, 0x56, 0xbb, 0xb6, 0x67, 0x98, 0xd7, 0x0c, - 0xcb, 0x73, 0x3d, 0x27, 0xde, 0x48, 0xfd, 0xa4, 0x02, 0x93, 0xf3, 0xd5, 0x4a, 0x8d, 0xcd, 0xe0, - 0xb2, 0xdd, 0x68, 0x18, 0x56, 0x03, 0x3d, 0x08, 0x23, 0xdb, 0xc4, 0xd9, 0xb0, 0x5d, 0xc3, 0xdb, - 0x9d, 0x51, 0xae, 0x28, 0x57, 0x07, 0x16, 0xc6, 0xf7, 0xf7, 0x8a, 0x23, 0xcf, 0xfa, 0x85, 0x38, - 0x84, 0xa3, 0x0a, 0x9c, 0xde, 0xf2, 0xbc, 0xd6, 0xbc, 0xae, 0x13, 0xd7, 0x0d, 0x6a, 0xcc, 0x14, - 0x58, 0xb3, 0x73, 0xfb, 0x7b, 0xc5, 0xd3, 0x37, 0xd6, 0xd6, 0xaa, 0x31, 0x30, 0x4e, 0x6b, 0xa3, - 0xfe, 0x82, 0x02, 0x53, 0x41, 0x67, 0x30, 0x79, 0xa5, 0x4d, 0x5c, 0xcf, 0x45, 0x18, 0xce, 0x36, - 0xb5, 0x9d, 0x55, 0xdb, 0x5a, 0x69, 0x7b, 0x9a, 0x67, 0x58, 0x8d, 0x8a, 0xb5, 0x69, 0x1a, 0x8d, - 0x2d, 0x4f, 0x74, 0x6d, 0x76, 0x7f, 0xaf, 0x78, 0x76, 0x25, 0xb5, 0x06, 0xce, 0x68, 0x49, 0x3b, - 0xdd, 0xd4, 0x76, 0x12, 0x08, 0xa5, 0x4e, 0xaf, 0x24, 0xc1, 0x38, 0xad, 0x8d, 0xfa, 0x4e, 0x98, - 0xe2, 0xe3, 0xc0, 0xc4, 0xf5, 0x1c, 0x43, 0xf7, 0x0c, 0xdb, 0x42, 0x57, 0xa0, 0xdf, 0xd2, 0x9a, - 0x84, 0xf5, 0x70, 0x64, 0x61, 0xec, 0x4b, 0x7b, 0xc5, 0x37, 0xed, 0xef, 0x15, 0xfb, 0x57, 0xb5, - 0x26, 0xc1, 0x0c, 0xa2, 0xfe, 0x8f, 0x02, 0x5c, 0x4c, 0xb4, 0x7b, 0xce, 0xf0, 0xb6, 0x6e, 0xb5, - 0xe8, 0x7f, 0x2e, 0xfa, 0x41, 0x05, 0xa6, 0xb4, 0x78, 0x05, 0x86, 0x70, 0xf4, 0xe1, 0xc5, 0xb9, - 0xc3, 0x7f, 0xe0, 0x73, 0x09, 0x6a, 0x0b, 0xe7, 0x45, 0xbf, 0x92, 0x03, 0xc0, 0x49, 0xd2, 0xe8, - 0xe3, 0x0a, 0x0c, 0xd9, 0xbc, 0x73, 0x33, 0x85, 0x2b, 0x7d, 0x57, 0x47, 0x1f, 0xfe, 0xce, 0x23, - 0xe9, 0x86, 0x34, 0xe8, 0x39, 0xf1, 0x77, 0xd1, 0xf2, 0x9c, 0xdd, 0x85, 0x53, 0xa2, 0x7b, 0x43, - 0xa2, 0x14, 0xfb, 0xe4, 0x67, 0x1f, 0x87, 0x31, 0xb9, 0x26, 0x9a, 0x84, 0xbe, 0xdb, 0x84, 0x6f, - 0xd5, 0x11, 0x4c, 0xff, 0x45, 0xd3, 0x30, 0xb0, 0xad, 0x99, 0x6d, 0xc2, 0x96, 0x74, 0x04, 0xf3, - 0x1f, 0x8f, 0x17, 0xde, 0xad, 0xa8, 0x0f, 0xc3, 0xc0, 0x7c, 0xbd, 0x6e, 0x5b, 0xe8, 0x01, 0x18, - 0x22, 0x96, 0xb6, 0x61, 0x92, 0x3a, 0x6b, 0x38, 0x1c, 0xd2, 0x5b, 0xe4, 0xc5, 0xd8, 0x87, 0xab, - 0x3f, 0x56, 0x80, 0x41, 0xd6, 0xc8, 0x45, 0x3f, 0xa2, 0xc0, 0xe9, 0xdb, 0xed, 0x0d, 0xe2, 0x58, - 0xc4, 0x23, 0x6e, 0x59, 0x73, 0xb7, 0x36, 0x6c, 0xcd, 0xa9, 0x8b, 0x85, 0xb9, 0x9e, 0x67, 0x46, - 0x6e, 0x26, 0xd1, 0xf1, 0x3d, 0x98, 0x02, 0xc0, 0x69, 0xc4, 0xd1, 0x36, 0x8c, 0x59, 0x0d, 0xc3, - 0xda, 0xa9, 0x58, 0x0d, 0x87, 0xb8, 0x2e, 0x1b, 0xf4, 0xe8, 0xc3, 0x4f, 0xe7, 0xe9, 0xcc, 0xaa, - 0x84, 0x67, 0x61, 0x72, 0x7f, 0xaf, 0x38, 0x26, 0x97, 0xe0, 0x08, 0x1d, 0xf5, 0xef, 0x15, 0x38, - 0x35, 0x5f, 0x6f, 0x1a, 0x2e, 0xe5, 0xb4, 0x55, 0xb3, 0xdd, 0x30, 0xba, 0xd8, 0xfa, 0xe8, 0x19, - 0x18, 0xd4, 0x6d, 0x6b, 0xd3, 0x68, 0x88, 0x7e, 0xbe, 0x7d, 0x8e, 0x73, 0xae, 0x39, 0x99, 0x73, - 0xb1, 0xee, 0x09, 0x8e, 0x37, 0x87, 0xb5, 0x3b, 0x8b, 0x3e, 0x43, 0x5f, 0x80, 0xfd, 0xbd, 0xe2, - 0x60, 0x89, 0x21, 0xc0, 0x02, 0x11, 0xba, 0x0a, 0xc3, 0x75, 0xc3, 0xe5, 0x8b, 0xd9, 0xc7, 0x16, - 0x73, 0x6c, 0x7f, 0xaf, 0x38, 0x5c, 0x16, 0x65, 0x38, 0x80, 0xa2, 0x65, 0x98, 0xa6, 0x33, 0xc8, - 0xdb, 0xd5, 0x88, 0xee, 0x10, 0x8f, 0x76, 0x6d, 0xa6, 0x9f, 0x75, 0x77, 0x66, 0x7f, 0xaf, 0x38, - 0x7d, 0x33, 0x05, 0x8e, 0x53, 0x5b, 0xa9, 0x4b, 0x30, 0x3c, 0x6f, 0x12, 0x87, 0x32, 0x04, 0xf4, - 0x38, 0x4c, 0x90, 0xa6, 0x66, 0x98, 0x98, 0xe8, 0xc4, 0xd8, 0x26, 0x8e, 0x3b, 0xa3, 0x5c, 0xe9, - 0xbb, 0x3a, 0xb2, 0x80, 0xf6, 0xf7, 0x8a, 0x13, 0x8b, 0x11, 0x08, 0x8e, 0xd5, 0x54, 0x3f, 0xa2, - 0xc0, 0xe8, 0x7c, 0xbb, 0x6e, 0x78, 0x7c, 0x5c, 0xc8, 0x81, 0x51, 0x8d, 0xfe, 0xac, 0xda, 0xa6, - 0xa1, 0xef, 0x8a, 0xcd, 0xf5, 0x54, 0xae, 0xcf, 0x2d, 0x44, 0xb3, 0x70, 0x6a, 0x7f, 0xaf, 0x38, - 0x2a, 0x15, 0x60, 0x99, 0x88, 0xba, 0x05, 0x32, 0x0c, 0xbd, 0x0f, 0xc6, 0xf8, 0x70, 0x57, 0xb4, - 0x16, 0x26, 0x9b, 0xa2, 0x0f, 0xf7, 0x49, 0x6b, 0xe5, 0x13, 0x9a, 0xbb, 0xb5, 0xf1, 0x32, 0xd1, - 0x3d, 0x4c, 0x36, 0x89, 0x43, 0x2c, 0x9d, 0xf0, 0x6d, 0x53, 0x92, 0x1a, 0xe3, 0x08, 0x2a, 0xf5, - 0xff, 0x51, 0xe0, 0xd2, 0x7c, 0xdb, 0xdb, 0xb2, 0x1d, 0xe3, 0x2e, 0x71, 0xc2, 0xe9, 0x0e, 0x30, - 0xa0, 0x27, 0x61, 0x42, 0x0b, 0x2a, 0xac, 0x86, 0xdb, 0xe9, 0xac, 0xd8, 0x4e, 0x13, 0xf3, 0x11, - 0x28, 0x8e, 0xd5, 0x46, 0x0f, 0x03, 0xb8, 0xe1, 0xda, 0x32, 0x1e, 0xb0, 0x80, 0x44, 0x5b, 0x90, - 0x56, 0x55, 0xaa, 0xa5, 0xfe, 0x05, 0x3d, 0x0a, 0xb7, 0x35, 0xc3, 0xd4, 0x36, 0x0c, 0xd3, 0xf0, - 0x76, 0x5f, 0xb0, 0x2d, 0xd2, 0xc5, 0x6e, 0x5e, 0x87, 0x73, 0x6d, 0x4b, 0xe3, 0xed, 0x4c, 0xb2, - 0xc2, 0xf7, 0xef, 0xda, 0x6e, 0x8b, 0x70, 0x2e, 0x39, 0xb2, 0x70, 0x61, 0x7f, 0xaf, 0x78, 0x6e, - 0x3d, 0xbd, 0x0a, 0xce, 0x6a, 0x4b, 0x4f, 0x3d, 0x09, 0xf4, 0xac, 0x6d, 0xb6, 0x9b, 0x02, 0x6b, - 0x1f, 0xc3, 0xca, 0x4e, 0xbd, 0xf5, 0xd4, 0x1a, 0x38, 0xa3, 0xa5, 0xfa, 0x85, 0x02, 0x0c, 0x2e, - 0x68, 0xfa, 0xed, 0x76, 0x0b, 0xbd, 0x0d, 0x86, 0x5b, 0x8e, 0xbd, 0x6d, 0xd4, 0x89, 0x23, 0xc6, - 0x36, 0x29, 0xc6, 0x36, 0x5c, 0x15, 0xe5, 0x38, 0xa8, 0x81, 0x0c, 0x98, 0xf0, 0xff, 0x2f, 0xf5, - 0xf0, 0xe5, 0xb2, 0x2f, 0xa1, 0x1a, 0x41, 0x84, 0x63, 0x88, 0x91, 0x0a, 0x83, 0x0e, 0x69, 0xd0, - 0xa3, 0xae, 0x8f, 0x75, 0x8b, 0x7d, 0xed, 0x98, 0x95, 0x60, 0x01, 0x41, 0xef, 0x87, 0x09, 0xdd, - 0x21, 0x75, 0x62, 0x79, 0x86, 0x66, 0xba, 0x74, 0x73, 0x0e, 0x74, 0xbf, 0x39, 0x59, 0x27, 0x4a, - 0x91, 0xe6, 0x38, 0x86, 0x4e, 0xfd, 0x52, 0x01, 0xc6, 0xf8, 0x44, 0x2d, 0xb4, 0xf5, 0xdb, 0xc4, - 0x43, 0xdf, 0x05, 0xc3, 0x54, 0xbe, 0xab, 0x6b, 0x9e, 0x26, 0x3e, 0x84, 0x6f, 0xcb, 0x1c, 0x3a, - 0xfb, 0x06, 0x69, 0xed, 0x90, 0xfa, 0x0a, 0xf1, 0xb4, 0x70, 0xff, 0x85, 0x65, 0x38, 0xc0, 0x8a, - 0x36, 0xa1, 0xdf, 0x6d, 0x11, 0x5d, 0x4c, 0x6c, 0x39, 0xcf, 0xa7, 0x2e, 0xf7, 0xb8, 0xd6, 0x22, - 0x7a, 0xb8, 0x5d, 0xe9, 0x2f, 0xcc, 0xf0, 0x23, 0x0b, 0x06, 0x5d, 0x4f, 0xf3, 0xda, 0x2e, 0x9b, - 0xdf, 0xd1, 0x87, 0x97, 0x7a, 0xa6, 0xc4, 0xb0, 0x2d, 0x4c, 0x08, 0x5a, 0x83, 0xfc, 0x37, 0x16, - 0x54, 0xd4, 0x7f, 0xa3, 0xc0, 0xa4, 0x5c, 0x7d, 0xd9, 0x70, 0x3d, 0xf4, 0x1d, 0x89, 0xe9, 0x9c, - 0xeb, 0x6e, 0x3a, 0x69, 0x6b, 0x36, 0x99, 0xc1, 0x6e, 0xf5, 0x4b, 0xa4, 0xa9, 0x24, 0x30, 0x60, - 0x78, 0xa4, 0xe9, 0x4b, 0x29, 0x4f, 0xf7, 0x3a, 0xc2, 0x85, 0x71, 0x41, 0x6c, 0xa0, 0x42, 0xd1, - 0x62, 0x8e, 0x5d, 0xfd, 0x2e, 0x98, 0x96, 0x6b, 0xf9, 0xfb, 0x9a, 0xb2, 0x0c, 0x6f, 0xb7, 0x95, - 0x60, 0x19, 0xf4, 0x13, 0xc4, 0x0c, 0x82, 0xde, 0x1a, 0xec, 0x71, 0xce, 0x99, 0x82, 0xb9, 0x8b, - 0xee, 0x73, 0xf5, 0xc3, 0x7d, 0xd1, 0xb9, 0xa3, 0xcb, 0x88, 0xb6, 0x63, 0x5f, 0xee, 0xe8, 0xc3, - 0x37, 0x7a, 0x1d, 0xa0, 0xdf, 0xf5, 0xd7, 0x0b, 0x0f, 0xb8, 0x0a, 0xc3, 0x2e, 0x21, 0x75, 0xe9, - 0x5c, 0x66, 0xa7, 0x79, 0x4d, 0x94, 0xe1, 0x00, 0x7a, 0xfc, 0x9c, 0xe0, 0xb5, 0x7e, 0x40, 0xc9, - 0xdd, 0x2e, 0x4f, 0x06, 0x2f, 0x11, 0x4b, 0xd1, 0xcb, 0x64, 0x88, 0x0f, 0x27, 0x86, 0x18, 0xdd, - 0x85, 0x71, 0x53, 0x73, 0xbd, 0x5b, 0x2d, 0x7a, 0x6f, 0xf3, 0xf7, 0xcc, 0xe8, 0xc3, 0xf3, 0x79, - 0x16, 0x7d, 0x59, 0x46, 0xb4, 0x30, 0xb5, 0xbf, 0x57, 0x1c, 0x8f, 0x14, 0xe1, 0x28, 0x29, 0xf4, - 0x32, 0x8c, 0xd0, 0x82, 0x45, 0xc7, 0xb1, 0x1d, 0xc1, 0x2f, 0x9e, 0xc8, 0x4b, 0x97, 0x21, 0xe1, - 0xf7, 0xc8, 0xe0, 0x27, 0x0e, 0xd1, 0xa3, 0xf7, 0x02, 0xb2, 0x37, 0xd8, 0x4d, 0xbe, 0x7e, 0x9d, - 0x5f, 0x52, 0xe9, 0x60, 0xe9, 0xf2, 0xf7, 0x2d, 0xcc, 0x8a, 0x7d, 0x89, 0x6e, 0x25, 0x6a, 0xe0, - 0x94, 0x56, 0xe8, 0x36, 0xa0, 0xe0, 0xa2, 0xcb, 0x4f, 0xfb, 0x03, 0xb6, 0x46, 0x50, 0x49, 0x6c, - 0x8d, 0xb3, 0x94, 0xd8, 0xf5, 0x04, 0x0a, 0x9c, 0x82, 0x56, 0xfd, 0xad, 0x02, 0x8c, 0xf2, 0x2d, - 0xc2, 0x2f, 0x23, 0xc7, 0x7f, 0x56, 0x90, 0xc8, 0x59, 0x51, 0xca, 0xff, 0xf9, 0xb3, 0x0e, 0x67, - 0x1e, 0x15, 0xcd, 0xd8, 0x51, 0xb1, 0xd8, 0x2b, 0xa1, 0xce, 0x27, 0xc5, 0x1f, 0x2b, 0x70, 0x4a, - 0xaa, 0x7d, 0x02, 0x07, 0x45, 0x3d, 0x7a, 0x50, 0x3c, 0xd5, 0xe3, 0xf8, 0x32, 0xce, 0x09, 0x3b, - 0x32, 0x2c, 0xc6, 0xc3, 0x1f, 0x06, 0xd8, 0x60, 0xec, 0x44, 0x12, 0x6d, 0x83, 0x25, 0x5f, 0x08, - 0x20, 0x58, 0xaa, 0x15, 0x61, 0x8a, 0x85, 0x4e, 0x4c, 0x51, 0xfd, 0xf7, 0x7d, 0x30, 0x95, 0x98, - 0xf6, 0x24, 0x1f, 0x51, 0xbe, 0x41, 0x7c, 0xa4, 0xf0, 0x8d, 0xe0, 0x23, 0x7d, 0xb9, 0xf8, 0x48, - 0xf7, 0x07, 0x91, 0x03, 0xa8, 0x69, 0x34, 0x78, 0xb3, 0x9a, 0xa7, 0x39, 0xde, 0x9a, 0xd1, 0x24, - 0x82, 0xe3, 0x7c, 0x6b, 0x77, 0x5b, 0x96, 0xb6, 0xe0, 0x8c, 0x67, 0x25, 0x81, 0x09, 0xa7, 0x60, - 0x57, 0xbf, 0xa7, 0x00, 0x43, 0x0b, 0x9a, 0xcb, 0x7a, 0xfa, 0x41, 0x18, 0x13, 0xa8, 0x2b, 0x4d, - 0xad, 0x41, 0x7a, 0x51, 0x47, 0x08, 0x94, 0x2b, 0x12, 0x3a, 0x7e, 0xa3, 0x93, 0x4b, 0x70, 0x84, - 0x1c, 0xda, 0x85, 0xd1, 0x66, 0x78, 0x7b, 0x11, 0x4b, 0xbc, 0xd4, 0x3b, 0x75, 0x8a, 0x8d, 0x5f, - 0x5b, 0xa5, 0x02, 0x2c, 0xd3, 0x52, 0x5f, 0x82, 0xd3, 0x29, 0x3d, 0xee, 0xe2, 0xe2, 0xf6, 0x16, - 0x18, 0xa2, 0x77, 0xef, 0x50, 0x0c, 0x1b, 0xdd, 0xdf, 0x2b, 0x0e, 0x3d, 0xcb, 0x8b, 0xb0, 0x0f, - 0x53, 0xdf, 0x45, 0x05, 0x80, 0x78, 0x9f, 0xba, 0x51, 0xf0, 0x0d, 0x00, 0x94, 0xe6, 0xb1, 0xed, - 0xf1, 0xad, 0xf4, 0x14, 0x0c, 0xb4, 0xb6, 0x34, 0xd7, 0x6f, 0xf1, 0x80, 0xcf, 0x2a, 0xaa, 0xb4, - 0xf0, 0xde, 0x5e, 0x71, 0x46, 0x16, 0x44, 0x44, 0x23, 0x06, 0xc3, 0xbc, 0x1d, 0xdd, 0x61, 0x74, - 0x93, 0x97, 0xec, 0x66, 0xcb, 0x24, 0x14, 0xca, 0x76, 0x58, 0x21, 0xdf, 0x0e, 0x5b, 0x4e, 0x60, - 0xc2, 0x29, 0xd8, 0x7d, 0x9a, 0x15, 0xcb, 0xf0, 0x0c, 0x2d, 0xa0, 0xd9, 0x97, 0x9f, 0x66, 0x14, - 0x13, 0x4e, 0xc1, 0x8e, 0x3e, 0xa9, 0xc0, 0x6c, 0xb4, 0x78, 0xc9, 0xb0, 0x0c, 0x77, 0x8b, 0xd4, - 0x19, 0xf1, 0xfe, 0x43, 0x13, 0xbf, 0xbc, 0xbf, 0x57, 0x9c, 0x5d, 0xce, 0xc4, 0x88, 0x3b, 0x50, - 0x43, 0x3f, 0xa0, 0xc0, 0x85, 0xd8, 0xbc, 0x38, 0x46, 0xa3, 0x41, 0x1c, 0xd1, 0x9b, 0xc3, 0x7f, - 0xe0, 0xc5, 0xfd, 0xbd, 0xe2, 0x85, 0xe5, 0x6c, 0x94, 0xb8, 0x13, 0x3d, 0xf4, 0x53, 0x0a, 0x9c, - 0x6d, 0x11, 0xab, 0x6e, 0x58, 0x8d, 0xe7, 0x6c, 0xe7, 0x36, 0x71, 0x5c, 0x6c, 0x9b, 0xa6, 0xdd, - 0xf6, 0xdc, 0x99, 0x41, 0x76, 0x86, 0x55, 0xf2, 0x7c, 0x73, 0xd5, 0x34, 0x8c, 0x0b, 0x97, 0xc5, - 0x16, 0x3d, 0x9b, 0x0a, 0x76, 0x71, 0x46, 0x47, 0xd4, 0xdf, 0x54, 0xa0, 0xaf, 0x84, 0x2b, 0xe8, - 0xc1, 0xc8, 0x27, 0x72, 0x4e, 0xfe, 0x44, 0xee, 0xed, 0x15, 0x87, 0x4a, 0xb8, 0x22, 0x7d, 0x8c, - 0x3f, 0xa0, 0xc0, 0x94, 0x6e, 0x5b, 0x9e, 0x46, 0xe7, 0x0e, 0x73, 0x59, 0xd9, 0x3f, 0x97, 0x73, - 0x5d, 0x86, 0x4b, 0x31, 0x64, 0xa1, 0xb2, 0x3b, 0x0e, 0x71, 0x71, 0x92, 0xb2, 0xfa, 0x79, 0x05, - 0xa6, 0x4b, 0x5a, 0x4b, 0xa8, 0x82, 0xca, 0x64, 0xd3, 0xa0, 0x1b, 0xa4, 0x1b, 0xcd, 0x3e, 0xda, - 0x82, 0x41, 0xa6, 0x6d, 0x76, 0x7b, 0xb9, 0xcb, 0x87, 0xb4, 0x9f, 0x65, 0xb8, 0xb8, 0x1e, 0x84, - 0xff, 0x8f, 0x05, 0x7e, 0xf5, 0x09, 0x98, 0x8c, 0xd7, 0x43, 0x45, 0x5f, 0xa6, 0xe1, 0xca, 0xc7, - 0x91, 0xb8, 0x38, 0xf2, 0xf8, 0xf0, 0xff, 0xfb, 0x5a, 0xf1, 0x4d, 0x1f, 0xfe, 0xb3, 0x2b, 0x6f, - 0x52, 0xbf, 0xaa, 0xc0, 0x58, 0xc9, 0xb4, 0xdb, 0xf5, 0xaa, 0x63, 0x6f, 0x1a, 0x26, 0x79, 0x63, - 0x68, 0x39, 0xe4, 0x1e, 0x67, 0x89, 0xae, 0x4c, 0xeb, 0x20, 0x57, 0x7c, 0x83, 0x68, 0x1d, 0xe4, - 0x2e, 0x67, 0x48, 0x93, 0x2f, 0xc2, 0x19, 0xb9, 0x56, 0xa8, 0x32, 0xbd, 0x02, 0xfd, 0xb7, 0x0d, - 0xab, 0x1e, 0xdf, 0x98, 0x37, 0x0d, 0xab, 0x8e, 0x19, 0x24, 0xd8, 0xba, 0x85, 0xcc, 0x33, 0xeb, - 0x6f, 0x46, 0xa2, 0xd3, 0xc6, 0x84, 0xd5, 0xab, 0x30, 0xac, 0x6b, 0x0b, 0x6d, 0xab, 0x6e, 0x06, - 0xbb, 0x9e, 0x4e, 0x41, 0x69, 0x9e, 0x97, 0xe1, 0x00, 0x8a, 0xee, 0x02, 0x84, 0xaf, 0x13, 0xbd, - 0x08, 0x01, 0xe1, 0xc3, 0x47, 0x8d, 0x78, 0x9e, 0x61, 0x35, 0xdc, 0x70, 0x5f, 0x85, 0x30, 0x2c, - 0x51, 0x43, 0x1f, 0x84, 0x71, 0x59, 0x22, 0xe1, 0x6a, 0xd2, 0x9c, 0xcb, 0x10, 0x11, 0x7d, 0xce, - 0x08, 0xc2, 0xe3, 0x72, 0xa9, 0x8b, 0xa3, 0xd4, 0xd0, 0x6e, 0x20, 0x7f, 0x71, 0x25, 0x6d, 0x7f, - 0xfe, 0x1b, 0x85, 0x2c, 0xfa, 0x4c, 0x0b, 0xe2, 0x63, 0x11, 0xa5, 0x71, 0x84, 0x54, 0x8a, 0x62, - 0x66, 0xe0, 0xb8, 0x14, 0x33, 0x04, 0x86, 0xb8, 0x6a, 0xca, 0x3f, 0x6e, 0x1e, 0xcf, 0x33, 0x40, - 0xae, 0xe5, 0x0a, 0x9f, 0xdb, 0xf8, 0x6f, 0x17, 0xfb, 0xb8, 0xd1, 0x36, 0x8c, 0x51, 0xc1, 0xba, - 0x46, 0x4c, 0xa2, 0x7b, 0xb6, 0x33, 0x33, 0x94, 0xff, 0x39, 0xab, 0x26, 0xe1, 0xe1, 0x52, 0xac, - 0x5c, 0x82, 0x23, 0x74, 0x02, 0xcd, 0xdd, 0x70, 0xa6, 0xe6, 0xae, 0x0d, 0xa3, 0xdb, 0x92, 0x2a, - 0x7e, 0x84, 0x4d, 0xc2, 0x93, 0x79, 0x3a, 0x16, 0xea, 0xe5, 0x17, 0x4e, 0x0b, 0x42, 0xa3, 0xb2, - 0x0e, 0x5f, 0xa6, 0x83, 0x36, 0x60, 0x68, 0x83, 0xcb, 0xa0, 0x33, 0xc0, 0xe6, 0xe2, 0x3d, 0x3d, - 0x88, 0xd6, 0x5c, 0xce, 0x15, 0x3f, 0xb0, 0x8f, 0x18, 0xbd, 0x04, 0x83, 0xa6, 0xd1, 0x34, 0x3c, - 0x77, 0x66, 0x94, 0x91, 0xc8, 0xb5, 0xb4, 0xcb, 0x0c, 0x03, 0x3f, 0xac, 0xf8, 0xff, 0x58, 0x60, - 0x45, 0x9f, 0x56, 0xe0, 0xb4, 0xd8, 0x86, 0xc1, 0xa1, 0x65, 0x10, 0x77, 0x66, 0x8c, 0xcd, 0xe1, - 0x8d, 0xde, 0x0e, 0xc9, 0xf0, 0x80, 0x5e, 0xb8, 0x20, 0x66, 0xf3, 0xf4, 0x4a, 0x92, 0x18, 0x4e, - 0xeb, 0x81, 0xfa, 0x2f, 0xc6, 0x61, 0xaa, 0x64, 0xb6, 0x5d, 0x8f, 0x38, 0xf3, 0xc2, 0x92, 0x85, - 0x38, 0xe8, 0xa3, 0x0a, 0x9c, 0x65, 0xff, 0x96, 0xed, 0x3b, 0x56, 0x99, 0x98, 0xda, 0xee, 0xfc, - 0x26, 0xad, 0x51, 0xaf, 0x1f, 0xee, 0xf0, 0x28, 0xb7, 0xc5, 0x35, 0x99, 0xbd, 0xd8, 0xd4, 0x52, - 0x31, 0xe2, 0x0c, 0x4a, 0xe8, 0xfb, 0x15, 0x38, 0x9f, 0x02, 0x2a, 0x13, 0x93, 0x78, 0xbe, 0xf0, - 0x7f, 0xd8, 0x7e, 0x5c, 0xda, 0xdf, 0x2b, 0x9e, 0xaf, 0x65, 0x21, 0xc5, 0xd9, 0xf4, 0xd0, 0x0f, - 0x2a, 0x30, 0x9b, 0x02, 0x5d, 0xd2, 0x0c, 0xb3, 0xed, 0xf8, 0xf7, 0x82, 0xc3, 0x76, 0x87, 0x89, - 0xe7, 0xb5, 0x4c, 0xac, 0xb8, 0x03, 0x45, 0xf4, 0x21, 0x38, 0x13, 0x40, 0xd7, 0x2d, 0x8b, 0x90, - 0x7a, 0xe4, 0x96, 0x70, 0xd8, 0xae, 0x9c, 0xdf, 0xdf, 0x2b, 0x9e, 0xa9, 0xa5, 0x21, 0xc4, 0xe9, - 0x74, 0x50, 0x03, 0x2e, 0x85, 0x00, 0xcf, 0x30, 0x8d, 0xbb, 0xfc, 0x22, 0xb3, 0xe5, 0x10, 0x77, - 0xcb, 0x36, 0xeb, 0x8c, 0x15, 0x2b, 0x0b, 0x6f, 0xde, 0xdf, 0x2b, 0x5e, 0xaa, 0x75, 0xaa, 0x88, - 0x3b, 0xe3, 0x41, 0x75, 0x18, 0x73, 0x75, 0xcd, 0xaa, 0x58, 0x1e, 0x71, 0xb6, 0x35, 0x73, 0x66, - 0x30, 0xd7, 0x00, 0x39, 0x03, 0x94, 0xf0, 0xe0, 0x08, 0x56, 0xf4, 0x6e, 0x18, 0x26, 0x3b, 0x2d, - 0xcd, 0xaa, 0x13, 0xce, 0x74, 0x47, 0x16, 0x2e, 0xd2, 0xa3, 0x7e, 0x51, 0x94, 0xdd, 0xdb, 0x2b, - 0x8e, 0xf9, 0xff, 0xaf, 0xd8, 0x75, 0x82, 0x83, 0xda, 0xe8, 0x03, 0x30, 0xcd, 0x4c, 0x6d, 0xea, - 0x84, 0x1d, 0x21, 0xae, 0x7f, 0x57, 0x1c, 0xce, 0xd5, 0x4f, 0xf6, 0x0c, 0xbf, 0x92, 0x82, 0x0f, - 0xa7, 0x52, 0xa1, 0xcb, 0xd0, 0xd4, 0x76, 0xae, 0x3b, 0x9a, 0x4e, 0x36, 0xdb, 0xe6, 0x1a, 0x71, - 0x9a, 0x86, 0xc5, 0x95, 0x25, 0x44, 0xb7, 0xad, 0x3a, 0x65, 0xd4, 0xca, 0xd5, 0x01, 0xbe, 0x0c, - 0x2b, 0x9d, 0x2a, 0xe2, 0xce, 0x78, 0xd0, 0x23, 0x30, 0x66, 0x34, 0x2c, 0xdb, 0x21, 0x6b, 0x9a, - 0x61, 0x79, 0xee, 0x0c, 0x30, 0x21, 0x9b, 0x4d, 0x6b, 0x45, 0x2a, 0xc7, 0x91, 0x5a, 0x68, 0x1b, - 0x90, 0x45, 0xee, 0x54, 0xed, 0x3a, 0xdb, 0x02, 0xeb, 0x2d, 0xb6, 0x91, 0x05, 0x9b, 0x3d, 0xec, - 0xd4, 0xb0, 0xab, 0xf4, 0x6a, 0x02, 0x1b, 0x4e, 0xa1, 0x80, 0x96, 0x00, 0x35, 0xb5, 0x9d, 0xc5, - 0x66, 0xcb, 0xdb, 0x5d, 0x68, 0x9b, 0xb7, 0x05, 0xd7, 0x18, 0x63, 0x73, 0xc1, 0x15, 0x4d, 0x09, - 0x28, 0x4e, 0x69, 0x81, 0x34, 0xb8, 0xc0, 0xc7, 0x53, 0xd6, 0x48, 0xd3, 0xb6, 0x5c, 0xe2, 0xb9, - 0xd2, 0x26, 0x9d, 0x19, 0x67, 0x06, 0x17, 0xec, 0x62, 0x5b, 0xc9, 0xae, 0x86, 0x3b, 0xe1, 0x88, - 0x9a, 0x9c, 0x4d, 0x1c, 0x60, 0x72, 0xf6, 0x28, 0x8c, 0xbb, 0x9e, 0xe6, 0x78, 0xed, 0x96, 0x58, - 0x86, 0x53, 0x6c, 0x19, 0x98, 0x1e, 0xb2, 0x26, 0x03, 0x70, 0xb4, 0x1e, 0x5d, 0x3e, 0xae, 0x6c, - 0x16, 0xed, 0x26, 0xc3, 0xe5, 0xab, 0x49, 0xe5, 0x38, 0x52, 0x0b, 0xad, 0xc3, 0xb9, 0xa6, 0xb6, - 0x13, 0x7c, 0xbe, 0x55, 0xcd, 0xd1, 0x4c, 0x93, 0x98, 0x86, 0xdb, 0x9c, 0x99, 0x62, 0x3d, 0x65, - 0x2f, 0xfc, 0x2b, 0xe9, 0x55, 0x70, 0x56, 0x5b, 0x61, 0x83, 0x56, 0x76, 0x34, 0x23, 0x82, 0x12, - 0x45, 0x6c, 0xd0, 0xe2, 0x60, 0x9c, 0xd6, 0x46, 0xfd, 0xef, 0xfd, 0x30, 0x93, 0x38, 0xc1, 0x7c, - 0x43, 0xb2, 0x03, 0x79, 0x94, 0x72, 0x44, 0x3c, 0xaa, 0x05, 0x57, 0x82, 0x0a, 0xd7, 0x5b, 0xed, - 0x54, 0x5a, 0x05, 0x46, 0xeb, 0xfe, 0xfd, 0xbd, 0xe2, 0x95, 0xda, 0x01, 0x75, 0xf1, 0x81, 0xd8, - 0xb2, 0xf9, 0x7f, 0xdf, 0x09, 0xf1, 0xff, 0x0f, 0xc0, 0xb4, 0x04, 0x70, 0x88, 0x56, 0xdf, 0xed, - 0xe1, 0xfc, 0x61, 0x6c, 0xaf, 0x96, 0x82, 0x0f, 0xa7, 0x52, 0xc9, 0x64, 0xba, 0x03, 0x27, 0xc1, - 0x74, 0xd5, 0xbd, 0x3e, 0x18, 0x29, 0xd9, 0x56, 0x9d, 0xeb, 0x45, 0x1e, 0x8a, 0xbc, 0x7a, 0x5f, - 0x92, 0x65, 0xe7, 0x7b, 0x7b, 0xc5, 0xf1, 0xa0, 0xa2, 0x24, 0x4c, 0x3f, 0x16, 0xbc, 0x2f, 0xf1, - 0x1b, 0xe9, 0x9b, 0xa3, 0x0f, 0x43, 0xf7, 0xf6, 0x8a, 0xa7, 0x82, 0x66, 0xd1, 0xb7, 0x22, 0xca, - 0x51, 0x4d, 0xcd, 0xf5, 0xd6, 0x1c, 0xcd, 0x72, 0x8d, 0x1e, 0x14, 0x93, 0xc1, 0x83, 0xc0, 0x72, - 0x02, 0x1b, 0x4e, 0xa1, 0x80, 0x5e, 0x86, 0x09, 0x5a, 0xba, 0xde, 0xaa, 0x6b, 0x1e, 0xc9, 0xa9, - 0x8f, 0x0c, 0x6c, 0x98, 0x96, 0x23, 0x98, 0x70, 0x0c, 0x33, 0xb7, 0x12, 0xd0, 0x5c, 0xdb, 0x62, - 0xeb, 0x19, 0xb1, 0x12, 0xa0, 0xa5, 0x58, 0x40, 0xd1, 0x03, 0x30, 0xd4, 0x24, 0xae, 0xab, 0x35, - 0x08, 0x93, 0x0a, 0x46, 0xc2, 0x8b, 0xd5, 0x0a, 0x2f, 0xc6, 0x3e, 0x1c, 0xbd, 0x0d, 0x06, 0x74, - 0xbb, 0x4e, 0xdc, 0x99, 0x21, 0xc6, 0xf8, 0xe8, 0x19, 0x30, 0x50, 0xa2, 0x05, 0xf7, 0xf6, 0x8a, - 0x23, 0xec, 0xf9, 0x84, 0xfe, 0xc2, 0xbc, 0x92, 0xfa, 0x93, 0x0a, 0x4c, 0xc6, 0x95, 0x65, 0x5d, - 0x58, 0x37, 0x9c, 0x9c, 0xa1, 0x80, 0xfa, 0x69, 0x05, 0xc6, 0x68, 0x0f, 0x1d, 0xdb, 0xac, 0x9a, - 0x9a, 0x45, 0xd0, 0xf7, 0x29, 0x30, 0xb9, 0x65, 0x34, 0xb6, 0x64, 0x3b, 0x2e, 0x21, 0xae, 0xe7, - 0x52, 0x36, 0xdd, 0x88, 0xe1, 0x5a, 0x98, 0xde, 0xdf, 0x2b, 0x4e, 0xc6, 0x4b, 0x71, 0x82, 0xa6, - 0xfa, 0x97, 0x05, 0x38, 0x27, 0xf7, 0x6c, 0x3e, 0x34, 0x91, 0x47, 0x7f, 0xac, 0x00, 0x34, 0x0d, - 0x6b, 0xde, 0x34, 0xed, 0x3b, 0xcc, 0xf8, 0x94, 0x5e, 0x80, 0x5e, 0xcc, 0xab, 0xe4, 0x4c, 0xa1, - 0x30, 0xb7, 0x12, 0x60, 0xe7, 0x0f, 0x93, 0xcf, 0xfb, 0xca, 0x93, 0x10, 0x70, 0x6f, 0xaf, 0x58, - 0x4c, 0xda, 0xe5, 0xcf, 0x61, 0x61, 0xfc, 0xbe, 0x6c, 0xb8, 0xde, 0x47, 0xff, 0xa2, 0x63, 0x15, - 0xfe, 0x42, 0x19, 0x0e, 0x64, 0xb6, 0x09, 0xa7, 0x62, 0x84, 0x53, 0x0c, 0x73, 0xcb, 0xb2, 0x61, - 0xee, 0x01, 0x4c, 0x6a, 0xce, 0x37, 0xc5, 0x9f, 0x7b, 0xa6, 0xad, 0x59, 0x1e, 0x9d, 0x69, 0xc9, - 0x90, 0xf7, 0xcf, 0x0b, 0x30, 0x2d, 0x26, 0xc0, 0xa4, 0x57, 0x94, 0x96, 0x69, 0xef, 0x36, 0x89, - 0x75, 0x12, 0xc6, 0x5a, 0xfe, 0x47, 0x50, 0xc8, 0xfc, 0x08, 0x9a, 0x89, 0x8f, 0xa0, 0x2f, 0xcf, - 0x47, 0x10, 0xf0, 0x8a, 0x03, 0x14, 0x33, 0x18, 0xce, 0x1a, 0x16, 0xed, 0xe8, 0x75, 0xb6, 0x61, - 0x42, 0x93, 0x4a, 0xc6, 0x9f, 0x86, 0xf9, 0xdd, 0xb3, 0x92, 0x5a, 0x03, 0x67, 0xb4, 0x54, 0xff, - 0x5a, 0x81, 0x99, 0xb4, 0xf9, 0x3d, 0x01, 0x5d, 0x6a, 0x33, 0xaa, 0x4b, 0xbd, 0xd1, 0xc3, 0xb7, - 0x11, 0xe9, 0x7a, 0x86, 0x4e, 0xf5, 0xaf, 0x0a, 0x70, 0x36, 0xac, 0x5e, 0xb1, 0x5c, 0x4f, 0x33, - 0x4d, 0x2e, 0x97, 0x1e, 0xff, 0x5e, 0x6a, 0x45, 0x54, 0xe2, 0xab, 0xbd, 0x0d, 0x55, 0xee, 0x7b, - 0xa6, 0x5d, 0xc7, 0x4e, 0xcc, 0xae, 0xa3, 0x7a, 0x84, 0x34, 0x3b, 0x9b, 0x78, 0xfc, 0x47, 0x05, - 0x66, 0xd3, 0x1b, 0x9e, 0xc0, 0xa6, 0xb2, 0xa3, 0x9b, 0xea, 0xbd, 0x47, 0x37, 0xea, 0x8c, 0x6d, - 0xf5, 0x0b, 0x85, 0xac, 0xd1, 0x32, 0xbd, 0xfa, 0x26, 0x9c, 0x72, 0x48, 0xc3, 0x70, 0x3d, 0x61, - 0x80, 0x70, 0x38, 0x1b, 0x6b, 0xff, 0x3d, 0xed, 0x14, 0x8e, 0xe2, 0xc0, 0x71, 0xa4, 0x68, 0x15, - 0x86, 0x5c, 0x42, 0xea, 0x14, 0x7f, 0xa1, 0x7b, 0xfc, 0x81, 0x10, 0x51, 0xe3, 0x6d, 0xb1, 0x8f, - 0x04, 0x7d, 0x07, 0x8c, 0xd7, 0x83, 0x2f, 0x8a, 0x62, 0xed, 0xeb, 0x1e, 0x2b, 0xbb, 0xa2, 0x95, - 0xe5, 0xd6, 0x38, 0x8a, 0x4c, 0xfd, 0x3b, 0x05, 0x2e, 0x76, 0xda, 0x5b, 0xe8, 0x15, 0x00, 0xdd, - 0x97, 0x0a, 0x5d, 0x71, 0x78, 0x3e, 0x91, 0x73, 0x2d, 0x39, 0x96, 0xf0, 0x03, 0x0d, 0x8a, 0x5c, - 0x2c, 0x11, 0x49, 0xb1, 0xf6, 0x2b, 0x1c, 0x93, 0xb5, 0x9f, 0xfa, 0x9f, 0x14, 0x99, 0x15, 0xc9, - 0x6b, 0xfb, 0x46, 0x63, 0x45, 0x72, 0xdf, 0x33, 0xdf, 0xe9, 0xbe, 0x52, 0x80, 0x2b, 0xe9, 0x4d, - 0xa4, 0xf3, 0xfc, 0x69, 0x18, 0x6c, 0x71, 0x3f, 0x08, 0x6e, 0x12, 0x7e, 0x95, 0x72, 0x16, 0xee, - 0xa5, 0x70, 0x6f, 0xaf, 0x38, 0x9b, 0xc6, 0xe8, 0x85, 0x7f, 0x83, 0x68, 0x87, 0x8c, 0xd8, 0x83, - 0x02, 0x17, 0xda, 0xdf, 0xd1, 0x25, 0x73, 0xd1, 0x36, 0x88, 0xd9, 0xf5, 0x1b, 0xc2, 0x47, 0x14, - 0x98, 0x88, 0xec, 0x68, 0x77, 0x66, 0x80, 0xed, 0xd1, 0x5c, 0x86, 0x56, 0x91, 0x4f, 0x25, 0x94, - 0x06, 0x22, 0xc5, 0x2e, 0x8e, 0x11, 0x8c, 0xb1, 0x59, 0x79, 0x56, 0xdf, 0x70, 0x6c, 0x56, 0xee, - 0x7c, 0x06, 0x9b, 0xfd, 0x89, 0x42, 0xd6, 0x68, 0x19, 0x9b, 0xbd, 0x03, 0x23, 0xbe, 0x18, 0xe9, - 0xb3, 0x8b, 0xa5, 0x5e, 0xfb, 0xc4, 0xd1, 0x2d, 0x4c, 0x89, 0xfe, 0x8c, 0xf8, 0x25, 0x2e, 0x0e, - 0x69, 0xa1, 0xef, 0x55, 0x00, 0xc2, 0x85, 0x11, 0x1f, 0xd5, 0xda, 0xd1, 0x4d, 0x87, 0x24, 0xd6, - 0x4c, 0xd0, 0x4f, 0x5a, 0xda, 0x14, 0x12, 0x5d, 0xf5, 0x5f, 0xf7, 0x03, 0x4a, 0xf6, 0xbd, 0xbb, - 0xe7, 0xe2, 0x03, 0x84, 0xdc, 0x16, 0x4c, 0x3a, 0x54, 0x5a, 0xd4, 0x0d, 0x93, 0x5d, 0x59, 0xed, - 0xb6, 0x97, 0x53, 0xf3, 0xc1, 0xae, 0x55, 0x38, 0x86, 0x0b, 0x27, 0xb0, 0xa3, 0xb7, 0xc0, 0x50, - 0xcb, 0x31, 0x9a, 0x9a, 0xb3, 0xcb, 0x2e, 0xc5, 0xc3, 0xfc, 0x2d, 0xab, 0xca, 0x8b, 0xb0, 0x0f, - 0x43, 0x1f, 0x80, 0x11, 0xd3, 0xd8, 0x24, 0xfa, 0xae, 0x6e, 0x12, 0xa1, 0x2a, 0xbf, 0x75, 0x34, - 0x6b, 0xbe, 0xec, 0xa3, 0x15, 0x16, 0x88, 0xfe, 0x4f, 0x1c, 0x12, 0x44, 0x15, 0x38, 0x7d, 0x87, - 0xd9, 0xc4, 0x98, 0xc4, 0x75, 0x6b, 0xed, 0x56, 0xcb, 0x76, 0x3c, 0x52, 0x67, 0x0a, 0xf5, 0x61, - 0xae, 0xd8, 0x7b, 0x2e, 0x09, 0xc6, 0x69, 0x6d, 0xd0, 0x13, 0x00, 0x5a, 0xdb, 0xb3, 0xb9, 0x43, - 0xe2, 0xcc, 0x30, 0xbb, 0xb5, 0x5f, 0xa2, 0x8b, 0x3d, 0x1f, 0x94, 0xde, 0xdb, 0x2b, 0x8e, 0x0a, - 0xdd, 0x1f, 0x5b, 0x1a, 0xa9, 0x01, 0x7a, 0x11, 0xa6, 0x75, 0x0e, 0x2a, 0xd9, 0xcd, 0x96, 0xe6, - 0x19, 0xe2, 0x46, 0x3c, 0xc2, 0x10, 0x7d, 0xcb, 0xfe, 0x5e, 0x71, 0xba, 0x94, 0x02, 0x8f, 0xa3, - 0x4c, 0x45, 0xa2, 0x7e, 0xb2, 0x00, 0x17, 0x3a, 0x4c, 0x10, 0xc2, 0xf4, 0xc3, 0x13, 0xeb, 0x27, - 0xb6, 0xd9, 0x23, 0xfc, 0x63, 0x11, 0x85, 0xf7, 0xf6, 0x8a, 0xf7, 0x75, 0x40, 0x50, 0xa3, 0xfb, - 0x9c, 0x34, 0x76, 0x71, 0x88, 0x06, 0x55, 0x60, 0xb0, 0x1e, 0xbe, 0x7d, 0x8d, 0x2c, 0x3c, 0x44, - 0x8f, 0x02, 0xae, 0xa5, 0xee, 0x16, 0x9b, 0x40, 0x80, 0x96, 0x61, 0x88, 0xdb, 0x54, 0x12, 0x71, - 0xac, 0x3c, 0xcc, 0x54, 0x26, 0xbc, 0xa8, 0x5b, 0x64, 0x3e, 0x0a, 0xf5, 0x6f, 0x15, 0x18, 0x2a, - 0xd9, 0x0e, 0x29, 0xaf, 0xd6, 0xd0, 0x2e, 0x8c, 0x4a, 0x1e, 0xf1, 0x82, 0xc5, 0xe6, 0xe4, 0x39, - 0x0c, 0xa3, 0x74, 0xb5, 0xf7, 0x7d, 0xf8, 0x82, 0x02, 0x2c, 0xd3, 0x42, 0xaf, 0xd0, 0x39, 0xbf, - 0xe3, 0x18, 0x1e, 0x25, 0xdc, 0x8b, 0x91, 0x0d, 0x27, 0x8c, 0x7d, 0x5c, 0x7c, 0xb7, 0x07, 0x3f, - 0x71, 0x48, 0x45, 0xad, 0x52, 0xf6, 0x12, 0xef, 0x26, 0x7a, 0x1c, 0xfa, 0x9b, 0x76, 0xdd, 0x5f, - 0xf7, 0xb7, 0xfa, 0xcc, 0x63, 0xc5, 0xae, 0xd3, 0xb9, 0x3d, 0x9b, 0x6c, 0xc1, 0xde, 0x93, 0x58, - 0x1b, 0x75, 0x15, 0x26, 0xe3, 0xf4, 0xd1, 0xe3, 0x30, 0xa1, 0xdb, 0xcd, 0xa6, 0x6d, 0xd5, 0xda, - 0x9b, 0x9b, 0xc6, 0x0e, 0x89, 0x38, 0x57, 0x96, 0x22, 0x10, 0x1c, 0xab, 0xa9, 0x7e, 0x4e, 0x81, - 0x3e, 0xba, 0x2e, 0x2a, 0x0c, 0xd6, 0xed, 0xa6, 0x66, 0x58, 0xa2, 0x57, 0xec, 0x95, 0xba, 0xcc, - 0x4a, 0xb0, 0x80, 0xa0, 0x16, 0x8c, 0xf8, 0x12, 0x59, 0x4f, 0x66, 0xe1, 0xe5, 0xd5, 0x5a, 0xe0, - 0x55, 0x13, 0x1c, 0x13, 0x7e, 0x89, 0x8b, 0x43, 0x22, 0xaa, 0x06, 0x53, 0xe5, 0xd5, 0x5a, 0xc5, - 0xd2, 0xcd, 0x76, 0x9d, 0x2c, 0xee, 0xb0, 0x3f, 0x94, 0xcf, 0x19, 0xbc, 0x44, 0x8c, 0x93, 0xf1, - 0x39, 0x51, 0x09, 0xfb, 0x30, 0x5a, 0x8d, 0xf0, 0x16, 0xc2, 0xd7, 0x90, 0x55, 0x13, 0x48, 0xb0, - 0x0f, 0x53, 0xbf, 0x5a, 0x80, 0x51, 0xa9, 0x43, 0xc8, 0x84, 0x21, 0x3e, 0x5c, 0xb7, 0x17, 0x7f, - 0xf2, 0x44, 0xaf, 0x39, 0x75, 0x3e, 0xa1, 0x2e, 0xf6, 0x49, 0xc8, 0x3c, 0xbb, 0xd0, 0x81, 0x67, - 0xcf, 0x45, 0x5c, 0x36, 0xf9, 0x27, 0x39, 0x91, 0xed, 0xae, 0x89, 0x2e, 0x8a, 0xe3, 0x89, 0xdb, - 0x65, 0x0f, 0xc7, 0x8e, 0xa6, 0x4d, 0x18, 0xb8, 0x6b, 0x5b, 0xc4, 0x15, 0xba, 0xf0, 0x23, 0x1a, - 0x20, 0xb3, 0xa6, 0x7b, 0x81, 0xe2, 0xc5, 0x1c, 0xbd, 0xfa, 0x53, 0x0a, 0x40, 0x59, 0xf3, 0x34, - 0x6e, 0xba, 0xd1, 0x85, 0x75, 0xe0, 0xc5, 0xc8, 0xa9, 0x3a, 0x9c, 0xf0, 0x0c, 0xeb, 0x77, 0x8d, - 0xbb, 0xfe, 0xf0, 0x03, 0x69, 0x9d, 0x63, 0xaf, 0x19, 0x77, 0x09, 0x66, 0x70, 0xf4, 0x20, 0x8c, - 0x10, 0x4b, 0x77, 0x76, 0x5b, 0xf4, 0x60, 0xe1, 0x2a, 0x1e, 0xf6, 0x85, 0x2e, 0xfa, 0x85, 0x38, - 0x84, 0xab, 0x0f, 0x41, 0xf4, 0xca, 0xd5, 0x85, 0xf1, 0xf2, 0xdf, 0x2b, 0x70, 0xae, 0xdc, 0xd6, - 0xcc, 0xf9, 0x16, 0xdd, 0xa8, 0x9a, 0xb9, 0x64, 0x73, 0x1b, 0x00, 0x7a, 0x0f, 0x79, 0x1b, 0x0c, - 0xfb, 0x42, 0x4e, 0xdc, 0x75, 0xd4, 0x67, 0x94, 0x38, 0xa8, 0x81, 0x34, 0x18, 0x76, 0x7d, 0xb1, - 0xbb, 0xd0, 0x83, 0xd8, 0xed, 0x93, 0x08, 0xc4, 0xee, 0x00, 0x2d, 0x57, 0x7e, 0xb1, 0x05, 0xaa, - 0x11, 0x67, 0xdb, 0xd0, 0xc9, 0xbc, 0xae, 0xdb, 0x6d, 0xcb, 0x73, 0x85, 0x2b, 0xb8, 0x50, 0x7e, - 0xa5, 0xd5, 0xc0, 0x19, 0x2d, 0xd5, 0x2f, 0x29, 0xd0, 0xbf, 0xb8, 0x56, 0x2a, 0xa3, 0xef, 0x80, - 0xfe, 0x80, 0x65, 0xe4, 0x34, 0xf5, 0xa1, 0x78, 0xb8, 0x9e, 0x8e, 0xaf, 0xf7, 0x0a, 0x65, 0x38, - 0x0c, 0x2b, 0xda, 0x80, 0x41, 0xb2, 0x4d, 0x68, 0x57, 0x0b, 0x47, 0x82, 0x9f, 0xb1, 0xb4, 0x45, - 0x86, 0x11, 0x0b, 0xcc, 0xea, 0xa7, 0x14, 0x80, 0xb0, 0x0a, 0xfa, 0xee, 0xb4, 0xd3, 0xe9, 0xe6, - 0x11, 0x6a, 0x9f, 0x3b, 0x1f, 0x51, 0xea, 0xaf, 0x0f, 0xc0, 0x79, 0xda, 0x1d, 0xb1, 0x55, 0x0d, - 0xdb, 0xba, 0x49, 0x76, 0xbf, 0x69, 0x26, 0xff, 0x4d, 0x33, 0xf9, 0x23, 0x34, 0x93, 0x7f, 0x11, - 0xce, 0xd3, 0x2d, 0x27, 0xc0, 0x84, 0xd9, 0x0c, 0x55, 0x1d, 0xd2, 0xd2, 0x1c, 0x52, 0x67, 0xf7, - 0x81, 0x61, 0x6e, 0x05, 0x35, 0x9f, 0x55, 0x09, 0x67, 0xb7, 0x57, 0x9f, 0x82, 0xc9, 0x70, 0xef, - 0x8a, 0x8f, 0xea, 0xc1, 0xf8, 0x25, 0x73, 0xc4, 0x97, 0x98, 0x92, 0x17, 0x43, 0xf5, 0x9e, 0x02, - 0x93, 0x8b, 0x3b, 0x2d, 0xc3, 0x61, 0xfe, 0xf9, 0xdc, 0xcd, 0x04, 0x3d, 0x10, 0x7a, 0xa3, 0x28, - 0xd1, 0x57, 0xbc, 0xb8, 0x47, 0x0a, 0xda, 0x84, 0x09, 0xc2, 0x9a, 0xb3, 0x5b, 0xa0, 0xe6, 0xe5, - 0xd9, 0xde, 0x3c, 0x28, 0x45, 0x04, 0x0b, 0x8e, 0x61, 0x45, 0x35, 0x98, 0xd0, 0x4d, 0xcd, 0x75, - 0x8d, 0x4d, 0x43, 0x0f, 0xbd, 0xa8, 0x46, 0x16, 0x1e, 0x64, 0x32, 0x57, 0x04, 0x72, 0x6f, 0xaf, - 0x78, 0x46, 0xf4, 0x33, 0x0a, 0xc0, 0x31, 0x14, 0xea, 0x67, 0x0a, 0x30, 0xbe, 0xb8, 0xd3, 0xb2, - 0xdd, 0xb6, 0x43, 0x58, 0xd5, 0x13, 0xd0, 0x6b, 0x3d, 0x00, 0x43, 0x5b, 0x9a, 0x55, 0x37, 0x89, - 0x23, 0x8e, 0xdd, 0x60, 0x6e, 0x6f, 0xf0, 0x62, 0xec, 0xc3, 0xd1, 0xab, 0x00, 0xae, 0xbe, 0x45, - 0xea, 0x6d, 0xc6, 0x1c, 0xfb, 0xf2, 0x33, 0xc7, 0xc8, 0x18, 0x6b, 0x01, 0x4a, 0x21, 0xd2, 0x04, - 0xbf, 0xb1, 0x44, 0x4e, 0xfd, 0x13, 0x05, 0xa6, 0x22, 0xed, 0x4e, 0x40, 0x5d, 0xb3, 0x19, 0x55, - 0xd7, 0xcc, 0xf7, 0x3c, 0xd6, 0x0c, 0x2d, 0xcd, 0xc7, 0x0b, 0x70, 0x2e, 0x63, 0x4e, 0x12, 0xf6, - 0xbe, 0xca, 0x09, 0xd9, 0xfb, 0xb6, 0x61, 0xd4, 0xb3, 0x4d, 0xe1, 0xec, 0xe7, 0xcf, 0x40, 0xae, - 0x23, 0x78, 0x2d, 0x40, 0x13, 0x5a, 0xf3, 0x86, 0x65, 0x2e, 0x96, 0xe9, 0xa8, 0xbf, 0xa9, 0xc0, - 0x48, 0xa0, 0x15, 0x7e, 0x5d, 0x3d, 0xa8, 0x77, 0x1f, 0x47, 0x47, 0xfd, 0xbd, 0x02, 0x9c, 0x0d, - 0x70, 0xfb, 0x6c, 0xae, 0xe6, 0x51, 0xbe, 0x71, 0xb0, 0x6a, 0xe9, 0x62, 0xc4, 0x13, 0x61, 0x38, - 0xe9, 0x98, 0xd7, 0x6a, 0x3b, 0x2d, 0xdb, 0xf5, 0xe5, 0x60, 0x7e, 0x61, 0xe0, 0x45, 0xd8, 0x87, - 0xa1, 0x55, 0x18, 0x70, 0x29, 0x3d, 0x71, 0xd6, 0x1d, 0x72, 0x36, 0x98, 0x28, 0xcf, 0xfa, 0x8b, - 0x39, 0x1a, 0xf4, 0xaa, 0xcc, 0xc3, 0x07, 0xf2, 0x2b, 0x2f, 0xe9, 0x48, 0xea, 0x81, 0x24, 0x1c, - 0x3e, 0x7d, 0x74, 0x52, 0x16, 0xaa, 0xcb, 0x30, 0x29, 0x8c, 0x5a, 0xf9, 0xb6, 0xb1, 0x74, 0x82, - 0xde, 0x1d, 0xd9, 0x19, 0xf7, 0xc7, 0x4c, 0x6a, 0xa6, 0xe3, 0xf5, 0xc3, 0x1d, 0xa3, 0xba, 0x30, - 0x7c, 0x5d, 0x74, 0x12, 0xcd, 0x42, 0xc1, 0xf0, 0xd7, 0x02, 0x04, 0x8e, 0x42, 0xa5, 0x8c, 0x0b, - 0x46, 0x17, 0x1e, 0x21, 0xf2, 0xb1, 0xd4, 0xd7, 0xf9, 0x58, 0x52, 0xbf, 0x5e, 0x80, 0x69, 0x9f, - 0xaa, 0x3f, 0xc6, 0xb2, 0x78, 0x2d, 0x3f, 0xe0, 0x52, 0x74, 0xb0, 0xaa, 0xf1, 0x16, 0xf4, 0x33, - 0x06, 0x98, 0xeb, 0x15, 0x3d, 0x40, 0x48, 0xbb, 0x83, 0x19, 0x22, 0xf4, 0x01, 0x18, 0x34, 0xe9, - 0x0d, 0xc3, 0x77, 0xd5, 0xc8, 0xa5, 0x98, 0x4d, 0x1b, 0x2e, 0xbf, 0xb8, 0x88, 0x10, 0x66, 0xc1, - 0x43, 0x28, 0x2f, 0xc4, 0x82, 0xe6, 0xec, 0x63, 0x30, 0x2a, 0x55, 0x3b, 0x54, 0xfc, 0xb2, 0xcf, - 0x15, 0x60, 0xe6, 0x06, 0x31, 0x9b, 0xa9, 0xa6, 0x0f, 0x45, 0x18, 0xd0, 0xb7, 0x34, 0x87, 0x87, - 0xc6, 0x1b, 0xe3, 0x9b, 0xbc, 0x44, 0x0b, 0x30, 0x2f, 0xa7, 0x17, 0x8e, 0x88, 0x73, 0xda, 0x93, - 0xd2, 0x4c, 0x86, 0x31, 0x13, 0xdf, 0x1f, 0x04, 0x55, 0x0c, 0x07, 0x1e, 0xa9, 0x40, 0x8f, 0x97, - 0xf7, 0xd6, 0x6e, 0xad, 0xa6, 0xb9, 0xa5, 0xa1, 0xbb, 0x30, 0x6e, 0xeb, 0x06, 0x26, 0x2d, 0xdb, - 0x35, 0x3c, 0xdb, 0xd9, 0x15, 0x8b, 0x96, 0xeb, 0x68, 0xb9, 0x55, 0xaa, 0x84, 0x88, 0xf8, 0xf3, - 0x61, 0xa4, 0x08, 0x47, 0x49, 0xa9, 0x5f, 0x54, 0x60, 0xf4, 0x86, 0xb1, 0x41, 0x1c, 0x6e, 0xb7, - 0xcb, 0x34, 0x24, 0x91, 0x20, 0x6f, 0xa3, 0x69, 0x01, 0xde, 0xd0, 0x0e, 0x8c, 0x88, 0x73, 0x38, - 0xf0, 0x3a, 0xbc, 0x9e, 0xcf, 0x5e, 0x28, 0x20, 0x2d, 0xce, 0x37, 0xe9, 0xc3, 0xf7, 0x4b, 0x5c, - 0x1c, 0x12, 0x53, 0x5f, 0x85, 0xd3, 0x29, 0x8d, 0xe8, 0x42, 0x32, 0xd3, 0x55, 0xf1, 0xd1, 0xf8, - 0xdc, 0x8a, 0x2e, 0x24, 0x2b, 0x47, 0xe7, 0xa1, 0x8f, 0x58, 0x75, 0xf1, 0xc5, 0x0c, 0xed, 0xef, - 0x15, 0xfb, 0x16, 0xad, 0x3a, 0xa6, 0x65, 0x94, 0x89, 0x9b, 0x76, 0x44, 0x62, 0x63, 0x4c, 0x7c, - 0x59, 0x94, 0xe1, 0x00, 0xca, 0x2c, 0xbc, 0xe2, 0xc6, 0x4c, 0xf4, 0x66, 0x31, 0xb9, 0x19, 0xe3, - 0x2d, 0xbd, 0xd8, 0x50, 0xc5, 0xf9, 0xd4, 0xc2, 0x8c, 0x98, 0x90, 0x04, 0xc7, 0xc3, 0x09, 0xba, - 0xea, 0xaf, 0xf4, 0xc3, 0xa5, 0x1b, 0xb6, 0x63, 0xdc, 0xb5, 0x2d, 0x4f, 0x33, 0xab, 0x76, 0x3d, - 0xb4, 0x6f, 0x15, 0x47, 0xd6, 0xc7, 0x14, 0x38, 0xa7, 0xb7, 0xda, 0xfc, 0x66, 0xe2, 0x9b, 0x88, - 0x56, 0x89, 0x63, 0xd8, 0x79, 0x1d, 0x35, 0x98, 0x39, 0x6f, 0xa9, 0xba, 0x9e, 0x86, 0x12, 0x67, - 0xd1, 0x62, 0xfe, 0x22, 0x75, 0xfb, 0x8e, 0xc5, 0x3a, 0x57, 0xf3, 0xd8, 0x6c, 0xde, 0x0d, 0x17, - 0x21, 0xa7, 0xbf, 0x48, 0x39, 0x15, 0x23, 0xce, 0xa0, 0x84, 0x3e, 0x04, 0x67, 0x0c, 0xde, 0x39, - 0x4c, 0xb4, 0xba, 0x61, 0x11, 0xd7, 0xe5, 0xc6, 0xe6, 0x3d, 0x38, 0x44, 0x54, 0xd2, 0x10, 0xe2, - 0x74, 0x3a, 0xe8, 0x25, 0x00, 0x77, 0xd7, 0xd2, 0xc5, 0xfc, 0xe7, 0x33, 0x44, 0xe5, 0x22, 0x72, - 0x80, 0x05, 0x4b, 0x18, 0xe9, 0x45, 0xcb, 0x0b, 0x36, 0xe5, 0x20, 0x33, 0x26, 0x66, 0x17, 0xad, - 0x70, 0x0f, 0x85, 0x70, 0xf5, 0x53, 0x0a, 0x4c, 0x54, 0xac, 0xaa, 0xa9, 0xe9, 0x84, 0xdb, 0x55, - 0xba, 0xe8, 0x1a, 0x8c, 0xb8, 0xc1, 0x8b, 0x0c, 0xe7, 0x08, 0xe1, 0xf7, 0x19, 0xbc, 0xc5, 0x84, - 0x75, 0x98, 0x95, 0xb6, 0x61, 0x89, 0xc3, 0x6e, 0xc9, 0x76, 0x38, 0x22, 0xf1, 0xdd, 0x71, 0x2b, - 0xed, 0x24, 0x18, 0xa7, 0xb5, 0x51, 0x7f, 0x5e, 0x81, 0xe9, 0x68, 0x77, 0x84, 0x49, 0xc3, 0x8f, - 0x2b, 0x30, 0x1d, 0x71, 0xa6, 0x16, 0xe0, 0x5e, 0xe2, 0x3b, 0x55, 0x53, 0xf0, 0x71, 0x13, 0xdf, - 0x34, 0x08, 0x4e, 0xa5, 0xaf, 0xfe, 0x13, 0x05, 0x86, 0x44, 0xac, 0x47, 0xf4, 0xd6, 0x98, 0xf6, - 0x3c, 0x38, 0xda, 0x62, 0x1a, 0xf4, 0x5d, 0x66, 0x9f, 0x21, 0x8e, 0x26, 0x71, 0xca, 0xe4, 0x52, - 0xbf, 0x0a, 0xc2, 0xe1, 0x39, 0x17, 0xb1, 0xd3, 0xf0, 0x9f, 0x66, 0x24, 0x62, 0xea, 0x6b, 0x0a, - 0x4c, 0x25, 0x5a, 0x75, 0x21, 0x8e, 0x9e, 0xa0, 0xc5, 0xea, 0x57, 0xfa, 0xe9, 0x96, 0xf4, 0x28, - 0xbf, 0x37, 0xb9, 0x62, 0xfb, 0x04, 0xee, 0xbf, 0x0f, 0xc2, 0x88, 0xd1, 0x6c, 0xb6, 0x3d, 0xf6, - 0x88, 0x38, 0x10, 0x6a, 0x8b, 0x2b, 0x7e, 0x21, 0x0e, 0xe1, 0xc8, 0x12, 0x92, 0x16, 0x3f, 0x05, - 0x97, 0xf3, 0xad, 0x9c, 0x3c, 0xc0, 0x39, 0x2a, 0x15, 0x71, 0x71, 0x28, 0x4d, 0x10, 0xfb, 0x3e, - 0x05, 0xc0, 0xf5, 0x1c, 0xc3, 0x6a, 0xd0, 0x42, 0x21, 0x8d, 0xe1, 0x23, 0x20, 0x5b, 0x0b, 0x90, - 0x72, 0xe2, 0x61, 0xfc, 0xc7, 0x00, 0x80, 0x25, 0xca, 0x68, 0x5e, 0x08, 0xa1, 0xfc, 0xc8, 0x7c, - 0x7b, 0x4c, 0xdc, 0xbe, 0x94, 0x62, 0x09, 0xcb, 0x09, 0x85, 0x52, 0xea, 0xec, 0xa3, 0x30, 0x12, - 0xd0, 0x3b, 0x48, 0xa8, 0x1b, 0x93, 0x84, 0xba, 0xd9, 0x27, 0xe0, 0x54, 0xac, 0xbb, 0x87, 0x92, - 0x09, 0xff, 0x54, 0x01, 0x14, 0x1d, 0xfd, 0x09, 0x68, 0x0e, 0x1a, 0x51, 0xcd, 0xc1, 0x42, 0xef, - 0x4b, 0x96, 0xa1, 0x3a, 0xf8, 0x59, 0x04, 0x2c, 0x14, 0x6e, 0x10, 0x1a, 0x5a, 0x9c, 0xfc, 0x54, - 0x50, 0x09, 0x3d, 0xc0, 0xc5, 0x97, 0xdb, 0x83, 0xa0, 0x72, 0x33, 0x86, 0x2b, 0x14, 0x54, 0xe2, - 0x10, 0x9c, 0xa0, 0x8b, 0x3e, 0xa1, 0xc0, 0xa4, 0x16, 0x0d, 0x85, 0xeb, 0xcf, 0x4c, 0xae, 0x00, - 0x5d, 0xb1, 0xb0, 0xba, 0x61, 0x5f, 0x62, 0x00, 0x17, 0x27, 0xc8, 0xa2, 0x47, 0x60, 0x4c, 0x6b, - 0x19, 0xf3, 0xed, 0xba, 0x41, 0x6f, 0x9e, 0x7e, 0xc4, 0x50, 0xa6, 0x0d, 0x99, 0xaf, 0x56, 0x82, - 0x72, 0x1c, 0xa9, 0x15, 0xc4, 0x9c, 0x2d, 0x85, 0x86, 0xc3, 0xbd, 0xc4, 0x9c, 0x15, 0x73, 0x18, - 0xc6, 0x9c, 0x15, 0x53, 0x27, 0x13, 0x41, 0x16, 0x80, 0x6d, 0xd4, 0x75, 0x41, 0x72, 0x30, 0xff, - 0x1b, 0xc8, 0xad, 0x4a, 0xb9, 0x24, 0x28, 0x32, 0xf1, 0x21, 0xfc, 0x8d, 0x25, 0x0a, 0xe8, 0xd3, - 0x0a, 0x8c, 0x0b, 0xde, 0x2d, 0x68, 0x0e, 0xb1, 0x25, 0x7a, 0x21, 0xef, 0x7e, 0x89, 0xed, 0xc9, - 0x39, 0x2c, 0x23, 0xe7, 0x7c, 0x27, 0x08, 0x20, 0x10, 0x81, 0xe1, 0x68, 0x3f, 0x98, 0x0c, 0xe0, - 0x46, 0x1e, 0xa1, 0x44, 0x07, 0x87, 0xf3, 0xcb, 0x00, 0xb5, 0x14, 0x7c, 0xc2, 0xc9, 0x28, 0x05, - 0x82, 0x53, 0xe9, 0x53, 0xb9, 0xf6, 0xd4, 0x1d, 0xcd, 0xd3, 0xb7, 0x4a, 0x9a, 0xbe, 0xc5, 0xde, - 0x20, 0xb9, 0x3b, 0x65, 0xce, 0x7d, 0xfd, 0x5c, 0x14, 0xd5, 0xc2, 0xe9, 0xfd, 0xbd, 0xe2, 0xa9, - 0x58, 0x21, 0x8e, 0x13, 0x44, 0x36, 0x0c, 0x3b, 0x22, 0x1e, 0xbc, 0xf0, 0x80, 0xcf, 0x17, 0x02, - 0x3d, 0x1e, 0x5c, 0x9e, 0xdf, 0x8c, 0xfc, 0x5f, 0x38, 0x20, 0x82, 0x1a, 0x70, 0x89, 0xdf, 0x0d, - 0xe7, 0x2d, 0xdb, 0xda, 0x6d, 0xda, 0x6d, 0x77, 0xbe, 0xed, 0x6d, 0x11, 0xcb, 0xf3, 0x55, 0xe1, - 0xa3, 0xec, 0x18, 0x65, 0x4e, 0x73, 0x8b, 0x9d, 0x2a, 0xe2, 0xce, 0x78, 0xd0, 0xf3, 0x30, 0xcc, - 0xde, 0xe9, 0xd6, 0xd6, 0x96, 0x99, 0x67, 0xe6, 0xe1, 0xc5, 0x65, 0x36, 0x84, 0x45, 0x81, 0x03, - 0x07, 0xd8, 0xd0, 0x6d, 0x18, 0x32, 0x79, 0x40, 0x7f, 0xe6, 0xa1, 0x99, 0x93, 0x29, 0xc6, 0x93, - 0x03, 0xf0, 0x0b, 0xb4, 0xf8, 0x81, 0x7d, 0x0a, 0xa8, 0x05, 0x57, 0xea, 0x64, 0x53, 0x6b, 0x9b, - 0xde, 0xaa, 0xed, 0x61, 0xe6, 0xa1, 0x16, 0x68, 0x3c, 0x7d, 0x27, 0xdc, 0x09, 0x16, 0x83, 0x8d, - 0xf9, 0xfe, 0x95, 0x0f, 0xa8, 0x8b, 0x0f, 0xc4, 0x86, 0x76, 0xe1, 0x3e, 0x51, 0x87, 0xb9, 0xc4, - 0xe9, 0x5b, 0x74, 0x96, 0x93, 0x44, 0x4f, 0x31, 0xa2, 0xdf, 0xb2, 0xbf, 0x57, 0xbc, 0xaf, 0x7c, - 0x70, 0x75, 0xdc, 0x0d, 0x4e, 0xe6, 0x65, 0x44, 0x62, 0x4f, 0x40, 0x33, 0x93, 0xf9, 0xe7, 0x38, - 0xfe, 0x9c, 0xc4, 0xcd, 0xe1, 0xe2, 0xa5, 0x38, 0x41, 0x13, 0x7d, 0x41, 0x81, 0x19, 0xd7, 0x73, - 0xda, 0xba, 0xd7, 0x76, 0x48, 0x3d, 0xb6, 0x43, 0xa7, 0x58, 0x87, 0x72, 0x09, 0x70, 0xb5, 0x0c, - 0x9c, 0xcc, 0x1d, 0x7c, 0x26, 0x0b, 0x8a, 0x33, 0xfb, 0x82, 0x3e, 0xaf, 0xc0, 0xb9, 0x28, 0x90, - 0xde, 0xe9, 0x79, 0x3f, 0x51, 0xfe, 0x47, 0x96, 0x5a, 0x3a, 0x4a, 0x7e, 0x83, 0xcf, 0x00, 0xe2, - 0xac, 0x8e, 0xc4, 0x5f, 0xc6, 0x4f, 0x9f, 0xf0, 0xcb, 0xf8, 0xec, 0xd3, 0x80, 0x92, 0xc7, 0xc7, - 0x41, 0x72, 0xe0, 0xb0, 0x2c, 0x07, 0x7e, 0x76, 0x00, 0x2e, 0xd0, 0x53, 0x29, 0xbc, 0xfd, 0xac, - 0x68, 0x96, 0xd6, 0x78, 0x7d, 0x4a, 0x4c, 0x5f, 0x54, 0xe0, 0xdc, 0x56, 0xba, 0x6a, 0x47, 0xdc, - 0xbf, 0x9e, 0xc9, 0xa5, 0x82, 0xeb, 0xa4, 0x2d, 0xe2, 0x0c, 0xbb, 0x63, 0x15, 0x9c, 0xd5, 0x29, - 0xf4, 0x34, 0x4c, 0x5a, 0x76, 0x9d, 0x94, 0x2a, 0x65, 0xbc, 0xa2, 0xb9, 0xb7, 0x6b, 0xbe, 0xa1, - 0xce, 0x00, 0xff, 0x5e, 0x57, 0x63, 0x30, 0x9c, 0xa8, 0x8d, 0xb6, 0x01, 0xb5, 0xec, 0xfa, 0xe2, - 0x36, 0x4f, 0x7c, 0xd1, 0x9b, 0xc9, 0x2c, 0x33, 0x16, 0xa8, 0x26, 0xb0, 0xe1, 0x14, 0x0a, 0x4c, - 0x37, 0x45, 0x3b, 0xb3, 0x62, 0x5b, 0x86, 0x67, 0x3b, 0x2c, 0xc0, 0x41, 0x4f, 0x2a, 0x1a, 0xa6, - 0x9b, 0x5a, 0x4d, 0xc5, 0x88, 0x33, 0x28, 0xa9, 0xff, 0x4d, 0x81, 0x53, 0x74, 0x5b, 0x54, 0x1d, - 0x7b, 0x67, 0xf7, 0xf5, 0xb8, 0x21, 0x1f, 0x10, 0x36, 0x8b, 0x5c, 0xb7, 0x73, 0x46, 0xb2, 0x57, - 0x1c, 0x61, 0x7d, 0x0e, 0x4d, 0x14, 0x65, 0xb5, 0x72, 0x5f, 0xb6, 0x5a, 0x59, 0xfd, 0x74, 0x81, - 0xdf, 0x5c, 0x7c, 0xb5, 0xee, 0xeb, 0xf2, 0x3b, 0x7c, 0x14, 0xc6, 0x69, 0xd9, 0x8a, 0xb6, 0x53, - 0x2d, 0x3f, 0x6b, 0x9b, 0xbe, 0x37, 0x36, 0xd3, 0xb5, 0xdf, 0x94, 0x01, 0x38, 0x5a, 0x0f, 0x3d, - 0x0e, 0x43, 0x2d, 0x1e, 0x27, 0x4c, 0xdc, 0x99, 0xaf, 0x70, 0xc3, 0x3e, 0x56, 0x74, 0x6f, 0xaf, - 0x38, 0x15, 0x3e, 0xf1, 0xfa, 0xd1, 0xca, 0xfc, 0x06, 0xea, 0xaf, 0x9c, 0x05, 0x86, 0xdc, 0x24, - 0xde, 0xeb, 0x71, 0x4e, 0x1e, 0x82, 0x51, 0xbd, 0xd5, 0x2e, 0x2d, 0xd5, 0x9e, 0x69, 0xdb, 0x4c, - 0x17, 0xc2, 0xf2, 0xc3, 0x50, 0xee, 0x5d, 0xaa, 0xae, 0xfb, 0xc5, 0x58, 0xae, 0x43, 0xb9, 0x83, - 0xde, 0x6a, 0x0b, 0x7e, 0x5b, 0x95, 0xfd, 0x55, 0x18, 0x77, 0x28, 0x55, 0xd7, 0x23, 0x30, 0x9c, - 0xa8, 0x8d, 0x3e, 0x04, 0x63, 0x44, 0x7c, 0xb8, 0x37, 0x34, 0xa7, 0x2e, 0xf8, 0x42, 0x25, 0xef, - 0xe0, 0x83, 0xa9, 0xf5, 0xb9, 0x01, 0xbf, 0x01, 0x2e, 0x4a, 0x24, 0x70, 0x84, 0x20, 0x7a, 0x11, - 0xce, 0xfb, 0xbf, 0xe9, 0x2a, 0xdb, 0xf5, 0x38, 0xa3, 0x18, 0xe0, 0x66, 0x33, 0x8b, 0x59, 0x95, - 0x70, 0x76, 0x7b, 0xf4, 0x73, 0x0a, 0x9c, 0x0d, 0xa0, 0x86, 0x65, 0x34, 0xdb, 0x4d, 0x4c, 0x74, - 0x53, 0x33, 0x9a, 0xe2, 0xde, 0xf7, 0xdc, 0x91, 0x0d, 0x34, 0x8a, 0x9e, 0x33, 0xab, 0x74, 0x18, - 0xce, 0xe8, 0x12, 0x7a, 0x4d, 0x81, 0x2b, 0x3e, 0xa8, 0xea, 0x10, 0xd7, 0x6d, 0x3b, 0x24, 0x8c, - 0x05, 0x20, 0xa6, 0x64, 0x28, 0x17, 0xef, 0x64, 0x02, 0xf0, 0xe2, 0x01, 0xb8, 0xf1, 0x81, 0xd4, - 0xe5, 0xed, 0x52, 0xb3, 0x37, 0x3d, 0x71, 0x51, 0x3c, 0xae, 0xed, 0x42, 0x49, 0xe0, 0x08, 0x41, - 0xf4, 0xf3, 0x0a, 0x9c, 0x93, 0x0b, 0xe4, 0xdd, 0xc2, 0x6f, 0x88, 0xcf, 0x1f, 0x59, 0x67, 0x62, - 0xf8, 0xb9, 0x84, 0x97, 0x01, 0xc4, 0x59, 0xbd, 0xa2, 0x6c, 0xbb, 0xc9, 0x36, 0x26, 0xbf, 0x45, - 0x0e, 0x70, 0xb6, 0xcd, 0xf7, 0xaa, 0x8b, 0x7d, 0x18, 0x7a, 0x04, 0xc6, 0x5a, 0x76, 0xbd, 0x6a, - 0xd4, 0x5d, 0x16, 0xc3, 0x8c, 0xdd, 0xf5, 0xfa, 0xf8, 0x74, 0x54, 0xed, 0x7a, 0xb5, 0x52, 0xe6, - 0xe5, 0x38, 0x52, 0x0b, 0xcd, 0x01, 0x6c, 0x6a, 0x86, 0x59, 0xbb, 0xa3, 0xb5, 0x6e, 0xf9, 0x41, - 0x71, 0x98, 0x2e, 0x62, 0x29, 0x28, 0xc5, 0x52, 0x0d, 0xba, 0x7e, 0x94, 0xef, 0x60, 0xc2, 0xc3, - 0x4e, 0xb3, 0xeb, 0xd1, 0x51, 0xac, 0x9f, 0x8f, 0x90, 0x77, 0xf8, 0xa6, 0x44, 0x02, 0x47, 0x08, - 0xa2, 0x8f, 0x29, 0x30, 0xe1, 0xee, 0xba, 0x1e, 0x69, 0x06, 0x7d, 0x38, 0x75, 0xd4, 0x7d, 0x60, - 0x3a, 0xf1, 0x5a, 0x84, 0x08, 0x8e, 0x11, 0x65, 0xe1, 0x85, 0x9a, 0x5a, 0x83, 0x5c, 0x2f, 0xdd, - 0x30, 0x1a, 0x5b, 0x41, 0x74, 0x97, 0x2a, 0x71, 0x74, 0x62, 0x79, 0xec, 0x62, 0x35, 0x20, 0xc2, - 0x0b, 0x65, 0x57, 0xc3, 0x9d, 0x70, 0xa0, 0x97, 0x60, 0x56, 0x80, 0x97, 0xed, 0x3b, 0x09, 0x0a, - 0x3c, 0x8a, 0x0f, 0x33, 0x80, 0xac, 0x64, 0xd6, 0xc2, 0x1d, 0x30, 0xa0, 0x0a, 0x9c, 0x76, 0x89, - 0xc3, 0xde, 0x04, 0x79, 0x44, 0xc8, 0x6a, 0xdb, 0x34, 0x5d, 0x76, 0xb5, 0x11, 0x2e, 0x3f, 0xb5, - 0x24, 0x18, 0xa7, 0xb5, 0x41, 0x4f, 0x04, 0x6e, 0xc1, 0xbb, 0xb4, 0xe0, 0x99, 0x6a, 0x8d, 0xdd, - 0x44, 0x06, 0xb8, 0xa6, 0x04, 0x47, 0x41, 0x38, 0x5e, 0x97, 0x9e, 0xe6, 0x7e, 0xd1, 0x42, 0xdb, - 0x71, 0xbd, 0x99, 0x69, 0xd6, 0x98, 0x9d, 0xe6, 0x58, 0x06, 0xe0, 0x68, 0x3d, 0xf4, 0x38, 0x4c, - 0xb8, 0x44, 0xd7, 0xed, 0x66, 0x4b, 0xdc, 0x93, 0x67, 0xce, 0xb0, 0xde, 0xf3, 0x15, 0x8c, 0x40, - 0x70, 0xac, 0x26, 0xda, 0x85, 0xd3, 0x41, 0x08, 0xdd, 0x65, 0xbb, 0xb1, 0xa2, 0xed, 0x30, 0xe1, - 0xf8, 0x6c, 0x9e, 0x10, 0x0f, 0x7c, 0xba, 0x4a, 0x49, 0x74, 0x38, 0x8d, 0x06, 0x5a, 0x86, 0xe9, - 0x58, 0xf1, 0x92, 0x61, 0x12, 0x77, 0xe6, 0x1c, 0x1b, 0x36, 0x53, 0x76, 0x95, 0x52, 0xe0, 0x38, - 0xb5, 0x15, 0xba, 0x05, 0x67, 0x5a, 0x8e, 0xed, 0x11, 0xdd, 0xbb, 0x49, 0x05, 0x02, 0x53, 0x0c, - 0xd0, 0x9d, 0x99, 0x61, 0x73, 0xc1, 0xde, 0x43, 0xab, 0x69, 0x15, 0x70, 0x7a, 0x3b, 0xf4, 0x59, - 0x05, 0x2e, 0xbb, 0x9e, 0x43, 0xb4, 0xa6, 0x61, 0x35, 0x4a, 0xb6, 0x65, 0x11, 0xc6, 0x98, 0x2a, - 0xf5, 0xd0, 0x63, 0xee, 0x7c, 0xae, 0x53, 0x44, 0xdd, 0xdf, 0x2b, 0x5e, 0xae, 0x75, 0xc4, 0x8c, - 0x0f, 0xa0, 0x8c, 0x5e, 0x05, 0x68, 0x92, 0xa6, 0xed, 0xec, 0x52, 0x8e, 0x34, 0x33, 0x9b, 0xff, - 0xbe, 0xbb, 0x12, 0x60, 0xe1, 0x9f, 0x7f, 0xe4, 0x25, 0x37, 0x04, 0x62, 0x89, 0x1c, 0x9d, 0x6a, - 0xca, 0x6f, 0x45, 0x14, 0x2b, 0xe9, 0xa3, 0xb9, 0xc0, 0x56, 0x8e, 0x4d, 0xf5, 0x4a, 0x5a, 0x05, - 0x9c, 0xde, 0x0e, 0xb9, 0x30, 0xc5, 0xbe, 0x50, 0x71, 0x92, 0x5f, 0x2f, 0xcd, 0x37, 0xc8, 0xcc, - 0xc5, 0x5c, 0x93, 0x4b, 0x65, 0xff, 0xa9, 0x4a, 0x1c, 0x19, 0x4e, 0xe2, 0x0f, 0x89, 0x6a, 0x3b, - 0x21, 0xd1, 0x4b, 0xbd, 0x12, 0x95, 0x90, 0xe1, 0x24, 0x7e, 0x75, 0xaf, 0x00, 0x67, 0x52, 0x4f, - 0x49, 0xca, 0x3c, 0xf8, 0x14, 0xcf, 0xfb, 0xf9, 0xbf, 0xc4, 0xb3, 0x27, 0x63, 0x1e, 0x2b, 0x51, - 0x10, 0x8e, 0xd7, 0xa5, 0x32, 0x2c, 0xa3, 0xb6, 0x54, 0x0b, 0xdb, 0x17, 0x42, 0x19, 0xb6, 0x12, - 0x83, 0xe1, 0x44, 0x6d, 0x54, 0x12, 0xf3, 0xb1, 0x54, 0xab, 0xd0, 0x6b, 0xa0, 0xbb, 0xe4, 0x10, - 0xff, 0x76, 0x10, 0x8e, 0x4f, 0x06, 0xe2, 0x64, 0x7d, 0x3a, 0x0a, 0xfa, 0x43, 0xee, 0x45, 0x7f, - 0x38, 0x8a, 0xd5, 0x28, 0x08, 0xc7, 0xeb, 0xfa, 0xf7, 0xf4, 0x48, 0x17, 0x06, 0xc2, 0x51, 0xac, - 0xc6, 0x60, 0x38, 0x51, 0x5b, 0xfd, 0xb3, 0x7e, 0xb8, 0xaf, 0x0b, 0xc9, 0x12, 0x35, 0xd3, 0xa7, - 0xfb, 0xf0, 0x3c, 0xaf, 0xbb, 0xe5, 0x69, 0x65, 0x2c, 0xcf, 0xe1, 0xe9, 0x75, 0xbb, 0x9c, 0x6e, - 0xd6, 0x72, 0x1e, 0x9e, 0x64, 0xf7, 0xcb, 0xdf, 0x4c, 0x5f, 0xfe, 0x9c, 0xb3, 0x7a, 0xe0, 0x76, - 0x69, 0x65, 0x6c, 0x97, 0x9c, 0xb3, 0xda, 0xc5, 0xf6, 0xfa, 0xf3, 0x7e, 0xb8, 0xbf, 0x1b, 0x29, - 0x37, 0xe7, 0xfe, 0x4a, 0xe1, 0x2d, 0xc7, 0xba, 0xbf, 0xb2, 0xfc, 0xb9, 0x8f, 0x71, 0x7f, 0x75, - 0x64, 0x9f, 0xc7, 0xb3, 0xbf, 0xb2, 0x66, 0xf5, 0xb8, 0xf6, 0x57, 0xd6, 0xac, 0x76, 0xb1, 0xbf, - 0xfe, 0x26, 0x7e, 0x3e, 0x04, 0xa2, 0x76, 0x05, 0xfa, 0xf4, 0x56, 0x3b, 0x27, 0x93, 0x62, 0x56, - 0x86, 0xa5, 0xea, 0x3a, 0xa6, 0x38, 0x10, 0x86, 0x41, 0xbe, 0x7f, 0x72, 0xb2, 0x20, 0x66, 0x39, - 0xca, 0xb7, 0x24, 0x16, 0x98, 0xe8, 0x54, 0x91, 0xd6, 0x16, 0x69, 0x12, 0x47, 0x33, 0x6b, 0x9e, - 0xed, 0x68, 0x8d, 0xbc, 0xdc, 0x86, 0xbf, 0xa0, 0xc4, 0x70, 0xe1, 0x04, 0x76, 0x3a, 0x21, 0x2d, - 0xa3, 0x9e, 0x93, 0xbf, 0xb0, 0x09, 0xa9, 0x56, 0xca, 0x98, 0xe2, 0x50, 0x7f, 0x7d, 0x18, 0xa4, - 0xe0, 0xf4, 0xe8, 0x93, 0x0a, 0x4c, 0xe9, 0xf1, 0x98, 0x9c, 0xbd, 0xd8, 0x43, 0x25, 0x02, 0x7c, - 0xf2, 0x2d, 0x9f, 0x28, 0xc6, 0x49, 0xb2, 0xe8, 0xc3, 0x0a, 0x57, 0xf2, 0x05, 0xaf, 0x79, 0x62, - 0x5a, 0xaf, 0x1f, 0xd1, 0xbb, 0x77, 0xa8, 0x2d, 0x0c, 0x9f, 0x58, 0xa3, 0x04, 0xd1, 0x6b, 0x0a, - 0x9c, 0xb9, 0x9d, 0xf6, 0x36, 0x21, 0x26, 0xff, 0x56, 0xde, 0xae, 0x64, 0x3c, 0x76, 0x70, 0x09, - 0x32, 0xb5, 0x02, 0x4e, 0xef, 0x48, 0x30, 0x4b, 0x81, 0xba, 0x56, 0x7c, 0xa7, 0xb9, 0x67, 0x29, - 0xa6, 0xf7, 0x0d, 0x67, 0x29, 0x00, 0xe0, 0x28, 0x41, 0xd4, 0x82, 0x91, 0xdb, 0xbe, 0x8e, 0x5c, - 0xe8, 0xc5, 0x4a, 0x79, 0xa9, 0x4b, 0x8a, 0x76, 0x6e, 0xef, 0x15, 0x14, 0xe2, 0x90, 0x08, 0xda, - 0x82, 0xa1, 0xdb, 0x9c, 0x57, 0x08, 0x7d, 0xd6, 0x7c, 0xcf, 0xb7, 0x7f, 0xae, 0x56, 0x11, 0x45, - 0xd8, 0x47, 0x2f, 0xfb, 0x12, 0x0c, 0x1f, 0xe0, 0xe2, 0xf6, 0x59, 0x05, 0xce, 0x6c, 0x13, 0xc7, - 0x33, 0xf4, 0xf8, 0xcb, 0xd0, 0x48, 0x7e, 0x0d, 0xc5, 0xb3, 0x69, 0x08, 0xf9, 0x36, 0x49, 0x05, - 0xe1, 0xf4, 0x2e, 0xa0, 0x67, 0xa1, 0x9f, 0x78, 0x7a, 0x5d, 0x04, 0x70, 0x7e, 0x77, 0x5e, 0x97, - 0x5d, 0xee, 0xf7, 0x42, 0xff, 0xc3, 0x0c, 0x9f, 0xfa, 0x57, 0x0a, 0x24, 0x74, 0xd3, 0xe8, 0x87, - 0x14, 0x18, 0xdb, 0x24, 0x9a, 0xd7, 0x76, 0xc8, 0x75, 0x61, 0x13, 0xda, 0x77, 0x75, 0xf4, 0xe1, - 0x67, 0x8f, 0x42, 0x25, 0x3e, 0xb7, 0x24, 0x21, 0xe6, 0xc6, 0x2a, 0x41, 0xc2, 0x09, 0x19, 0x84, - 0x23, 0x3d, 0x98, 0x7d, 0x0a, 0xa6, 0x12, 0x0d, 0x0f, 0xf5, 0x4c, 0xf9, 0x6b, 0x0a, 0xa4, 0xe5, - 0x36, 0x47, 0x2f, 0xc1, 0x80, 0x56, 0xaf, 0x07, 0x29, 0x2e, 0x1f, 0xcb, 0x67, 0x37, 0x55, 0x97, - 0x23, 0x05, 0xb1, 0x9f, 0x98, 0xa3, 0x45, 0x4b, 0x80, 0xb4, 0xc8, 0xbb, 0xf4, 0x4a, 0x18, 0xa2, - 0x82, 0x3d, 0xa7, 0xcd, 0x27, 0xa0, 0x38, 0xa5, 0x85, 0xfa, 0x71, 0x05, 0x50, 0x32, 0x45, 0x09, - 0x72, 0x60, 0x58, 0xec, 0x5f, 0x7f, 0x95, 0xca, 0x39, 0xbd, 0xe9, 0x22, 0xae, 0xa1, 0xa1, 0x11, - 0x9e, 0x28, 0x70, 0x71, 0x40, 0x47, 0xfd, 0x3b, 0x05, 0xc2, 0x34, 0x78, 0xe8, 0x9d, 0x30, 0x5a, - 0x27, 0xae, 0xee, 0x18, 0x2d, 0x2f, 0x74, 0x24, 0x0d, 0x1c, 0xd2, 0xca, 0x21, 0x08, 0xcb, 0xf5, - 0x90, 0x0a, 0x83, 0x9e, 0xe6, 0xde, 0xae, 0x94, 0xc5, 0x65, 0x8f, 0x1d, 0xcd, 0x6b, 0xac, 0x04, - 0x0b, 0x48, 0x18, 0x3a, 0xb6, 0xaf, 0x8b, 0xd0, 0xb1, 0x68, 0xf3, 0x08, 0xe2, 0xe4, 0xa2, 0x83, - 0x63, 0xe4, 0xaa, 0x3f, 0x53, 0x80, 0x53, 0xb4, 0xca, 0x8a, 0x66, 0x58, 0x1e, 0xb1, 0x98, 0xdb, - 0x54, 0xce, 0x49, 0x68, 0xc0, 0xb8, 0x17, 0x71, 0x5a, 0x3e, 0xbc, 0x53, 0x6d, 0x60, 0xe9, 0x15, - 0x75, 0x55, 0x8e, 0xe2, 0x45, 0x8f, 0xf9, 0x7e, 0x6b, 0xfc, 0x5a, 0x7c, 0x9f, 0xbf, 0x55, 0x99, - 0x33, 0xda, 0x3d, 0xe1, 0x01, 0x1e, 0xe4, 0x4e, 0x8c, 0xb8, 0xa8, 0x3d, 0x0a, 0xe3, 0xc2, 0x43, - 0x82, 0xc7, 0x00, 0x16, 0xd7, 0x62, 0x76, 0xac, 0x2c, 0xc9, 0x00, 0x1c, 0xad, 0xa7, 0xfe, 0x61, - 0x01, 0xa2, 0x19, 0x1a, 0xf3, 0xce, 0x52, 0x32, 0x00, 0x72, 0xe1, 0xd8, 0x02, 0x20, 0xf3, 0x1c, - 0xe5, 0xcc, 0x9c, 0x5b, 0xbc, 0xb3, 0xcb, 0xf9, 0x89, 0x59, 0x39, 0x0e, 0x6a, 0x84, 0xd3, 0xda, - 0x7f, 0xe8, 0x69, 0x7d, 0xa7, 0xb0, 0xfc, 0x1d, 0x88, 0x84, 0xa1, 0xf6, 0x2d, 0x7f, 0xa7, 0x22, - 0x0d, 0x25, 0x2f, 0xbb, 0x79, 0x10, 0x39, 0x4e, 0xe8, 0xba, 0x88, 0x10, 0xd9, 0xee, 0x9a, 0xed, - 0x69, 0x26, 0x9b, 0x4e, 0xa1, 0x74, 0x5d, 0x91, 0x01, 0x38, 0x5a, 0x4f, 0x5d, 0x85, 0x37, 0x2f, - 0xdb, 0x5a, 0x7d, 0x41, 0x33, 0xe9, 0xd6, 0x75, 0x84, 0x59, 0x9e, 0xcb, 0x4e, 0xe6, 0xaa, 0x63, - 0x7b, 0xb6, 0x6e, 0x9b, 0xf4, 0xdc, 0xd4, 0x82, 0x58, 0xc1, 0xec, 0x21, 0xd2, 0x3f, 0x37, 0x45, - 0xac, 0x5d, 0xec, 0xc3, 0xd5, 0xdf, 0x51, 0x60, 0x48, 0xe4, 0x3d, 0xe9, 0xc2, 0xb1, 0x74, 0x13, - 0x06, 0xd8, 0xed, 0xa8, 0x17, 0xa9, 0xb4, 0xb6, 0x65, 0xdb, 0x5e, 0x24, 0x61, 0x12, 0x4f, 0x39, - 0xc6, 0x92, 0x44, 0x72, 0xf4, 0xcc, 0x1e, 0xd5, 0xd1, 0xb7, 0x0c, 0x8f, 0x30, 0xb3, 0x1b, 0xb1, - 0xf1, 0xb9, 0x3d, 0xaa, 0x54, 0x8e, 0x23, 0xb5, 0xd4, 0xff, 0x3a, 0x00, 0x57, 0xfc, 0x1c, 0x2e, - 0x71, 0x51, 0x2d, 0xe0, 0xb9, 0xbb, 0x41, 0x52, 0x19, 0x16, 0x14, 0xdf, 0xd7, 0xa9, 0xe6, 0xbb, - 0x25, 0x9f, 0x93, 0xd2, 0xc6, 0xc8, 0xe8, 0x70, 0x1a, 0x0d, 0x1e, 0x7d, 0x9d, 0x15, 0xdf, 0x20, - 0x9a, 0xe9, 0x6d, 0xf9, 0xb4, 0x0b, 0xbd, 0x44, 0x5f, 0x4f, 0xe2, 0xc3, 0xa9, 0x54, 0x98, 0x49, - 0x87, 0x9f, 0xcc, 0xc6, 0x21, 0x9a, 0x6c, 0x4f, 0xd2, 0x83, 0xbb, 0xd1, 0x4a, 0x2a, 0x46, 0x9c, - 0x41, 0x89, 0xa9, 0x1b, 0xb5, 0x1d, 0xa6, 0xbd, 0xc0, 0xc4, 0x73, 0x0c, 0x96, 0xf8, 0x2a, 0x78, - 0xab, 0x58, 0x89, 0x82, 0x70, 0xbc, 0x2e, 0x7a, 0x1c, 0x26, 0x98, 0x89, 0x4c, 0x18, 0xce, 0x73, - 0x20, 0x0c, 0xea, 0xb4, 0x1a, 0x81, 0xe0, 0x58, 0x4d, 0xf4, 0x3d, 0x0a, 0x4c, 0x1b, 0xb2, 0x33, - 0x8d, 0x3f, 0xfa, 0x7c, 0x99, 0x51, 0xd8, 0xf3, 0x92, 0xbf, 0x8d, 0x53, 0xd0, 0xe2, 0x54, 0x62, - 0x68, 0x19, 0xa6, 0x85, 0x47, 0x74, 0x74, 0x0f, 0xf0, 0x58, 0x6f, 0x6c, 0x4d, 0xcb, 0x29, 0x70, - 0x9c, 0xda, 0x4a, 0xfd, 0x48, 0x01, 0xc6, 0x0e, 0x99, 0xc4, 0xb4, 0x2d, 0xc9, 0x1c, 0x3d, 0xf8, - 0x2d, 0xca, 0x54, 0xbb, 0x10, 0x3b, 0xd0, 0xf3, 0x30, 0xd1, 0x66, 0x13, 0xe1, 0x47, 0x42, 0x13, - 0xdf, 0xf4, 0xb7, 0xd1, 0x95, 0x5b, 0x8f, 0x40, 0xee, 0xed, 0x15, 0x67, 0x65, 0xf4, 0x51, 0x28, - 0x8e, 0xe1, 0x51, 0x7f, 0xa3, 0x00, 0x48, 0xae, 0xbe, 0x64, 0x6a, 0xdb, 0xb6, 0x83, 0x3e, 0xaf, - 0xc0, 0x98, 0x2e, 0xa7, 0x8d, 0xe2, 0x02, 0xd6, 0xf3, 0xbd, 0x0e, 0x96, 0xa3, 0x9f, 0x93, 0xf3, - 0x41, 0x71, 0x41, 0xd8, 0x77, 0xb7, 0x1e, 0x93, 0x41, 0xf7, 0x62, 0xbf, 0x71, 0xa4, 0x4f, 0xb3, - 0x1f, 0x53, 0x60, 0x2a, 0x81, 0x29, 0x45, 0x32, 0x7e, 0x21, 0x1a, 0x03, 0xfd, 0x48, 0x12, 0x44, - 0xca, 0xf2, 0xf5, 0x4f, 0x0e, 0xc0, 0xe9, 0x94, 0x15, 0x65, 0x26, 0x36, 0x24, 0x26, 0x5d, 0xf6, - 0x62, 0x62, 0x93, 0x90, 0x54, 0x03, 0x13, 0x9b, 0x38, 0x04, 0x27, 0xe8, 0xa2, 0x67, 0xa1, 0x4f, - 0x77, 0x0c, 0xb1, 0x69, 0x1f, 0xcd, 0x35, 0x05, 0xb8, 0xb2, 0x30, 0x2a, 0x28, 0xf6, 0x95, 0x70, - 0x05, 0x53, 0x84, 0xf4, 0x2c, 0x96, 0x8f, 0x11, 0x5f, 0x60, 0x65, 0x67, 0xb1, 0x7c, 0xda, 0xb8, - 0x38, 0x5a, 0x0f, 0x3d, 0x0f, 0x33, 0xe2, 0xa6, 0xea, 0x47, 0x32, 0xb1, 0x2d, 0xd7, 0xa3, 0x1c, - 0xdf, 0x13, 0x32, 0x05, 0xb3, 0x7e, 0xbd, 0x99, 0x51, 0x07, 0x67, 0xb6, 0x46, 0xdf, 0x0d, 0x13, - 0x11, 0xee, 0xe1, 0x47, 0x25, 0xcb, 0xe9, 0x32, 0x23, 0x63, 0xe2, 0xbc, 0x32, 0x5a, 0x86, 0x63, - 0xd4, 0xd0, 0xff, 0xad, 0xc0, 0x54, 0xb0, 0x51, 0x77, 0xf9, 0x96, 0xf7, 0x33, 0xf8, 0x2d, 0x1d, - 0xcd, 0x17, 0x24, 0xa5, 0x57, 0x8d, 0x13, 0xc2, 0x49, 0xda, 0xea, 0x8f, 0x0e, 0x82, 0x9c, 0xd1, - 0x19, 0xad, 0xf4, 0xa2, 0x97, 0x0c, 0xf7, 0x80, 0xaf, 0x9b, 0x5c, 0x81, 0xbe, 0x46, 0xab, 0x9d, - 0x53, 0x31, 0x19, 0xa0, 0xbb, 0x4e, 0xd1, 0x35, 0x5a, 0x6d, 0xf4, 0x6c, 0xa0, 0xea, 0xcc, 0xa7, - 0x8c, 0x0c, 0x5c, 0x25, 0x63, 0xea, 0x4e, 0x9f, 0xbd, 0xf7, 0x67, 0xb2, 0xf7, 0x26, 0x0c, 0xb9, - 0x42, 0x0f, 0x3a, 0x90, 0x3f, 0x8c, 0xa4, 0x34, 0xd3, 0x42, 0xef, 0xc9, 0x35, 0x34, 0xbe, 0x5a, - 0xd4, 0xa7, 0x41, 0x2f, 0x82, 0x6d, 0x76, 0x2e, 0x89, 0x20, 0x49, 0xec, 0x22, 0xb8, 0xce, 0x4a, - 0xb0, 0x80, 0x24, 0x84, 0xb9, 0xa1, 0x6e, 0x84, 0x39, 0xf4, 0xb9, 0x38, 0xff, 0x1e, 0x66, 0xbb, - 0xef, 0x99, 0x1e, 0x87, 0xf3, 0x7f, 0x18, 0xe3, 0xfe, 0xbf, 0xc2, 0xc3, 0x4f, 0x5a, 0x2c, 0x74, - 0x1f, 0x0c, 0xb0, 0x28, 0x4a, 0x42, 0x0e, 0x08, 0x94, 0x1b, 0x2c, 0x8e, 0x0e, 0xe6, 0x30, 0x54, - 0x13, 0xa1, 0x03, 0xf3, 0x6d, 0x7a, 0x66, 0xdb, 0x28, 0xe8, 0x49, 0x71, 0x06, 0xaf, 0x44, 0x7c, - 0x22, 0xd3, 0xee, 0x10, 0xeb, 0x30, 0xd4, 0x34, 0x2c, 0x66, 0xee, 0x91, 0x4f, 0x89, 0xce, 0x4d, - 0xb0, 0x38, 0x0a, 0xec, 0xe3, 0x52, 0xff, 0xbc, 0x40, 0x19, 0x44, 0x78, 0xa9, 0xdf, 0xe5, 0x81, - 0x70, 0x85, 0xf7, 0xb5, 0x92, 0x5f, 0x09, 0x28, 0x21, 0x9d, 0x0f, 0x10, 0x72, 0x43, 0x85, 0xf0, - 0x37, 0x96, 0x88, 0x51, 0xd2, 0x9e, 0xd1, 0x24, 0xcf, 0x19, 0x56, 0xdd, 0xbe, 0x23, 0xa6, 0xb7, - 0x57, 0xd2, 0x6b, 0x01, 0x42, 0x4e, 0x3a, 0xfc, 0x8d, 0x25, 0x62, 0xf4, 0x48, 0x62, 0xc9, 0x38, - 0x2c, 0x96, 0x89, 0x57, 0xf4, 0x8d, 0xe7, 0xd9, 0x16, 0x76, 0xc7, 0xec, 0x48, 0x2a, 0x65, 0xd4, - 0xc1, 0x99, 0xad, 0xd5, 0x9f, 0x53, 0xe0, 0x4c, 0xea, 0x54, 0xa0, 0xeb, 0x30, 0x15, 0x9a, 0xc3, - 0xca, 0x42, 0xc2, 0x70, 0xc8, 0xe3, 0x6f, 0xc6, 0x2b, 0xe0, 0x64, 0x1b, 0x9e, 0xdf, 0x2c, 0x21, - 0x84, 0x08, 0x5b, 0x5a, 0xf9, 0xaa, 0x25, 0x83, 0x71, 0x5a, 0x1b, 0xf5, 0xc5, 0x48, 0x67, 0xc3, - 0xc9, 0xa2, 0x5f, 0xc6, 0x06, 0x69, 0x04, 0x3e, 0xe9, 0xc1, 0x97, 0xb1, 0x40, 0x0b, 0x31, 0x87, - 0xa1, 0x4b, 0x72, 0xa8, 0x8c, 0x80, 0xbb, 0xfb, 0xe1, 0x32, 0xd4, 0x5f, 0x53, 0xe0, 0xdc, 0x8a, - 0x66, 0xb5, 0x35, 0x93, 0x3b, 0xbf, 0x57, 0x6d, 0xdb, 0x14, 0xd3, 0xd4, 0x29, 0xdf, 0xba, 0xf2, - 0x7a, 0xc9, 0xb7, 0xfe, 0x7e, 0x38, 0x97, 0x61, 0x7f, 0x83, 0xca, 0x30, 0xe6, 0xde, 0xd1, 0x5a, - 0x0b, 0x64, 0x4b, 0xdb, 0x36, 0x44, 0x60, 0x2d, 0x6e, 0xa6, 0x3d, 0x56, 0x93, 0xca, 0xef, 0xc5, - 0x7e, 0xe3, 0x48, 0x2b, 0xd5, 0x03, 0x10, 0xe6, 0xfc, 0x86, 0xd5, 0x40, 0x9b, 0x30, 0xac, 0x99, - 0xc4, 0xf1, 0xc2, 0xe0, 0x91, 0xdf, 0x9e, 0x4b, 0x4f, 0x2b, 0x70, 0x70, 0xf7, 0x35, 0xff, 0x17, - 0x0e, 0x70, 0xab, 0xff, 0x48, 0x81, 0xb3, 0xe9, 0xa1, 0x94, 0xba, 0xb8, 0x16, 0x35, 0x61, 0xd4, - 0x09, 0x9b, 0x89, 0x8f, 0xf6, 0x5d, 0x72, 0x86, 0x0a, 0xc9, 0xf1, 0x86, 0x5e, 0x04, 0x4b, 0x8e, - 0xed, 0xfa, 0x3b, 0x37, 0x9e, 0xb4, 0x22, 0xd0, 0x8a, 0x49, 0x3d, 0xc1, 0x32, 0x7e, 0x96, 0x40, - 0x86, 0x52, 0x77, 0x5b, 0x9a, 0x4e, 0xea, 0x27, 0x9c, 0x53, 0xfd, 0x08, 0xb2, 0x36, 0xa4, 0xf7, - 0xfd, 0x78, 0x13, 0xc8, 0x64, 0xd0, 0x3c, 0x38, 0x81, 0x4c, 0x7a, 0xc3, 0x37, 0x48, 0x66, 0x83, - 0xf4, 0xce, 0x67, 0x38, 0xbe, 0xff, 0xe7, 0xc1, 0xac, 0xd1, 0x1e, 0x32, 0x31, 0xfb, 0xf6, 0x31, - 0x26, 0x66, 0x9f, 0xf8, 0x66, 0x52, 0xf6, 0x94, 0xa4, 0xec, 0xb1, 0x44, 0xe1, 0x83, 0x27, 0x94, - 0x28, 0xfc, 0x15, 0x18, 0x6c, 0x69, 0x0e, 0xb1, 0xfc, 0xb7, 0xdc, 0x4a, 0x3e, 0x43, 0x83, 0x70, - 0xa3, 0x85, 0x5c, 0x30, 0xf8, 0x24, 0xab, 0x8c, 0x00, 0x16, 0x84, 0x52, 0x82, 0xa7, 0x0c, 0x1f, - 0x57, 0x74, 0xc2, 0x30, 0x45, 0xf9, 0xc8, 0x71, 0xa4, 0x28, 0x57, 0xff, 0x56, 0x81, 0x8b, 0x9d, - 0xd8, 0x12, 0x53, 0xa0, 0xe8, 0xb1, 0xcf, 0xb0, 0x17, 0x05, 0x4a, 0x82, 0xdb, 0x06, 0x0a, 0x94, - 0x38, 0x04, 0x27, 0xe8, 0xa2, 0xf7, 0x02, 0xb2, 0x37, 0xb8, 0x5d, 0xcf, 0x75, 0x4a, 0x83, 0x7b, - 0xd3, 0x16, 0x98, 0xaf, 0x42, 0x90, 0xd7, 0xf2, 0x56, 0xa2, 0x06, 0x4e, 0x69, 0xa5, 0xfe, 0x4a, - 0x01, 0x60, 0x95, 0x78, 0x77, 0x6c, 0xe7, 0x36, 0x3d, 0xe3, 0x2f, 0x46, 0x9e, 0x0e, 0x86, 0xbf, - 0x71, 0xf1, 0x28, 0x2f, 0x42, 0x7f, 0xcb, 0xae, 0xbb, 0xe2, 0xfe, 0xc1, 0x3a, 0xc2, 0x5c, 0x35, - 0x58, 0x29, 0x2a, 0xc2, 0x00, 0x33, 0x7a, 0x12, 0x17, 0x68, 0xf6, 0xf0, 0xc0, 0x1e, 0x57, 0x30, - 0x2f, 0xa7, 0x1c, 0x52, 0xc4, 0x34, 0x70, 0xc5, 0xe3, 0xce, 0x18, 0x8f, 0x21, 0xce, 0xcb, 0x70, - 0x00, 0x45, 0x8f, 0x03, 0x18, 0xad, 0x25, 0xad, 0x69, 0x98, 0x86, 0xf8, 0x5c, 0x47, 0x98, 0x46, - 0x1c, 0x2a, 0x55, 0xbf, 0xf4, 0xde, 0x5e, 0x71, 0x58, 0xfc, 0xda, 0xc5, 0x52, 0x6d, 0xf5, 0x8b, - 0x0a, 0x4c, 0x86, 0x93, 0x27, 0xb6, 0x8a, 0xdf, 0x73, 0x1e, 0x0c, 0x38, 0xb3, 0xe7, 0x3c, 0x6c, - 0x7f, 0xe7, 0x9e, 0x73, 0x05, 0x56, 0x56, 0xcf, 0x1f, 0x82, 0x51, 0xc2, 0x43, 0x1e, 0x55, 0xca, - 0x98, 0xf3, 0xb8, 0x11, 0x7e, 0x9d, 0x5b, 0x0c, 0x8b, 0xb1, 0x5c, 0x47, 0xfd, 0xfb, 0x3e, 0x18, - 0x5b, 0x6d, 0x18, 0xd6, 0x8e, 0x1f, 0xdb, 0x29, 0x78, 0x78, 0x57, 0x8e, 0xe7, 0xe1, 0xfd, 0x79, - 0x98, 0x31, 0xe5, 0x67, 0x2e, 0x2e, 0x38, 0x69, 0x56, 0x23, 0x98, 0x01, 0x76, 0x8f, 0x59, 0xce, - 0xa8, 0x83, 0x33, 0x5b, 0x23, 0x0f, 0x06, 0x75, 0x3f, 0xbf, 0x62, 0xee, 0x78, 0x45, 0xf2, 0x5c, - 0xcc, 0xc9, 0xa1, 0x3b, 0x02, 0x9e, 0x27, 0xb6, 0xa7, 0xa0, 0x85, 0x3e, 0xaa, 0xc0, 0x19, 0xb2, - 0xc3, 0x43, 0xd7, 0xac, 0x39, 0xda, 0xe6, 0xa6, 0xa1, 0x0b, 0x8f, 0x3f, 0xbe, 0x13, 0x97, 0xf7, - 0xf7, 0x8a, 0x67, 0x16, 0xd3, 0x2a, 0xdc, 0xdb, 0x2b, 0x5e, 0x4b, 0x8d, 0x24, 0xc4, 0x56, 0x33, - 0xb5, 0x09, 0x4e, 0x27, 0x35, 0xfb, 0x18, 0x8c, 0x1e, 0xc2, 0x4f, 0x3c, 0x12, 0x2f, 0xe8, 0x57, - 0x0b, 0x30, 0x46, 0xb7, 0xdb, 0xb2, 0xad, 0x6b, 0x66, 0x79, 0xb5, 0x86, 0x1e, 0x88, 0x87, 0x49, - 0x0c, 0x9e, 0x18, 0x13, 0xa1, 0x12, 0x97, 0x61, 0x7a, 0xd3, 0x76, 0x74, 0xb2, 0x56, 0xaa, 0xae, - 0xd9, 0xc2, 0xf8, 0xac, 0xbc, 0x5a, 0x13, 0xf7, 0x3a, 0xf6, 0xe4, 0xb1, 0x94, 0x02, 0xc7, 0xa9, - 0xad, 0xd0, 0x2d, 0x38, 0x13, 0x96, 0xaf, 0xb7, 0xb8, 0xc3, 0x02, 0x45, 0xd7, 0x17, 0x3a, 0x5c, - 0x2c, 0xa5, 0x55, 0xc0, 0xe9, 0xed, 0x90, 0x06, 0x17, 0xc4, 0xdb, 0xca, 0x92, 0xed, 0xdc, 0xd1, - 0x9c, 0x7a, 0x14, 0x6d, 0x7f, 0x98, 0xab, 0xbc, 0x9c, 0x5d, 0x0d, 0x77, 0xc2, 0xa1, 0xfe, 0x9d, - 0x02, 0xd1, 0x20, 0x94, 0xe8, 0x3c, 0xf4, 0x39, 0x22, 0x7d, 0x9f, 0x08, 0xc6, 0x48, 0xaf, 0x08, - 0xb4, 0x0c, 0xcd, 0x01, 0x38, 0x61, 0x24, 0xcc, 0x42, 0x98, 0xd6, 0x42, 0x8a, 0x61, 0x29, 0xd5, - 0xa0, 0xa8, 0x3c, 0xad, 0x21, 0x18, 0x1e, 0x43, 0xb5, 0xa6, 0x35, 0x30, 0x2d, 0x63, 0xf9, 0x4b, - 0x8c, 0x06, 0x71, 0x7d, 0x75, 0x34, 0xcf, 0x5f, 0xc2, 0x4a, 0xb0, 0x80, 0x20, 0x0d, 0xc6, 0x5b, - 0x6d, 0x53, 0xc4, 0x53, 0xa2, 0x57, 0x1f, 0xae, 0x36, 0xbc, 0x9a, 0x96, 0x9c, 0x8f, 0xad, 0x7e, - 0x6a, 0x86, 0xbe, 0xaa, 0x8c, 0x02, 0x47, 0x31, 0xaa, 0x3f, 0x31, 0x08, 0x52, 0x78, 0x9d, 0x43, - 0x48, 0xa1, 0x3f, 0xad, 0xc0, 0xb4, 0x6e, 0x1a, 0xc4, 0xf2, 0x62, 0x91, 0x2a, 0xf8, 0xf1, 0xb1, - 0x9e, 0x2b, 0xee, 0x4f, 0x8b, 0x58, 0x95, 0xb2, 0x70, 0x6f, 0x29, 0xa5, 0x20, 0x17, 0x2e, 0x40, - 0x29, 0x10, 0x9c, 0xda, 0x19, 0x36, 0x1e, 0x56, 0x5e, 0x29, 0xcb, 0xd1, 0x33, 0x4b, 0xa2, 0x0c, - 0x07, 0x50, 0xca, 0x79, 0x1b, 0x8e, 0xdd, 0x6e, 0xb9, 0x25, 0xe6, 0xc5, 0xca, 0x17, 0x85, 0x71, - 0xde, 0xeb, 0x61, 0x31, 0x96, 0xeb, 0xa0, 0x47, 0x60, 0x8c, 0xff, 0xac, 0x3a, 0x64, 0xd3, 0xd8, - 0x11, 0x87, 0x12, 0x53, 0x9e, 0x5e, 0x97, 0xca, 0x71, 0xa4, 0x16, 0x8b, 0xdf, 0xe6, 0xba, 0x6d, - 0xe2, 0xac, 0xe3, 0x65, 0x91, 0xe3, 0x99, 0xc7, 0x6f, 0xf3, 0x0b, 0x71, 0x08, 0x47, 0x3f, 0xa2, - 0xc0, 0x84, 0x43, 0x5e, 0x69, 0x1b, 0x0e, 0x15, 0x61, 0x34, 0xa3, 0xe9, 0x8a, 0x18, 0x47, 0xb8, - 0xb7, 0xb8, 0x4a, 0x73, 0x38, 0x82, 0x94, 0x33, 0xc8, 0xc0, 0x94, 0x23, 0x0a, 0xc4, 0xb1, 0x1e, - 0xd0, 0xa9, 0x72, 0x8d, 0x86, 0x65, 0x58, 0x8d, 0x79, 0xb3, 0xe1, 0x8a, 0x44, 0x56, 0x5c, 0xe7, - 0x18, 0x16, 0x63, 0xb9, 0x0e, 0x7a, 0x14, 0xc6, 0xdb, 0x2e, 0x65, 0x7b, 0x4d, 0xc2, 0xe7, 0x77, - 0x24, 0xb4, 0x75, 0x59, 0x97, 0x01, 0x38, 0x5a, 0x0f, 0x3d, 0x0e, 0x13, 0x7e, 0x81, 0x98, 0x65, - 0xe0, 0xd9, 0x54, 0xd8, 0xdb, 0x64, 0x04, 0x82, 0x63, 0x35, 0x67, 0xe7, 0xe1, 0x74, 0xca, 0x30, - 0x0f, 0xc5, 0x5b, 0xff, 0x97, 0x02, 0x67, 0xb8, 0xd4, 0xe5, 0x67, 0x87, 0xf6, 0x73, 0x5b, 0xa4, - 0xa7, 0x89, 0x50, 0x8e, 0x35, 0x4d, 0xc4, 0x37, 0x20, 0x1d, 0x86, 0xfa, 0xb3, 0x05, 0x78, 0xf3, - 0x81, 0xdf, 0x25, 0xfa, 0xff, 0x14, 0x18, 0x25, 0x3b, 0x9e, 0xa3, 0x05, 0xae, 0xfe, 0x74, 0x93, - 0x6e, 0x1e, 0x0b, 0x13, 0x98, 0x5b, 0x0c, 0x09, 0xf1, 0x8d, 0x1b, 0x5c, 0xa5, 0x24, 0x08, 0x96, - 0xfb, 0x43, 0xb9, 0x2d, 0xcf, 0x36, 0x24, 0x1b, 0xc5, 0x09, 0x2e, 0x28, 0x20, 0xb3, 0x4f, 0xc2, - 0x64, 0x1c, 0xf3, 0xa1, 0xf6, 0xca, 0x17, 0x14, 0x48, 0x0d, 0xc7, 0x89, 0x4a, 0x30, 0xa5, 0xb5, - 0x3d, 0x3b, 0xf2, 0xae, 0x27, 0x44, 0x49, 0x66, 0xf4, 0x3d, 0x1f, 0x07, 0xe2, 0x64, 0x7d, 0xae, - 0x80, 0xb5, 0xda, 0x9a, 0x19, 0x45, 0xc3, 0x05, 0x2e, 0xa1, 0x80, 0x4d, 0x80, 0x71, 0x5a, 0x1b, - 0xf5, 0x9f, 0x2a, 0x70, 0x26, 0x55, 0x2d, 0xd9, 0x85, 0x2e, 0x2e, 0x7d, 0xdb, 0x17, 0x8e, 0x73, - 0xdb, 0xab, 0xbf, 0x5c, 0x80, 0xa1, 0xaa, 0x63, 0xd3, 0xa3, 0xed, 0x04, 0x34, 0x70, 0x5a, 0x44, - 0x03, 0x97, 0x4b, 0xbf, 0x20, 0x3a, 0x9b, 0xa9, 0x72, 0x33, 0x62, 0x2a, 0xb7, 0xf9, 0x5e, 0x88, - 0x74, 0xd6, 0xb1, 0x7d, 0x59, 0x81, 0x51, 0x51, 0xf3, 0x04, 0x94, 0x6a, 0xdf, 0x15, 0x55, 0xaa, - 0xbd, 0xa7, 0x87, 0x71, 0x65, 0x68, 0xd1, 0x3e, 0xab, 0xc0, 0xb8, 0xa8, 0xb1, 0x42, 0x9a, 0x1b, - 0xc4, 0x41, 0x4b, 0x30, 0xe4, 0xb6, 0xd9, 0x42, 0x8a, 0x01, 0x5d, 0x90, 0xc5, 0x23, 0x67, 0x43, - 0xd3, 0x69, 0xf7, 0x6b, 0xbc, 0x8a, 0x94, 0x09, 0x99, 0x17, 0x60, 0xbf, 0x31, 0xdd, 0xfb, 0x8e, - 0x6d, 0x26, 0xc2, 0xd6, 0x63, 0xdb, 0x24, 0x98, 0x41, 0xe8, 0x3d, 0x8f, 0xfe, 0xf5, 0xef, 0x70, - 0xec, 0x9e, 0x47, 0xc1, 0x2e, 0xe6, 0xe5, 0xea, 0x17, 0x07, 0x82, 0xc9, 0x66, 0x97, 0xfa, 0x1b, - 0x30, 0xa2, 0x3b, 0x44, 0xf3, 0x48, 0x7d, 0x61, 0xb7, 0x9b, 0xce, 0x31, 0x39, 0xa0, 0xe4, 0xb7, - 0xc0, 0x61, 0x63, 0x7a, 0xe4, 0xca, 0x06, 0x9e, 0x85, 0x50, 0x3a, 0xc9, 0x34, 0xee, 0xfc, 0x76, - 0x18, 0xb0, 0xef, 0x58, 0x81, 0x73, 0x48, 0x47, 0xc2, 0x6c, 0x28, 0xb7, 0x68, 0x6d, 0xcc, 0x1b, - 0xc9, 0x69, 0x1b, 0xfa, 0x3b, 0xa4, 0x6d, 0x30, 0x61, 0xa8, 0xc9, 0x96, 0xa1, 0xa7, 0xc4, 0xb8, - 0x91, 0x05, 0x0d, 0x97, 0x88, 0xff, 0x76, 0xb1, 0x4f, 0x82, 0x8a, 0x4e, 0x96, 0xaf, 0xd1, 0x91, - 0x45, 0xa7, 0x40, 0xcd, 0x83, 0x43, 0x38, 0xda, 0x8d, 0xe6, 0x03, 0x19, 0xca, 0xaf, 0x27, 0x15, - 0xdd, 0x93, 0x52, 0x80, 0xf0, 0xa9, 0xcf, 0xca, 0x09, 0x82, 0xbe, 0xa0, 0xc0, 0xb9, 0x7a, 0x7a, - 0xc2, 0x35, 0xf1, 0x54, 0x9e, 0xcb, 0x31, 0x3b, 0x23, 0x87, 0xdb, 0x42, 0x51, 0x4c, 0x58, 0x56, - 0x92, 0x37, 0x9c, 0xd5, 0x19, 0xf5, 0xfb, 0xfb, 0x83, 0xaf, 0x49, 0x68, 0x3a, 0xd2, 0xf5, 0x50, - 0x4a, 0x1e, 0x3d, 0x14, 0x7a, 0x87, 0x9f, 0xfe, 0x8b, 0x6f, 0xd7, 0x4b, 0xf1, 0xf4, 0x5f, 0x63, - 0x82, 0x74, 0x24, 0xe5, 0x57, 0x1b, 0x4e, 0xbb, 0x9e, 0x66, 0x92, 0x9a, 0x21, 0x1e, 0x07, 0x5d, - 0x4f, 0x6b, 0xb6, 0x72, 0xe4, 0xdf, 0xe2, 0x81, 0x1a, 0x92, 0xa8, 0x70, 0x1a, 0x7e, 0xf4, 0xbd, - 0x2c, 0xf8, 0x9e, 0x66, 0xb2, 0xc7, 0x53, 0x9e, 0x83, 0x34, 0x24, 0x7e, 0x78, 0x73, 0x77, 0x11, - 0x5a, 0x2f, 0x1d, 0x1f, 0xce, 0xa4, 0x84, 0x5e, 0x85, 0x33, 0xf4, 0x00, 0x9c, 0xd7, 0x3d, 0x63, - 0xdb, 0xf0, 0x76, 0xc3, 0x2e, 0x1c, 0x3e, 0xe9, 0x16, 0xbb, 0x6d, 0x2f, 0xa7, 0x21, 0xc3, 0xe9, - 0x34, 0xd4, 0xbf, 0x51, 0x00, 0x25, 0xf7, 0x3a, 0x32, 0x61, 0xb8, 0xee, 0x47, 0x4e, 0x50, 0x8e, - 0x24, 0xab, 0x4e, 0x70, 0x84, 0x04, 0x01, 0x17, 0x02, 0x0a, 0xc8, 0x86, 0x91, 0x3b, 0x5b, 0x86, - 0x47, 0x4c, 0xc3, 0xf5, 0x8e, 0x28, 0x89, 0x4f, 0x10, 0x13, 0xfe, 0x39, 0x1f, 0x31, 0x0e, 0x69, - 0xa8, 0x9f, 0xea, 0x87, 0xe1, 0x20, 0x99, 0xe6, 0xc1, 0x66, 0xd6, 0x6d, 0x40, 0xba, 0x14, 0x10, - 0xb0, 0x17, 0x9d, 0x29, 0x93, 0x81, 0x4a, 0x09, 0x64, 0x38, 0x85, 0x00, 0x7a, 0x15, 0xa6, 0x0d, - 0x6b, 0xd3, 0xd1, 0x82, 0x70, 0x87, 0x25, 0x5f, 0x51, 0x96, 0x83, 0x30, 0xbb, 0x45, 0x57, 0x52, - 0xd0, 0xe1, 0x54, 0x22, 0x88, 0xc0, 0x10, 0xcf, 0x67, 0xec, 0xbf, 0xba, 0xe4, 0xd2, 0xd5, 0x73, - 0x59, 0x33, 0x64, 0xef, 0xbe, 0xec, 0xe9, 0xe3, 0xe6, 0xc1, 0x69, 0xf9, 0xff, 0xfe, 0x83, 0x94, - 0xd8, 0xf7, 0xa5, 0xfc, 0xf4, 0xc2, 0xb7, 0x2d, 0x1e, 0x9c, 0x36, 0x5a, 0x88, 0xe3, 0x04, 0xd5, - 0xdf, 0x55, 0x60, 0x80, 0xc7, 0x00, 0x3b, 0x7e, 0x51, 0xf3, 0xfd, 0x11, 0x51, 0xf3, 0x89, 0x3c, - 0x83, 0x64, 0x5d, 0xcd, 0xcc, 0xc8, 0xff, 0x3b, 0x0a, 0x8c, 0xb0, 0x1a, 0x27, 0x20, 0xfb, 0xbd, - 0x14, 0x95, 0xfd, 0x1e, 0xcb, 0x3d, 0x9a, 0x0c, 0xc9, 0xef, 0x77, 0xfb, 0xc4, 0x58, 0x98, 0x68, - 0x55, 0x81, 0xd3, 0xc2, 0x31, 0x76, 0xd9, 0xd8, 0x24, 0x74, 0x8b, 0x97, 0xb5, 0x5d, 0x57, 0xb8, - 0x6a, 0xf0, 0xa0, 0x33, 0x49, 0x30, 0x4e, 0x6b, 0x83, 0x7e, 0x55, 0xa1, 0x42, 0x8c, 0xe7, 0x18, - 0x7a, 0x4f, 0x8f, 0xc1, 0x41, 0xdf, 0xe6, 0x56, 0x38, 0x32, 0x7e, 0x37, 0x5d, 0x0f, 0xa5, 0x19, - 0x56, 0x7a, 0x6f, 0xaf, 0x58, 0x4c, 0xd1, 0x19, 0x87, 0x59, 0xa9, 0x5d, 0xef, 0xa3, 0x7f, 0xd1, - 0xb1, 0x0a, 0xbb, 0x8d, 0xf9, 0x3d, 0x46, 0x37, 0x60, 0xc0, 0xd5, 0xed, 0x96, 0xef, 0x5a, 0x7d, - 0x5f, 0x9a, 0x6e, 0x30, 0xae, 0x16, 0x0c, 0x26, 0xb8, 0x46, 0x5b, 0x62, 0x8e, 0x60, 0xf6, 0x65, - 0x18, 0x93, 0x7b, 0x9e, 0x72, 0xf7, 0x2d, 0x47, 0x2d, 0xe6, 0x0e, 0x69, 0x1c, 0x26, 0xdf, 0x95, - 0xff, 0xa8, 0x0f, 0x06, 0x31, 0x69, 0x88, 0x94, 0x69, 0x07, 0xdc, 0x39, 0x0d, 0x3f, 0xfd, 0x6f, - 0x21, 0xbf, 0x1f, 0x9e, 0x9c, 0x14, 0xe7, 0x05, 0xdb, 0x92, 0xe6, 0x40, 0xce, 0x00, 0x8c, 0xac, - 0x20, 0x91, 0x54, 0x5f, 0x7e, 0x83, 0x5a, 0x3e, 0xb0, 0x6e, 0x52, 0x47, 0xa1, 0x1f, 0x56, 0x00, - 0x69, 0xba, 0x4e, 0x5c, 0x17, 0x13, 0x97, 0xce, 0x3d, 0x17, 0x56, 0x39, 0x97, 0xcd, 0x17, 0x15, - 0x3b, 0x8e, 0x2d, 0x14, 0xdb, 0x12, 0x20, 0x17, 0xa7, 0x10, 0xef, 0x25, 0x9d, 0xd5, 0xef, 0x2b, - 0x30, 0x16, 0xc9, 0x16, 0xd6, 0x0c, 0x75, 0xe9, 0xf9, 0x4d, 0x76, 0x7c, 0xef, 0xaf, 0x0b, 0x1d, - 0x2a, 0x71, 0xfd, 0xfc, 0xad, 0x20, 0xdd, 0xc5, 0xd1, 0x24, 0x16, 0x53, 0x3f, 0xad, 0xc0, 0x59, - 0x7f, 0x40, 0xd1, 0xb8, 0xe6, 0xe8, 0x2a, 0x0c, 0x6b, 0x2d, 0x83, 0x29, 0x7a, 0x65, 0x55, 0xf9, - 0x7c, 0xb5, 0xc2, 0xca, 0x70, 0x00, 0x8d, 0xe4, 0x58, 0x2e, 0x1c, 0x98, 0x63, 0xf9, 0x2d, 0x52, - 0xd6, 0xe8, 0x81, 0x50, 0x76, 0x09, 0x08, 0x73, 0x63, 0x4e, 0xf5, 0x5d, 0x30, 0x52, 0xab, 0xdd, - 0xe0, 0x4b, 0x7a, 0x88, 0x17, 0x1f, 0xf5, 0x13, 0x7d, 0x30, 0x2e, 0x12, 0x34, 0x18, 0x4c, 0x05, - 0x74, 0x02, 0xe7, 0xdc, 0x1a, 0x8c, 0xb8, 0xc1, 0x1b, 0x46, 0x21, 0x9b, 0x4f, 0x05, 0xcf, 0x12, - 0xf1, 0x2c, 0x7b, 0xe1, 0xf3, 0x45, 0x88, 0x08, 0xdd, 0x84, 0xc1, 0x57, 0x28, 0xcf, 0xf5, 0xbf, - 0xd5, 0xae, 0x58, 0x5f, 0xf0, 0x21, 0x32, 0x76, 0xed, 0x62, 0x81, 0x02, 0xb9, 0xcc, 0x3d, 0x91, - 0x09, 0x81, 0xbd, 0x84, 0xea, 0x8c, 0xcc, 0x6c, 0x90, 0x33, 0x7e, 0x4c, 0x78, 0x39, 0xb2, 0x5f, - 0x38, 0x20, 0xc4, 0x52, 0x84, 0x46, 0x5a, 0xbc, 0x41, 0x52, 0x84, 0x46, 0xfa, 0x9c, 0x71, 0x5c, - 0x3f, 0x06, 0x67, 0x52, 0x27, 0xe3, 0x60, 0x11, 0x5b, 0xfd, 0x67, 0x05, 0xe8, 0xaf, 0x11, 0x52, - 0x3f, 0x81, 0x9d, 0xf9, 0x52, 0x44, 0x02, 0xfb, 0xf6, 0xdc, 0x49, 0x4a, 0xb3, 0x34, 0x7d, 0x9b, - 0x31, 0x4d, 0xdf, 0x93, 0xb9, 0x29, 0x74, 0x56, 0xf3, 0xfd, 0x55, 0x01, 0x86, 0x68, 0xb5, 0xf2, - 0x6a, 0x0d, 0x35, 0xa5, 0xad, 0x5c, 0xc8, 0x2f, 0x3e, 0x0b, 0x74, 0x07, 0x6d, 0x62, 0xfa, 0xe5, - 0x18, 0x22, 0xf1, 0x8b, 0x18, 0x64, 0xe5, 0x08, 0xc8, 0x89, 0xd8, 0x13, 0x8c, 0xa8, 0x9f, 0x57, - 0x06, 0x07, 0x84, 0xd0, 0x1d, 0xe9, 0x4e, 0xda, 0x9f, 0xdf, 0x76, 0x37, 0x9d, 0x68, 0x87, 0xeb, - 0xa9, 0xfa, 0x09, 0x05, 0x4e, 0xc5, 0x5a, 0x75, 0x71, 0x69, 0x3c, 0x16, 0x06, 0xa8, 0x7e, 0x5d, - 0xa1, 0xdf, 0x58, 0xca, 0x08, 0xba, 0xe8, 0x51, 0x98, 0x00, 0xac, 0xd0, 0x31, 0x01, 0xd8, 0x45, - 0xe8, 0xa7, 0x92, 0x91, 0x6c, 0xb3, 0x43, 0x05, 0x26, 0xcc, 0x4a, 0x91, 0x0e, 0x13, 0xba, 0x94, - 0xc2, 0x9d, 0x6c, 0x0a, 0xde, 0xd9, 0x15, 0x2b, 0x0e, 0x9e, 0x1b, 0x4b, 0x11, 0x14, 0x38, 0x86, - 0x52, 0xfd, 0x6d, 0x05, 0x86, 0xe9, 0x30, 0x4f, 0x80, 0x39, 0x7e, 0x67, 0x94, 0x39, 0xbe, 0x3b, - 0xef, 0x9e, 0xca, 0xe0, 0x89, 0x7f, 0x5d, 0x00, 0x96, 0xc1, 0x58, 0xd8, 0x17, 0x49, 0x96, 0x43, - 0x4a, 0x86, 0xcd, 0xd3, 0x15, 0x61, 0x78, 0x14, 0x53, 0x4a, 0x4b, 0xc6, 0x47, 0x6f, 0x8b, 0xd8, - 0x16, 0x45, 0x84, 0x8b, 0x14, 0xfb, 0xa2, 0xbb, 0x30, 0xee, 0x6e, 0xd9, 0xb6, 0x57, 0x0e, 0x3f, - 0x9e, 0xdc, 0x0f, 0x10, 0xcc, 0x59, 0xdc, 0x1f, 0x0a, 0x7f, 0xca, 0xad, 0xc9, 0xb8, 0x71, 0x94, - 0x14, 0x9a, 0x03, 0xd8, 0x30, 0x6d, 0xfd, 0x36, 0x37, 0x6d, 0xe2, 0xce, 0xc1, 0xcc, 0x78, 0x62, - 0x21, 0x28, 0xc5, 0x52, 0x8d, 0x9e, 0xac, 0xb8, 0xbe, 0xae, 0xf0, 0x99, 0x3e, 0xc4, 0x37, 0x7a, - 0x82, 0x86, 0x70, 0x6f, 0x85, 0x41, 0x87, 0xdd, 0x11, 0xc4, 0x8a, 0x05, 0x1f, 0x1f, 0xbf, 0x39, - 0x60, 0x01, 0xa5, 0xdb, 0x83, 0x5f, 0x7c, 0xfa, 0xc3, 0x07, 0x07, 0xf9, 0xba, 0xa2, 0xfe, 0xb2, - 0x18, 0x66, 0x90, 0x04, 0xbb, 0x05, 0xe3, 0xec, 0x66, 0x11, 0xcb, 0xbe, 0xfd, 0x8e, 0x2e, 0xbf, - 0x11, 0xb9, 0x69, 0x68, 0x96, 0x1b, 0x29, 0xc6, 0x51, 0x02, 0xe8, 0x51, 0x18, 0xf7, 0x47, 0xc7, - 0xad, 0x63, 0x0b, 0xa1, 0x87, 0x66, 0x55, 0x06, 0xe0, 0x68, 0x3d, 0xf5, 0x33, 0x05, 0xb8, 0xc4, - 0xfb, 0xce, 0x34, 0x2f, 0x65, 0xd2, 0x22, 0x56, 0x9d, 0x58, 0xfa, 0x2e, 0x93, 0xb3, 0xeb, 0x76, - 0x03, 0xbd, 0x0a, 0x83, 0x77, 0x08, 0xa9, 0x07, 0x4f, 0x18, 0xcf, 0xe5, 0xcf, 0x21, 0x9e, 0x41, - 0xe2, 0x39, 0x86, 0x9e, 0xbf, 0x06, 0xf3, 0xff, 0xb1, 0x20, 0x49, 0x89, 0xb7, 0x1c, 0x7b, 0x23, - 0x10, 0x07, 0x8f, 0x9e, 0x78, 0x95, 0xa1, 0xe7, 0xc4, 0xf9, 0xff, 0x58, 0x90, 0x54, 0xab, 0x70, - 0x5f, 0x17, 0x4d, 0x0f, 0x23, 0xf6, 0x1f, 0x84, 0x91, 0x8f, 0xfe, 0x30, 0x18, 0xff, 0x44, 0x81, - 0xfb, 0x25, 0x94, 0x8b, 0x3b, 0xf4, 0x26, 0x52, 0xd2, 0x5a, 0x9a, 0x4e, 0xef, 0xfa, 0x2c, 0x46, - 0xdd, 0xa1, 0xb2, 0xf6, 0x7e, 0x42, 0x81, 0x21, 0x6e, 0x91, 0xe7, 0xb3, 0xdf, 0x97, 0x7a, 0x9c, - 0xf2, 0xcc, 0x2e, 0xf9, 0xd9, 0xcc, 0xfc, 0xb1, 0xf1, 0xdf, 0x2e, 0xf6, 0xe9, 0xab, 0xff, 0x6a, - 0x00, 0xbe, 0xb5, 0x7b, 0x44, 0xe8, 0xeb, 0x8a, 0x9c, 0x6d, 0x9c, 0xeb, 0xc8, 0x9b, 0xc7, 0xdb, - 0xf9, 0x40, 0x1b, 0x24, 0x14, 0x0c, 0xcf, 0x25, 0x12, 0x92, 0x1f, 0x91, 0xa2, 0x29, 0x1c, 0x18, - 0xfa, 0xc7, 0x0a, 0x8c, 0xd1, 0x63, 0x29, 0x60, 0x2e, 0x7c, 0x99, 0x5a, 0xc7, 0x3c, 0xd2, 0x55, - 0x89, 0x64, 0x2c, 0xae, 0x95, 0x0c, 0xc2, 0x91, 0xbe, 0xa1, 0xf5, 0xe8, 0xf3, 0x1f, 0xbf, 0x22, - 0x5e, 0x4e, 0x93, 0x4b, 0x0e, 0x93, 0xee, 0x7f, 0xd6, 0x84, 0x89, 0xe8, 0xcc, 0x1f, 0xa7, 0x9a, - 0x6c, 0xf6, 0x29, 0x98, 0x4a, 0x8c, 0xfe, 0x50, 0x0a, 0x99, 0x2f, 0x0f, 0x42, 0x51, 0x9a, 0xea, - 0xb4, 0xf0, 0x34, 0xe8, 0x73, 0x0a, 0x8c, 0x6a, 0x96, 0x25, 0x0c, 0x9b, 0xfc, 0xfd, 0x5b, 0xef, - 0x71, 0x55, 0xd3, 0x48, 0xcd, 0xcd, 0x87, 0x64, 0x62, 0x96, 0x3b, 0x12, 0x04, 0xcb, 0xbd, 0xe9, - 0x60, 0x9d, 0x5b, 0x38, 0x31, 0xeb, 0x5c, 0xf4, 0x41, 0xff, 0x20, 0xee, 0xcb, 0x1f, 0xa8, 0xe2, - 0x80, 0xb9, 0x61, 0xe7, 0x7a, 0x86, 0x56, 0xf2, 0x07, 0x15, 0x76, 0xc8, 0x86, 0x51, 0x84, 0xc4, - 0x99, 0x94, 0xcb, 0xc8, 0xf2, 0xc0, 0x10, 0x45, 0xc1, 0xd9, 0x1d, 0x16, 0xe1, 0x28, 0x79, 0xf4, - 0x63, 0x0a, 0x8c, 0xdd, 0xb5, 0x2d, 0xcd, 0x14, 0x76, 0xd6, 0xe2, 0x99, 0xe6, 0xfd, 0xc7, 0x33, - 0x2f, 0x01, 0x19, 0x6e, 0x5b, 0x29, 0x97, 0xe0, 0x48, 0x37, 0x58, 0xc6, 0x7c, 0xe6, 0x5b, 0x3d, - 0x18, 0xca, 0xd3, 0xb2, 0x5f, 0xf5, 0xec, 0x93, 0x30, 0x19, 0xdf, 0x83, 0x87, 0xfa, 0x9e, 0xd6, - 0xe0, 0xc1, 0x43, 0xf4, 0xb7, 0xcb, 0xa3, 0x4f, 0xfd, 0x8d, 0xfe, 0xc8, 0x51, 0x9a, 0xb9, 0x3d, - 0xba, 0xd0, 0x95, 0xbf, 0x16, 0xfb, 0x98, 0x39, 0x8b, 0x36, 0x8e, 0x6b, 0xc3, 0x1e, 0xed, 0x17, - 0xdd, 0x77, 0x72, 0x5f, 0xf4, 0xeb, 0xed, 0x93, 0xea, 0x79, 0x67, 0x2e, 0x70, 0x5d, 0x80, 0x58, - 0xb0, 0x30, 0x59, 0x13, 0x0b, 0xd4, 0x69, 0xb8, 0x86, 0x1f, 0x6e, 0x5a, 0x12, 0xe9, 0x9e, 0xe5, - 0xc5, 0xd8, 0x87, 0xab, 0xcb, 0x91, 0xc3, 0x62, 0xcd, 0x6e, 0xd9, 0xa6, 0xdd, 0xd8, 0x9d, 0xbf, - 0xa3, 0x39, 0x04, 0xdb, 0x6d, 0x4f, 0x60, 0xeb, 0x56, 0x40, 0xfc, 0xc8, 0x00, 0x5c, 0x91, 0xd0, - 0xa5, 0x46, 0xe5, 0x3c, 0x8c, 0xaf, 0xc2, 0x6b, 0xf1, 0xe0, 0x99, 0x85, 0xfc, 0x46, 0xa6, 0x07, - 0xf5, 0x2b, 0x5f, 0x30, 0x4d, 0xb4, 0xa7, 0x00, 0x34, 0xb5, 0x1d, 0x11, 0xc9, 0x4d, 0x9c, 0x16, - 0xf5, 0x63, 0xe9, 0xe0, 0x4a, 0x40, 0x86, 0x77, 0xef, 0x79, 0x5f, 0xcf, 0x19, 0x02, 0x8e, 0x48, - 0x02, 0x94, 0x46, 0xd4, 0x73, 0xb4, 0xd0, 0xd9, 0x26, 0x9c, 0x8a, 0xf5, 0xfc, 0x58, 0xdf, 0x19, - 0xbf, 0x3c, 0xe4, 0x5f, 0x90, 0x45, 0x88, 0xb9, 0x5f, 0x52, 0xe0, 0x3c, 0xc9, 0x12, 0x38, 0xc5, - 0x6d, 0xf9, 0xf9, 0xe3, 0x12, 0x68, 0x45, 0xc2, 0xad, 0x2c, 0x30, 0xce, 0xee, 0x19, 0xda, 0x05, - 0x70, 0x83, 0x6f, 0xba, 0x97, 0xb8, 0x15, 0xa9, 0x4c, 0x82, 0xeb, 0x61, 0xc2, 0xdf, 0x58, 0x22, - 0x86, 0x7e, 0x4a, 0x81, 0x69, 0x33, 0x85, 0xc3, 0x09, 0x8e, 0x59, 0x3b, 0x86, 0xb3, 0x85, 0x5b, - 0xa8, 0xa4, 0x41, 0x70, 0x6a, 0x57, 0xd0, 0xcf, 0x64, 0x86, 0x18, 0xe6, 0x92, 0xc9, 0xda, 0x71, - 0x7c, 0x83, 0x39, 0xa2, 0x0d, 0x7f, 0x46, 0x01, 0x54, 0x4f, 0x5c, 0xbe, 0x85, 0x71, 0xe2, 0x33, - 0x47, 0xae, 0x62, 0xe0, 0x26, 0x46, 0xc9, 0x72, 0x9c, 0xd2, 0x09, 0xb6, 0xce, 0x5e, 0x0a, 0xcf, - 0x17, 0x2e, 0xc0, 0xbd, 0xae, 0x73, 0xda, 0x71, 0xc2, 0xd7, 0x39, 0x0d, 0x82, 0x53, 0xbb, 0xa2, - 0xfe, 0xd6, 0x08, 0xd7, 0x05, 0x33, 0x1b, 0x90, 0x97, 0x60, 0x70, 0x43, 0xd3, 0x6f, 0x8b, 0xf7, - 0xd7, 0x9c, 0x56, 0x49, 0x0b, 0x0c, 0x03, 0xd7, 0xc2, 0xf0, 0xff, 0xb1, 0xc0, 0x8a, 0x5e, 0x80, - 0xbe, 0xba, 0xe5, 0x7b, 0xd8, 0xbf, 0xa7, 0x87, 0xf7, 0x85, 0x30, 0x4e, 0x49, 0x79, 0xb5, 0x86, - 0x29, 0x52, 0x64, 0xc1, 0xb0, 0x25, 0x54, 0xa7, 0x42, 0xbb, 0xf5, 0x74, 0x5e, 0x02, 0x81, 0x0a, - 0x36, 0x50, 0xfc, 0xfa, 0x25, 0x38, 0xa0, 0x41, 0xe9, 0xc5, 0xde, 0x37, 0x73, 0xd3, 0x0b, 0x5e, - 0x84, 0xe4, 0x00, 0xae, 0xf1, 0x57, 0x21, 0x02, 0x83, 0x9e, 0x66, 0x58, 0x9e, 0xef, 0x2d, 0xff, - 0x44, 0x5e, 0x6a, 0x6b, 0x14, 0x4b, 0xa8, 0x21, 0x65, 0x3f, 0x5d, 0x2c, 0x90, 0xa3, 0x0d, 0x18, - 0xe4, 0x1e, 0xf3, 0xe2, 0x13, 0xca, 0xfd, 0xbe, 0xc6, 0x9d, 0xf0, 0xf9, 0x36, 0xe0, 0xff, 0x63, - 0x81, 0x19, 0xbd, 0x0c, 0xc3, 0xae, 0x6f, 0x8e, 0x36, 0xdc, 0xdb, 0xd4, 0x05, 0xb6, 0x68, 0xc2, - 0xff, 0x57, 0x18, 0xa1, 0x05, 0xf8, 0xd1, 0x06, 0x0c, 0x19, 0xe2, 0x4a, 0x35, 0x92, 0x7f, 0xdb, - 0xf9, 0xd7, 0x25, 0x76, 0xdb, 0xf0, 0x6f, 0x4a, 0x3e, 0xe2, 0x2c, 0x9b, 0x13, 0xf8, 0x06, 0xda, - 0x9c, 0xa0, 0x57, 0x00, 0x88, 0xaf, 0x42, 0x77, 0x67, 0x46, 0xf3, 0x6f, 0x19, 0x49, 0x11, 0xef, - 0x0b, 0x44, 0x41, 0x91, 0x8b, 0x25, 0x22, 0xe8, 0x55, 0x59, 0x67, 0x37, 0xd6, 0x5b, 0x10, 0x90, - 0x64, 0x58, 0x9b, 0xf0, 0xe9, 0x2e, 0x50, 0x17, 0x49, 0xaa, 0x34, 0xf5, 0xcb, 0x00, 0x10, 0x3e, - 0xeb, 0xa2, 0x4d, 0x18, 0xf6, 0xa9, 0xf4, 0x12, 0x86, 0xe7, 0xba, 0x00, 0xf3, 0xed, 0xe5, 0xff, - 0xc2, 0x01, 0x6e, 0x54, 0x4a, 0x0b, 0x07, 0x15, 0x66, 0xc8, 0xed, 0x2e, 0x14, 0xd4, 0x2b, 0x00, - 0x7a, 0x18, 0xe4, 0xb5, 0x2f, 0xff, 0x5a, 0x05, 0x01, 0x60, 0xc3, 0xb5, 0x92, 0x62, 0xc4, 0x4a, - 0x44, 0x32, 0xac, 0xd2, 0xfb, 0x73, 0x59, 0xa5, 0x3f, 0x01, 0xa7, 0x84, 0x15, 0x60, 0x85, 0xbd, - 0x32, 0x7a, 0xbb, 0xc2, 0x71, 0x93, 0xd9, 0x87, 0x96, 0xa2, 0x20, 0x1c, 0xaf, 0x8b, 0xfe, 0xa5, - 0x02, 0xc3, 0xba, 0x10, 0xd0, 0x04, 0x6f, 0x5b, 0xee, 0xed, 0x51, 0x7f, 0xce, 0x97, 0xf7, 0xb8, - 0x20, 0xff, 0xac, 0xcf, 0x55, 0xfd, 0xe2, 0x23, 0x12, 0xe3, 0x83, 0x5e, 0xa3, 0xdf, 0x53, 0x60, - 0x54, 0x33, 0x4d, 0x5b, 0xd7, 0x3c, 0x16, 0x1e, 0x90, 0x7b, 0x94, 0xde, 0xea, 0x71, 0x14, 0xf3, - 0x21, 0x46, 0x3e, 0x90, 0xf7, 0x05, 0x9a, 0x80, 0x10, 0x72, 0x44, 0x63, 0x91, 0xbb, 0x8f, 0xfe, - 0x7f, 0x05, 0xee, 0xe7, 0x6e, 0xbc, 0x25, 0x2a, 0x73, 0x6d, 0x1a, 0xba, 0xe6, 0x11, 0x1e, 0xb3, - 0xd4, 0x77, 0x21, 0xe3, 0x36, 0xf4, 0xc3, 0x87, 0xb6, 0xa1, 0xbf, 0xba, 0xbf, 0x57, 0xbc, 0xbf, - 0xd4, 0x05, 0x6e, 0xdc, 0x55, 0x0f, 0xd0, 0x5d, 0x18, 0x37, 0xe5, 0xf8, 0xe3, 0x82, 0xc9, 0xe7, - 0x7a, 0x7e, 0x8d, 0x04, 0x32, 0xe7, 0x0a, 0x86, 0x48, 0x11, 0x8e, 0x92, 0x9a, 0xbd, 0x0d, 0xe3, - 0x91, 0x8d, 0x76, 0xac, 0x8a, 0x6b, 0x0b, 0x26, 0xe3, 0xfb, 0xe1, 0x58, 0xef, 0x79, 0x37, 0x61, - 0x24, 0x10, 0x16, 0xd0, 0x25, 0x89, 0x50, 0x28, 0x7a, 0xdd, 0x24, 0xbb, 0x9c, 0x6a, 0x31, 0xa2, - 0x43, 0xe1, 0x4a, 0x42, 0x16, 0xbd, 0x51, 0x20, 0x54, 0xff, 0x40, 0xbc, 0xaa, 0xae, 0x91, 0x66, - 0xcb, 0xd4, 0x3c, 0xf2, 0xc6, 0xb7, 0x43, 0x52, 0xff, 0x83, 0xc2, 0xcf, 0x1b, 0x2e, 0xda, 0x20, - 0x0d, 0x46, 0x9b, 0x3c, 0xf9, 0x1d, 0x8b, 0xf5, 0xa8, 0xe4, 0x8f, 0x32, 0xb9, 0x12, 0xa2, 0xc1, - 0x32, 0x4e, 0x74, 0x07, 0x46, 0x7c, 0x61, 0xd0, 0xd7, 0xd4, 0x2c, 0xf5, 0x26, 0x9c, 0x05, 0x72, - 0x67, 0x70, 0xb4, 0xfa, 0x25, 0x2e, 0x0e, 0x69, 0xa9, 0x1a, 0xa0, 0x64, 0x1b, 0xf4, 0x40, 0xe8, - 0xcf, 0xa6, 0x44, 0xd3, 0xd5, 0x24, 0x7c, 0xda, 0x7c, 0x25, 0x6b, 0x21, 0x4b, 0xc9, 0xaa, 0xfe, - 0x66, 0x01, 0xa6, 0xc5, 0xd5, 0x73, 0x5e, 0xd7, 0xed, 0xb6, 0xe5, 0x85, 0x6e, 0xc6, 0xdc, 0x77, - 0x5f, 0x10, 0x61, 0xe2, 0x24, 0x77, 0xec, 0xc7, 0x02, 0x82, 0x6e, 0x71, 0xed, 0xa7, 0x55, 0x5f, - 0xb3, 0x6f, 0x13, 0x2b, 0xe4, 0x12, 0x72, 0x90, 0x8c, 0xc5, 0xb4, 0x0a, 0x38, 0xbd, 0x1d, 0xda, - 0x06, 0xd4, 0xd4, 0x76, 0xe2, 0xd8, 0xf2, 0x25, 0x41, 0x63, 0xf7, 0xc5, 0x95, 0x04, 0x36, 0x9c, - 0x42, 0x81, 0x1e, 0xa4, 0x54, 0x92, 0x6b, 0x79, 0xa4, 0xce, 0x87, 0xe8, 0x1b, 0x75, 0xb0, 0x83, - 0x74, 0x3e, 0x0a, 0xc2, 0xf1, 0xba, 0xea, 0xc7, 0x06, 0xe1, 0x7c, 0x74, 0x12, 0xe9, 0x17, 0xea, - 0xbb, 0xd7, 0x3f, 0xe5, 0xfb, 0x8e, 0xf1, 0x89, 0x7c, 0x20, 0xee, 0x3b, 0x36, 0x23, 0x1b, 0x0d, - 0x89, 0x46, 0x11, 0x3f, 0xb2, 0x6f, 0x80, 0xaf, 0x7c, 0x86, 0x73, 0x74, 0xdf, 0xb1, 0xc6, 0x04, - 0xf8, 0xa4, 0x02, 0xb3, 0xd1, 0xe2, 0x25, 0xc3, 0x32, 0xdc, 0x2d, 0x91, 0xf7, 0xe4, 0xf0, 0xae, - 0x6b, 0x2c, 0x73, 0xf2, 0x72, 0x26, 0x46, 0xdc, 0x81, 0x1a, 0xfa, 0x01, 0x05, 0x2e, 0xc4, 0xe6, - 0x25, 0x92, 0x85, 0xe5, 0xf0, 0x5e, 0x6c, 0x2c, 0xb8, 0xcb, 0x72, 0x36, 0x4a, 0xdc, 0x89, 0x5e, - 0xa7, 0x88, 0x9f, 0x83, 0xaf, 0x97, 0x88, 0x9f, 0xbf, 0x58, 0x80, 0x01, 0x66, 0x37, 0xf5, 0xc6, - 0x70, 0x38, 0x62, 0x5d, 0xcd, 0xb4, 0x77, 0x6d, 0xc4, 0xec, 0x5d, 0x9f, 0xca, 0x4f, 0xa2, 0xb3, - 0xc1, 0xeb, 0xfb, 0xe0, 0x2c, 0xab, 0x36, 0x5f, 0x67, 0x8a, 0x36, 0x97, 0xd4, 0xe7, 0xeb, 0x75, - 0x76, 0xbd, 0x3d, 0xf8, 0x8d, 0xec, 0x12, 0xf4, 0xb5, 0x1d, 0x33, 0x1e, 0x42, 0x76, 0x1d, 0x2f, - 0x63, 0x5a, 0xae, 0x7e, 0x52, 0x81, 0x49, 0x86, 0x5b, 0x62, 0x31, 0x68, 0x1b, 0x86, 0x1d, 0xc1, - 0x66, 0xc4, 0xda, 0x2c, 0xe7, 0x1e, 0x5a, 0x0a, 0xeb, 0xe2, 0x37, 0x36, 0xff, 0x17, 0x0e, 0x68, - 0xa9, 0x7f, 0x3a, 0x00, 0x33, 0x59, 0x8d, 0xd0, 0x8f, 0x28, 0x70, 0x56, 0x0f, 0x25, 0xce, 0xf9, - 0xb6, 0xb7, 0x65, 0x3b, 0x7e, 0x46, 0x85, 0xdc, 0xea, 0x90, 0xd2, 0x7c, 0xd0, 0x2b, 0x96, 0x46, - 0xa4, 0x94, 0x4a, 0x01, 0x67, 0x50, 0x46, 0xaf, 0x02, 0xb8, 0xee, 0xd6, 0x4d, 0xb2, 0xdb, 0xd2, - 0x0c, 0xdf, 0x72, 0xeb, 0x66, 0xfe, 0x6d, 0x50, 0xbb, 0x21, 0x50, 0x05, 0x9d, 0xe2, 0xba, 0xea, - 0xb0, 0x5c, 0x22, 0x87, 0x3e, 0xaa, 0xc0, 0xb8, 0x2d, 0xc7, 0x6b, 0xe9, 0xc5, 0x98, 0x3f, 0x35, - 0xf0, 0x0b, 0x97, 0xb4, 0xa3, 0xa0, 0x28, 0x49, 0xba, 0x2c, 0x53, 0x6e, 0xfc, 0x64, 0x13, 0xbc, - 0x6f, 0x25, 0x9f, 0x0c, 0x94, 0x71, 0x4c, 0xf2, 0x5b, 0x7b, 0x12, 0x9c, 0x24, 0xcf, 0x3a, 0x45, - 0x3c, 0xbd, 0xbe, 0x68, 0xe9, 0xce, 0x2e, 0x8b, 0x10, 0x40, 0x3b, 0x35, 0x98, 0xbf, 0x53, 0x8b, - 0x6b, 0xa5, 0x72, 0x04, 0x59, 0xb4, 0x53, 0x49, 0x70, 0x92, 0xbc, 0xfa, 0x91, 0x02, 0x9c, 0x63, - 0xcb, 0x7c, 0xb3, 0xbd, 0x41, 0xb8, 0xf1, 0xd5, 0x3f, 0xb8, 0x00, 0x3b, 0xbf, 0xa3, 0xc0, 0x08, - 0x9b, 0x83, 0x37, 0x88, 0x8f, 0x26, 0xeb, 0x6b, 0x86, 0x81, 0xf3, 0x6f, 0x2b, 0x30, 0x95, 0xc8, - 0x21, 0xd5, 0x95, 0x87, 0xdf, 0x89, 0xd9, 0xde, 0xbe, 0x25, 0xcc, 0x5b, 0xd9, 0x17, 0x06, 0xb6, - 0x88, 0xe7, 0xac, 0x54, 0x9f, 0x83, 0xf1, 0x88, 0x7d, 0xb3, 0x14, 0x02, 0x32, 0x2d, 0x78, 0xa5, - 0x1c, 0xe1, 0xb1, 0xd0, 0x29, 0x36, 0x65, 0xb8, 0xe5, 0x93, 0x9c, 0xed, 0x1f, 0xce, 0x96, 0x47, - 0x62, 0xcb, 0xfb, 0xcf, 0x38, 0x2c, 0xb0, 0xa4, 0xdb, 0xcb, 0x33, 0x0e, 0x8b, 0x50, 0x29, 0x02, - 0xc1, 0xf2, 0xff, 0xb1, 0xc0, 0x8a, 0x9e, 0x8e, 0x86, 0x79, 0x5d, 0x0d, 0xef, 0x76, 0xd3, 0xf1, - 0xe0, 0xac, 0x6c, 0x4b, 0x26, 0x6a, 0x23, 0xcc, 0x1f, 0x82, 0xf8, 0x59, 0x96, 0x2b, 0xbb, 0x4d, - 0x79, 0xb5, 0xc6, 0xe3, 0xff, 0x05, 0x0f, 0x40, 0x51, 0x8d, 0x77, 0xff, 0x49, 0x68, 0xbc, 0x1d, - 0x18, 0xdd, 0x32, 0x36, 0x88, 0x63, 0x71, 0x31, 0x66, 0x20, 0xbf, 0x84, 0x76, 0x23, 0x44, 0xc3, - 0xd5, 0x00, 0x52, 0x01, 0x96, 0x89, 0x20, 0x27, 0x12, 0xac, 0x7a, 0x30, 0xbf, 0x54, 0x12, 0xaa, - 0xa6, 0xc3, 0x71, 0x66, 0x04, 0xaa, 0xb6, 0x00, 0xac, 0x20, 0x82, 0x6b, 0x2f, 0x0f, 0x43, 0x61, - 0x1c, 0x58, 0x2e, 0x74, 0x84, 0xbf, 0xb1, 0x44, 0x81, 0xce, 0x6b, 0x33, 0x4c, 0x68, 0x20, 0xd4, - 0x8c, 0x4f, 0xf5, 0x98, 0x54, 0x42, 0xa8, 0x57, 0xc2, 0x02, 0x2c, 0x13, 0xa1, 0x63, 0x6c, 0x06, - 0x61, 0xfc, 0x85, 0x1a, 0x31, 0xd7, 0x18, 0xc3, 0x64, 0x00, 0x7c, 0x8c, 0xe1, 0x6f, 0x2c, 0x51, - 0x40, 0x2f, 0x4b, 0xef, 0x87, 0x90, 0x5f, 0x49, 0xd5, 0xd5, 0xdb, 0xe1, 0x3b, 0x43, 0x5d, 0xcd, - 0x28, 0xfb, 0x4e, 0x2f, 0x48, 0x7a, 0x1a, 0x96, 0xde, 0x80, 0xf2, 0x8e, 0x84, 0xde, 0x26, 0xf4, - 0xaa, 0x18, 0xeb, 0xe8, 0x55, 0x51, 0xa2, 0xd2, 0x99, 0xe4, 0x99, 0xc8, 0x18, 0xc2, 0x78, 0xf8, - 0x08, 0x52, 0x8b, 0x03, 0x71, 0xb2, 0x3e, 0x67, 0xf8, 0xa4, 0xce, 0xda, 0x4e, 0xc8, 0x0c, 0x9f, - 0x97, 0xe1, 0x00, 0x8a, 0xb6, 0x61, 0xcc, 0x95, 0x5c, 0x34, 0x66, 0x4e, 0xf5, 0xfa, 0x84, 0x28, - 0xdc, 0x33, 0x98, 0x2d, 0xa4, 0x5c, 0x82, 0x23, 0x74, 0xa2, 0xef, 0x5b, 0x93, 0x27, 0xfb, 0xbe, - 0x85, 0xda, 0x51, 0xeb, 0xeb, 0xa9, 0x23, 0x89, 0xe3, 0x72, 0xa0, 0x75, 0x36, 0x5d, 0x5a, 0xb2, - 0xd3, 0xb2, 0xdd, 0xb6, 0x43, 0x98, 0xd9, 0x27, 0x5b, 0x1e, 0x14, 0x2e, 0xed, 0x62, 0x1c, 0x88, - 0x93, 0xf5, 0xd1, 0xf7, 0x29, 0x30, 0xe9, 0xee, 0xba, 0x1e, 0x69, 0xd2, 0x63, 0xcb, 0xb6, 0x88, - 0xe5, 0xb9, 0x33, 0xa7, 0xf3, 0xc7, 0x05, 0xaf, 0xc5, 0x70, 0xf1, 0x63, 0x27, 0x5e, 0x8a, 0x13, - 0x34, 0xe9, 0xce, 0x91, 0x23, 0xc1, 0xcc, 0x4c, 0xe7, 0xdf, 0x39, 0x72, 0x94, 0x19, 0xbe, 0x73, - 0xe4, 0x12, 0x1c, 0xa1, 0x83, 0x1e, 0x85, 0x71, 0xd7, 0xcf, 0x61, 0xce, 0x66, 0xf0, 0x4c, 0x18, - 0xac, 0xb3, 0x26, 0x03, 0x70, 0xb4, 0x1e, 0xfa, 0x10, 0x8c, 0xc9, 0x67, 0xe7, 0xcc, 0xd9, 0xa3, - 0x0e, 0x5b, 0xcf, 0x7b, 0x2e, 0x83, 0x22, 0x04, 0x11, 0x86, 0xb3, 0x92, 0xf3, 0xa0, 0xfc, 0x7d, - 0x9f, 0x63, 0x43, 0xe0, 0xf7, 0xd9, 0xd4, 0x1a, 0x38, 0xa3, 0x25, 0xfa, 0x89, 0xf4, 0xe7, 0xf2, - 0x19, 0xb6, 0xa5, 0xab, 0x47, 0xf2, 0x5c, 0xfe, 0x9c, 0xe1, 0x6d, 0xdd, 0x6a, 0xf1, 0xc8, 0x62, - 0x87, 0x7c, 0x39, 0x57, 0xff, 0x48, 0x01, 0x08, 0x14, 0x26, 0x27, 0xf1, 0x54, 0x51, 0x8f, 0xe8, - 0x90, 0x16, 0x7a, 0x52, 0xf0, 0x64, 0x66, 0x25, 0x51, 0xbf, 0xa2, 0xc0, 0x44, 0x58, 0xed, 0x04, - 0xae, 0x46, 0x7a, 0xf4, 0x6a, 0xf4, 0x64, 0x6f, 0xe3, 0xca, 0xb8, 0x1f, 0xfd, 0xcf, 0x82, 0x3c, - 0x2a, 0x26, 0xfd, 0x6e, 0x47, 0x9e, 0xfe, 0x29, 0xe9, 0x1b, 0xbd, 0x3c, 0xfd, 0xcb, 0xd1, 0x37, - 0xc2, 0xf1, 0xa6, 0x98, 0x02, 0x7c, 0x77, 0x44, 0xfe, 0xec, 0x21, 0xee, 0x4d, 0x20, 0x6c, 0xfa, - 0xa4, 0xf9, 0x04, 0x1c, 0x24, 0x8c, 0xbe, 0x22, 0x1f, 0x4f, 0x3d, 0x64, 0x12, 0x89, 0x0c, 0xb8, - 0xb3, 0xd1, 0xc5, 0x2f, 0x4e, 0xc1, 0xa8, 0xa4, 0x5b, 0x8c, 0x19, 0x32, 0x28, 0x27, 0x61, 0xc8, - 0xe0, 0xc1, 0xa8, 0x1e, 0xa4, 0x92, 0xf4, 0xa7, 0xbd, 0x47, 0x9a, 0xc1, 0xb1, 0x18, 0x26, 0xa9, - 0x74, 0xb1, 0x4c, 0x86, 0x0a, 0x6f, 0xc1, 0x1e, 0xeb, 0x3b, 0x02, 0xf3, 0x92, 0x4e, 0xfb, 0xea, - 0x11, 0x00, 0x5f, 0xfe, 0x27, 0x75, 0x11, 0xa1, 0x3d, 0x30, 0x9d, 0xae, 0xb8, 0x37, 0x02, 0x18, - 0x96, 0xea, 0x25, 0x1f, 0xc6, 0x07, 0x4e, 0xec, 0x61, 0x9c, 0x6e, 0x03, 0xd3, 0x4f, 0xba, 0xdf, - 0x93, 0xb9, 0x5a, 0x90, 0xba, 0x3f, 0xdc, 0x06, 0x41, 0x91, 0x8b, 0x25, 0x22, 0x19, 0xf6, 0x2c, - 0x43, 0xb9, 0xec, 0x59, 0xda, 0x70, 0xda, 0x21, 0x9e, 0xb3, 0x5b, 0xda, 0xd5, 0x59, 0x6a, 0x13, - 0xc7, 0x63, 0x37, 0xf8, 0xe1, 0x7c, 0x01, 0x13, 0x71, 0x12, 0x15, 0x4e, 0xc3, 0x1f, 0x11, 0x80, - 0x47, 0x3a, 0x0a, 0xc0, 0xef, 0x84, 0x51, 0x8f, 0xe8, 0x5b, 0x96, 0xa1, 0x6b, 0x66, 0xa5, 0x2c, - 0xe2, 0x77, 0x87, 0xb2, 0x5c, 0x08, 0xc2, 0x72, 0x3d, 0xb4, 0x00, 0x7d, 0x6d, 0xa3, 0x2e, 0x6e, - 0x00, 0xdf, 0x16, 0x68, 0xe9, 0x2b, 0xe5, 0x7b, 0x7b, 0xc5, 0x37, 0x87, 0x06, 0x22, 0xc1, 0xa8, - 0xae, 0xb5, 0x6e, 0x37, 0xae, 0x79, 0xbb, 0x2d, 0xe2, 0xce, 0xad, 0x57, 0xca, 0x98, 0x36, 0x4e, - 0xb3, 0xf5, 0x19, 0x3b, 0x84, 0xad, 0xcf, 0x67, 0x14, 0x38, 0xad, 0xc5, 0x1f, 0x18, 0x88, 0x3b, - 0x33, 0x9e, 0x9f, 0x5b, 0xa6, 0x3f, 0x5a, 0x2c, 0x5c, 0x10, 0xe3, 0x3b, 0x3d, 0x9f, 0x24, 0x87, - 0xd3, 0xfa, 0x80, 0x1c, 0x40, 0x4d, 0xa3, 0x11, 0xe4, 0xbf, 0x17, 0xab, 0x3e, 0x91, 0x4f, 0x6f, - 0xb3, 0x92, 0xc0, 0x84, 0x53, 0xb0, 0xa3, 0x3b, 0x30, 0x2a, 0x09, 0x49, 0xe2, 0x26, 0x53, 0x3e, - 0x8a, 0x77, 0x10, 0x7e, 0xdb, 0x95, 0xdf, 0x38, 0x64, 0x4a, 0xc1, 0x23, 0xa7, 0xa4, 0x66, 0x10, - 0x0f, 0x7d, 0x6c, 0xd4, 0x93, 0xf9, 0x1f, 0x39, 0xd3, 0x31, 0xe2, 0x0e, 0xd4, 0x58, 0x98, 0x42, - 0x0a, 0x96, 0xee, 0xe6, 0x33, 0x53, 0xf9, 0xe3, 0xac, 0x2c, 0x47, 0x51, 0xf1, 0xad, 0x19, 0x2b, - 0xc4, 0x71, 0x82, 0x68, 0x09, 0x10, 0xe1, 0xaa, 0xf4, 0xf0, 0x72, 0xe6, 0xce, 0x20, 0xf6, 0xfe, - 0xce, 0x96, 0x74, 0x31, 0x01, 0xc5, 0x29, 0x2d, 0x90, 0x17, 0xd1, 0x95, 0xf4, 0x70, 0xcb, 0x89, - 0xe7, 0xcc, 0xe9, 0xa8, 0x31, 0xf9, 0x5e, 0x25, 0x91, 0x44, 0x99, 0x5f, 0x6e, 0x6e, 0xf4, 0x9e, - 0x44, 0x59, 0x90, 0xef, 0x26, 0x95, 0xf2, 0xe7, 0x15, 0x38, 0xd7, 0x4c, 0x4f, 0x16, 0xc9, 0xee, - 0x3c, 0x39, 0x1f, 0xae, 0x32, 0xf2, 0x4f, 0x32, 0x35, 0x46, 0x56, 0x72, 0x4a, 0x9c, 0xd5, 0x11, - 0xf5, 0x0f, 0x15, 0xa1, 0x8a, 0x3e, 0x41, 0x73, 0xa4, 0xe3, 0x7e, 0x27, 0x56, 0x9f, 0x83, 0x99, - 0x9a, 0x1f, 0x64, 0xb4, 0x1e, 0xcb, 0x25, 0xf0, 0x1e, 0x18, 0xe7, 0x4f, 0x41, 0x2b, 0x5a, 0x6b, - 0x35, 0x7c, 0x37, 0x08, 0xc2, 0x51, 0x94, 0x64, 0x20, 0x8e, 0xd6, 0x55, 0xbf, 0xa6, 0xc0, 0xb9, - 0x28, 0x66, 0xdb, 0x31, 0xee, 0xf6, 0x8e, 0x18, 0x7d, 0x5c, 0x81, 0xd1, 0xdb, 0xc1, 0x2b, 0x95, - 0x2f, 0xba, 0xe5, 0x72, 0xd9, 0xf0, 0x7b, 0x45, 0x1c, 0xe9, 0xd9, 0x2b, 0x99, 0xa0, 0x32, 0x04, - 0xba, 0x58, 0x26, 0xad, 0xfe, 0x17, 0x05, 0x12, 0xea, 0x03, 0xb4, 0x01, 0x43, 0x94, 0x48, 0x79, - 0xb5, 0x26, 0xf6, 0xc4, 0x7b, 0xf2, 0x49, 0x95, 0x0c, 0x05, 0x7f, 0x14, 0x11, 0x3f, 0xb0, 0x8f, - 0x18, 0x6d, 0x73, 0xff, 0x7f, 0x3f, 0xd1, 0x90, 0xd8, 0x1e, 0xb9, 0xc4, 0x76, 0x39, 0x61, 0x11, - 0xbf, 0xd6, 0xcb, 0x25, 0x38, 0x42, 0x47, 0x5d, 0x06, 0x08, 0x55, 0x3e, 0x3d, 0x9b, 0xf7, 0xfd, - 0xdb, 0xd3, 0x70, 0xa6, 0x67, 0x67, 0xc4, 0x8f, 0x2a, 0x70, 0x96, 0x6c, 0x1b, 0xba, 0x37, 0xbf, - 0xe9, 0x11, 0xe7, 0xd6, 0xad, 0x95, 0xb5, 0x2d, 0x87, 0xb8, 0x5b, 0xb6, 0x59, 0xef, 0xc6, 0x98, - 0x31, 0xc5, 0xf2, 0x8a, 0xa9, 0x26, 0x16, 0x53, 0x31, 0xe2, 0x0c, 0x4a, 0x4c, 0xdd, 0xb5, 0xcd, - 0x15, 0x01, 0x98, 0xde, 0xb9, 0xda, 0x8e, 0xeb, 0x89, 0xb8, 0x81, 0x5c, 0xdd, 0x15, 0x07, 0xe2, - 0x64, 0xfd, 0x38, 0x12, 0x96, 0x45, 0x8f, 0xc9, 0xed, 0x4a, 0x12, 0x09, 0x03, 0xe2, 0x64, 0x7d, - 0x19, 0x09, 0x5f, 0x29, 0x7a, 0x28, 0x0e, 0x24, 0x91, 0x04, 0x40, 0x9c, 0xac, 0x8f, 0xea, 0x70, - 0xd1, 0x21, 0xba, 0xdd, 0x6c, 0x12, 0xab, 0xce, 0x26, 0x65, 0x45, 0x73, 0x1a, 0x86, 0xb5, 0xe4, - 0x68, 0xac, 0x22, 0x7b, 0x3d, 0x50, 0x58, 0xc2, 0xdb, 0x8b, 0xb8, 0x43, 0x3d, 0xdc, 0x11, 0x0b, - 0x6a, 0xc2, 0xa9, 0x36, 0xe3, 0xff, 0x0e, 0x8b, 0x2b, 0xb6, 0xad, 0x99, 0xe2, 0x89, 0xe0, 0xb0, - 0x2b, 0xc6, 0x0e, 0xea, 0xf5, 0x28, 0x2a, 0x1c, 0xc7, 0x8d, 0x76, 0xa9, 0x78, 0x2e, 0xba, 0x23, - 0x91, 0x1c, 0xce, 0x45, 0x52, 0x88, 0xe8, 0x09, 0x74, 0x38, 0x8d, 0x06, 0xaa, 0xc0, 0x69, 0x4f, - 0x73, 0x1a, 0xc4, 0x2b, 0x55, 0xd7, 0xab, 0xc4, 0xd1, 0x29, 0x8f, 0x35, 0xb9, 0xb4, 0xae, 0x70, - 0x54, 0x6b, 0x49, 0x30, 0x4e, 0x6b, 0x83, 0x3e, 0x04, 0x6f, 0x89, 0x4e, 0xea, 0xb2, 0x7d, 0x87, - 0x38, 0x0b, 0x76, 0xdb, 0xaa, 0x47, 0x91, 0x03, 0x43, 0xfe, 0xc0, 0xfe, 0x5e, 0xf1, 0x2d, 0xb8, - 0x9b, 0x06, 0xb8, 0x3b, 0xbc, 0xc9, 0x0e, 0xac, 0xb7, 0x5a, 0xa9, 0x1d, 0x18, 0xcd, 0xea, 0x40, - 0x46, 0x03, 0xdc, 0x1d, 0x5e, 0x84, 0xe1, 0x2c, 0x9f, 0x18, 0x9e, 0x9e, 0x59, 0xa2, 0x38, 0xc6, - 0x28, 0xb2, 0xef, 0x77, 0x2d, 0xb5, 0x06, 0xce, 0x68, 0x49, 0xcf, 0x94, 0xab, 0x59, 0xc3, 0x4f, - 0x90, 0x19, 0x67, 0x64, 0xde, 0xb6, 0xbf, 0x57, 0xbc, 0x8a, 0xbb, 0x6c, 0x83, 0xbb, 0xc6, 0x9e, - 0xd2, 0x95, 0x70, 0x22, 0x12, 0x5d, 0x99, 0xc8, 0xea, 0x4a, 0x76, 0x1b, 0xdc, 0x35, 0x76, 0xf4, - 0xfd, 0x0a, 0x9c, 0xd7, 0x5b, 0xed, 0x1b, 0x86, 0xeb, 0xd9, 0x0d, 0x47, 0x6b, 0x96, 0x89, 0xae, - 0xed, 0xde, 0xd0, 0xcc, 0xcd, 0x65, 0x63, 0x93, 0x88, 0x4b, 0xc7, 0x61, 0x3f, 0x1c, 0xe6, 0x78, - 0x5b, 0xaa, 0xae, 0xa7, 0x23, 0xc5, 0xd9, 0xf4, 0xd0, 0x8f, 0x2a, 0x70, 0xb1, 0xc9, 0xba, 0x98, - 0xd1, 0xa1, 0xc9, 0x5c, 0x1d, 0x62, 0x5c, 0x6c, 0xa5, 0x03, 0x5e, 0xdc, 0x91, 0x2a, 0x9b, 0x24, - 0x5e, 0x61, 0xbe, 0xd1, 0x70, 0x48, 0x83, 0x61, 0x0d, 0xb8, 0xcb, 0x54, 0xfe, 0x49, 0x5a, 0xc9, - 0x42, 0x8a, 0xb3, 0xe9, 0xa1, 0x97, 0xe1, 0x72, 0x26, 0xb0, 0x64, 0xb7, 0x2d, 0x8f, 0x3d, 0xc2, - 0xf4, 0x2d, 0xa8, 0xfb, 0x7b, 0xc5, 0xcb, 0x2b, 0x1d, 0x6b, 0xe2, 0x03, 0x30, 0xb1, 0xd8, 0x23, - 0x91, 0x30, 0x00, 0xa7, 0x99, 0x24, 0xf6, 0x62, 0xae, 0x74, 0xbc, 0x47, 0xe8, 0xfb, 0xff, 0x95, - 0xa8, 0xef, 0xff, 0x34, 0xeb, 0xd5, 0xfb, 0x8e, 0xae, 0x57, 0xdf, 0x74, 0xf8, 0xef, 0xc6, 0x11, - 0xe4, 0x33, 0x0a, 0x08, 0xff, 0x4d, 0x74, 0x31, 0x62, 0x76, 0x34, 0x1c, 0x33, 0x39, 0xf2, 0xb3, - 0xe2, 0x16, 0x52, 0xb3, 0xe2, 0xbe, 0x55, 0x0a, 0xc2, 0x3c, 0x12, 0x5e, 0x96, 0x38, 0xe6, 0x30, - 0x0a, 0x33, 0x7a, 0x10, 0x46, 0x82, 0x4b, 0xb5, 0x50, 0x76, 0xb2, 0x8c, 0x34, 0xe1, 0xed, 0x3b, - 0x84, 0xab, 0xbf, 0xaf, 0x00, 0x84, 0xc9, 0x9e, 0xd1, 0x7d, 0x7e, 0xb0, 0x1a, 0xde, 0xc1, 0xe0, - 0xc5, 0x40, 0x0e, 0x58, 0x73, 0xb0, 0x33, 0x02, 0x52, 0x61, 0xb0, 0xcd, 0xd2, 0x5b, 0x0a, 0x07, - 0x02, 0x66, 0x02, 0xb3, 0xce, 0x4a, 0xb0, 0x80, 0xa0, 0x75, 0x18, 0x6a, 0x1a, 0x16, 0xf3, 0xf5, - 0xe8, 0xcf, 0xe5, 0xeb, 0xc1, 0xee, 0x03, 0x2b, 0x1c, 0x05, 0xf6, 0x71, 0xa9, 0xbf, 0xa4, 0xc0, - 0xa9, 0x68, 0x54, 0x6c, 0x16, 0xf2, 0x46, 0x04, 0x4b, 0x15, 0xc1, 0xf8, 0x59, 0x53, 0x11, 0x04, - 0x12, 0xfb, 0xb0, 0xe8, 0xeb, 0x74, 0x0f, 0xaf, 0x0f, 0xe9, 0xc1, 0xb9, 0x0f, 0x78, 0x08, 0xf8, - 0xf1, 0xd3, 0x30, 0xc8, 0x2f, 0xda, 0x54, 0x8e, 0x4f, 0x09, 0x7e, 0x75, 0x33, 0x7f, 0xbe, 0x89, - 0x3c, 0x11, 0x71, 0xe4, 0xac, 0x9b, 0x85, 0x8e, 0x59, 0x37, 0x31, 0xf4, 0xe9, 0x8e, 0xd1, 0x8b, - 0x25, 0x52, 0x09, 0x57, 0xb8, 0x25, 0x52, 0x09, 0x57, 0x30, 0x45, 0x86, 0xbc, 0x88, 0x89, 0x4e, - 0x7f, 0x7e, 0x15, 0x10, 0x9f, 0x00, 0xc9, 0x50, 0x67, 0xa2, 0xa3, 0x91, 0x8e, 0x1f, 0x69, 0x7f, - 0x20, 0xbf, 0x73, 0x90, 0x98, 0xf2, 0x6e, 0x22, 0xed, 0xfb, 0x1f, 0xd2, 0x60, 0xe6, 0x87, 0xb4, - 0x09, 0x43, 0xe2, 0x53, 0x10, 0x17, 0x82, 0xf7, 0xf4, 0x90, 0x5b, 0x5e, 0xca, 0x62, 0xc5, 0x0b, - 0xb0, 0x8f, 0x9c, 0xde, 0x32, 0x9b, 0xda, 0x8e, 0xd1, 0x6c, 0x37, 0xd9, 0x2d, 0x60, 0x40, 0xae, - 0xca, 0x8a, 0xb1, 0x0f, 0x67, 0x55, 0xb9, 0x4f, 0x15, 0x93, 0xda, 0xe5, 0xaa, 0xbc, 0x18, 0xfb, - 0x70, 0xf4, 0x02, 0x0c, 0x37, 0xb5, 0x9d, 0x5a, 0xdb, 0x69, 0x10, 0x61, 0xa0, 0x93, 0xad, 0x14, - 0x6a, 0x7b, 0x86, 0x39, 0x67, 0x58, 0x9e, 0xeb, 0x39, 0x73, 0x15, 0xcb, 0xbb, 0xe5, 0xd4, 0x3c, - 0x66, 0x00, 0xc4, 0x76, 0xdd, 0x8a, 0xc0, 0x82, 0x03, 0x7c, 0xc8, 0x84, 0x89, 0xa6, 0xb6, 0xb3, - 0x6e, 0x69, 0x3c, 0x89, 0x82, 0x90, 0xb2, 0xf3, 0x50, 0x60, 0x5a, 0xb9, 0x95, 0x08, 0x2e, 0x1c, - 0xc3, 0x9d, 0x62, 0x0c, 0x3a, 0x76, 0x5c, 0xc6, 0xa0, 0xf3, 0x41, 0x94, 0x02, 0xae, 0xd2, 0x3f, - 0x9f, 0x1a, 0x1f, 0xb0, 0x63, 0x04, 0x82, 0x97, 0x82, 0x08, 0x04, 0x13, 0xf9, 0xad, 0x17, 0x3b, - 0x44, 0x1f, 0x68, 0xc3, 0x68, 0x5d, 0xf3, 0x34, 0x5e, 0xea, 0xce, 0x9c, 0xca, 0xff, 0x3a, 0x5d, - 0x0e, 0xd0, 0x84, 0x2c, 0x29, 0x2c, 0x73, 0xb1, 0x4c, 0x07, 0xdd, 0x82, 0x33, 0xf4, 0x63, 0x35, - 0x89, 0x17, 0x56, 0x61, 0x3a, 0xb3, 0x49, 0xf6, 0xfd, 0x30, 0x2f, 0xb5, 0x9b, 0x69, 0x15, 0x70, - 0x7a, 0xbb, 0x30, 0x96, 0xed, 0x54, 0x7a, 0x2c, 0x5b, 0xf4, 0xa9, 0x34, 0xb3, 0x1b, 0xc4, 0xe6, - 0xf4, 0xbd, 0xf9, 0x79, 0x43, 0x6e, 0xe3, 0x9b, 0x7f, 0xae, 0xc0, 0x8c, 0xd8, 0x65, 0xc2, 0x54, - 0xc6, 0x24, 0xce, 0x8a, 0x66, 0x69, 0x0d, 0xe2, 0x08, 0x3d, 0xf9, 0x5a, 0x0f, 0xfc, 0x21, 0x81, - 0x33, 0x08, 0x0d, 0x71, 0xff, 0xfe, 0x5e, 0xf1, 0xca, 0x41, 0xb5, 0x70, 0x66, 0xdf, 0x90, 0x03, - 0x43, 0xee, 0xae, 0xab, 0x7b, 0xa6, 0x2b, 0x64, 0xd0, 0xeb, 0x3d, 0x70, 0xd6, 0x1a, 0xc7, 0xc4, - 0x59, 0x6b, 0x98, 0x3b, 0x91, 0x97, 0x62, 0x9f, 0x10, 0xfa, 0x61, 0x05, 0xa6, 0xc4, 0xe3, 0x99, - 0x14, 0x7a, 0xe7, 0x4c, 0x7e, 0x3f, 0x99, 0x52, 0x1c, 0x99, 0x6f, 0x1e, 0xc3, 0xb4, 0x49, 0x09, - 0x28, 0x4e, 0x52, 0xa7, 0x87, 0x6a, 0xcb, 0x31, 0x6c, 0xc7, 0xf0, 0x76, 0x99, 0x21, 0xd2, 0x80, - 0x1f, 0xc1, 0x9e, 0x97, 0xe1, 0x00, 0x8a, 0x6a, 0x30, 0xc1, 0xb5, 0x36, 0x35, 0xcf, 0xd1, 0x3c, - 0xd2, 0xd8, 0x15, 0xd6, 0x42, 0x0f, 0xb2, 0x1c, 0xc3, 0x11, 0xc8, 0xbd, 0xbd, 0xe2, 0x19, 0xb1, - 0x36, 0x51, 0x00, 0x8e, 0xa1, 0x40, 0x1f, 0x88, 0x19, 0x6f, 0xcd, 0xe4, 0xcf, 0x3f, 0xc8, 0xd7, - 0xe2, 0x30, 0x26, 0x5c, 0xbd, 0x46, 0x93, 0xeb, 0x21, 0x07, 0xcc, 0xec, 0xe3, 0x30, 0x26, 0xef, - 0x9a, 0x43, 0x05, 0xb1, 0xf3, 0x00, 0x25, 0x07, 0x7b, 0xdc, 0x71, 0x7e, 0xd4, 0x9f, 0x56, 0x60, - 0x32, 0x2e, 0xbb, 0xa0, 0x2d, 0x18, 0x12, 0x8c, 0x4c, 0x50, 0x9d, 0xcf, 0x6b, 0xb5, 0x6c, 0x12, - 0xe1, 0x1e, 0xcc, 0x45, 0x61, 0x51, 0x84, 0x7d, 0xf4, 0xb2, 0x47, 0x42, 0xa1, 0x83, 0x47, 0xc2, - 0x13, 0x70, 0x36, 0x9d, 0xa5, 0xd1, 0x8b, 0x84, 0x46, 0xef, 0x57, 0x42, 0x69, 0x1d, 0x5c, 0x24, - 0xd8, 0xa5, 0x0b, 0x73, 0x98, 0xfa, 0x41, 0x88, 0xe7, 0x3e, 0x43, 0x2f, 0xc3, 0x88, 0xeb, 0x6e, - 0x71, 0x3b, 0x33, 0x31, 0xc8, 0x7c, 0x4f, 0x3d, 0x7e, 0x1e, 0x1a, 0x7e, 0xf7, 0x09, 0x7e, 0xe2, - 0x10, 0xfd, 0xc2, 0xf3, 0x5f, 0xfa, 0xda, 0xe5, 0x37, 0xfd, 0xc1, 0xd7, 0x2e, 0xbf, 0xe9, 0xab, - 0x5f, 0xbb, 0xfc, 0xa6, 0x0f, 0xef, 0x5f, 0x56, 0xbe, 0xb4, 0x7f, 0x59, 0xf9, 0x83, 0xfd, 0xcb, - 0xca, 0x57, 0xf7, 0x2f, 0x2b, 0x7f, 0xb9, 0x7f, 0x59, 0xf9, 0xa1, 0x7f, 0x77, 0xf9, 0x4d, 0x2f, - 0x3c, 0x1c, 0x52, 0xbf, 0xe6, 0x13, 0x0d, 0xff, 0x69, 0xdd, 0x6e, 0xd0, 0x4b, 0xab, 0xeb, 0xdf, - 0x5a, 0x19, 0xf5, 0xff, 0x1d, 0x00, 0x00, 0xff, 0xff, 0xd6, 0xf4, 0xb6, 0x47, 0x63, 0x25, 0x01, - 0x00, + 0xd8, 0xa6, 0x87, 0xb1, 0xd5, 0xcc, 0x78, 0xec, 0xf1, 0x8c, 0x99, 0x87, 0x54, 0x25, 0x75, 0x97, + 0x5b, 0x52, 0x6b, 0x6e, 0x49, 0x33, 0xe3, 0x19, 0x18, 0x93, 0xca, 0xba, 0x2a, 0xe5, 0x74, 0x56, + 0x66, 0x4d, 0x66, 0x96, 0x5a, 0xea, 0xb1, 0xf1, 0x03, 0xcc, 0x67, 0x0f, 0x86, 0x0f, 0xf8, 0x00, + 0x7f, 0xb6, 0x21, 0x18, 0x4c, 0x60, 0xbe, 0x6f, 0xd9, 0x35, 0x1b, 0x10, 0x6c, 0x04, 0x10, 0x10, + 0x0b, 0x11, 0x2c, 0x86, 0xc0, 0x04, 0x81, 0x61, 0xd7, 0x2c, 0x50, 0x60, 0xad, 0x17, 0x36, 0x62, + 0x9f, 0xb1, 0x44, 0x2c, 0x11, 0xbd, 0x04, 0xbb, 0x71, 0x1f, 0x99, 0x79, 0xf3, 0x55, 0x2a, 0x65, + 0x49, 0xf2, 0xcc, 0xe2, 0x5f, 0x52, 0xdd, 0x73, 0xef, 0x39, 0xf7, 0x95, 0xe7, 0x9e, 0x7b, 0xee, + 0x79, 0xc0, 0x42, 0xdd, 0xf0, 0xb6, 0x5b, 0x9b, 0x73, 0xba, 0xdd, 0xb8, 0x56, 0xd7, 0x9c, 0x1a, + 0xb1, 0x88, 0x13, 0xfe, 0xd3, 0xbc, 0x5d, 0xbf, 0xa6, 0x35, 0x0d, 0xf7, 0x9a, 0x6e, 0x3b, 0xe4, + 0xda, 0xce, 0x83, 0x9b, 0xc4, 0xd3, 0x1e, 0xbc, 0x56, 0xa7, 0x30, 0xcd, 0x23, 0xb5, 0xb9, 0xa6, + 0x63, 0x7b, 0x36, 0x7a, 0x28, 0xc4, 0x31, 0xe7, 0x37, 0x0d, 0xff, 0x69, 0xde, 0xae, 0xcf, 0x51, + 0x1c, 0x73, 0x14, 0xc7, 0x9c, 0xc0, 0x31, 0xfb, 0x0e, 0x99, 0xae, 0x5d, 0xb7, 0xaf, 0x31, 0x54, + 0x9b, 0xad, 0x2d, 0xf6, 0x8b, 0xfd, 0x60, 0xff, 0x71, 0x12, 0xb3, 0xf7, 0xdf, 0x7e, 0x8f, 0x3b, + 0x67, 0xd8, 0xb4, 0x33, 0xd7, 0xb4, 0x96, 0x67, 0xbb, 0xba, 0x66, 0x1a, 0x56, 0xfd, 0xda, 0x4e, + 0xa2, 0x37, 0xb3, 0xaa, 0x54, 0x55, 0x74, 0xbb, 0x63, 0x1d, 0x67, 0x53, 0xd3, 0xd3, 0xea, 0xdc, + 0x08, 0xeb, 0x90, 0x5d, 0x8f, 0x58, 0xae, 0x61, 0x5b, 0xee, 0x3b, 0xe8, 0x48, 0x88, 0xb3, 0x23, + 0xcf, 0x4d, 0xa4, 0x42, 0x1a, 0xa6, 0x87, 0x43, 0x4c, 0x0d, 0x4d, 0xdf, 0x36, 0x2c, 0xe2, 0xec, + 0xf9, 0xcd, 0xaf, 0x39, 0xc4, 0xb5, 0x5b, 0x8e, 0x4e, 0x0e, 0xd5, 0xca, 0xbd, 0xd6, 0x20, 0x9e, + 0x96, 0x46, 0xeb, 0x5a, 0x56, 0x2b, 0xa7, 0x65, 0x79, 0x46, 0x23, 0x49, 0xe6, 0xdd, 0x07, 0x35, + 0x70, 0xf5, 0x6d, 0xd2, 0xd0, 0x12, 0xed, 0xde, 0x99, 0xd5, 0xae, 0xe5, 0x19, 0xe6, 0x35, 0xc3, + 0xf2, 0x5c, 0xcf, 0x89, 0x37, 0x52, 0x5f, 0x55, 0x60, 0x72, 0x7e, 0xad, 0x52, 0x65, 0x33, 0xb8, + 0x6c, 0xd7, 0xeb, 0x86, 0x55, 0x47, 0x0f, 0xc0, 0xc8, 0x0e, 0x71, 0x36, 0x6d, 0xd7, 0xf0, 0xf6, + 0x66, 0x94, 0x2b, 0xca, 0xd5, 0x81, 0x85, 0xf1, 0xfd, 0x76, 0x71, 0xe4, 0x19, 0xbf, 0x10, 0x87, + 0x70, 0x54, 0x81, 0xd3, 0xdb, 0x9e, 0xd7, 0x9c, 0xd7, 0x75, 0xe2, 0xba, 0x41, 0x8d, 0x99, 0x02, + 0x6b, 0x76, 0x6e, 0xbf, 0x5d, 0x3c, 0x7d, 0x63, 0x7d, 0x7d, 0x2d, 0x06, 0xc6, 0x69, 0x6d, 0xd4, + 0x5f, 0x54, 0x60, 0x2a, 0xe8, 0x0c, 0x26, 0x2f, 0xb7, 0x88, 0xeb, 0xb9, 0x08, 0xc3, 0xd9, 0x86, + 0xb6, 0xbb, 0x6a, 0x5b, 0x2b, 0x2d, 0x4f, 0xf3, 0x0c, 0xab, 0x5e, 0xb1, 0xb6, 0x4c, 0xa3, 0xbe, + 0xed, 0x89, 0xae, 0xcd, 0xee, 0xb7, 0x8b, 0x67, 0x57, 0x52, 0x6b, 0xe0, 0x8c, 0x96, 0xb4, 0xd3, + 0x0d, 0x6d, 0x37, 0x81, 0x50, 0xea, 0xf4, 0x4a, 0x12, 0x8c, 0xd3, 0xda, 0xa8, 0xef, 0x82, 0x29, + 0x3e, 0x0e, 0x4c, 0x5c, 0xcf, 0x31, 0x74, 0xcf, 0xb0, 0x2d, 0x74, 0x05, 0xfa, 0x2d, 0xad, 0x41, + 0x58, 0x0f, 0x47, 0x16, 0xc6, 0xbe, 0xd4, 0x2e, 0xbe, 0x69, 0xbf, 0x5d, 0xec, 0x5f, 0xd5, 0x1a, + 0x04, 0x33, 0x88, 0xfa, 0x3f, 0x0a, 0x70, 0x31, 0xd1, 0xee, 0x59, 0xc3, 0xdb, 0xbe, 0xd5, 0xa4, + 0xff, 0xb9, 0xe8, 0x07, 0x15, 0x98, 0xd2, 0xe2, 0x15, 0x18, 0xc2, 0xd1, 0x87, 0x16, 0xe7, 0x0e, + 0xff, 0x81, 0xcf, 0x25, 0xa8, 0x2d, 0x9c, 0x17, 0xfd, 0x4a, 0x0e, 0x00, 0x27, 0x49, 0xa3, 0x4f, + 0x28, 0x30, 0x64, 0xf3, 0xce, 0xcd, 0x14, 0xae, 0xf4, 0x5d, 0x1d, 0x7d, 0xe8, 0x3b, 0x8f, 0xa4, + 0x1b, 0xd2, 0xa0, 0xe7, 0xc4, 0xdf, 0x45, 0xcb, 0x73, 0xf6, 0x16, 0x4e, 0x89, 0xee, 0x0d, 0x89, + 0x52, 0xec, 0x93, 0x9f, 0x7d, 0x0c, 0xc6, 0xe4, 0x9a, 0x68, 0x12, 0xfa, 0x6e, 0x13, 0xbe, 0x55, + 0x47, 0x30, 0xfd, 0x17, 0x4d, 0xc3, 0xc0, 0x8e, 0x66, 0xb6, 0x08, 0x5b, 0xd2, 0x11, 0xcc, 0x7f, + 0x3c, 0x56, 0x78, 0x8f, 0xa2, 0x3e, 0x04, 0x03, 0xf3, 0xb5, 0x9a, 0x6d, 0xa1, 0xfb, 0x61, 0x88, + 0x58, 0xda, 0xa6, 0x49, 0x6a, 0xac, 0xe1, 0x70, 0x48, 0x6f, 0x91, 0x17, 0x63, 0x1f, 0xae, 0xfe, + 0x58, 0x01, 0x06, 0x59, 0x23, 0x17, 0xfd, 0x88, 0x02, 0xa7, 0x6f, 0xb7, 0x36, 0x89, 0x63, 0x11, + 0x8f, 0xb8, 0x65, 0xcd, 0xdd, 0xde, 0xb4, 0x35, 0xa7, 0x26, 0x16, 0xe6, 0x7a, 0x9e, 0x19, 0xb9, + 0x99, 0x44, 0xc7, 0xf7, 0x60, 0x0a, 0x00, 0xa7, 0x11, 0x47, 0x3b, 0x30, 0x66, 0xd5, 0x0d, 0x6b, + 0xb7, 0x62, 0xd5, 0x1d, 0xe2, 0xba, 0x6c, 0xd0, 0xa3, 0x0f, 0x3d, 0x95, 0xa7, 0x33, 0xab, 0x12, + 0x9e, 0x85, 0xc9, 0xfd, 0x76, 0x71, 0x4c, 0x2e, 0xc1, 0x11, 0x3a, 0xea, 0x3f, 0x28, 0x70, 0x6a, + 0xbe, 0xd6, 0x30, 0x5c, 0xca, 0x69, 0xd7, 0xcc, 0x56, 0xdd, 0xe8, 0x62, 0xeb, 0xa3, 0xa7, 0x61, + 0x50, 0xb7, 0xad, 0x2d, 0xa3, 0x2e, 0xfa, 0xf9, 0x8e, 0x39, 0xce, 0xb9, 0xe6, 0x64, 0xce, 0xc5, + 0xba, 0x27, 0x38, 0xde, 0x1c, 0xd6, 0xee, 0x2c, 0xfa, 0x0c, 0x7d, 0x01, 0xf6, 0xdb, 0xc5, 0xc1, + 0x12, 0x43, 0x80, 0x05, 0x22, 0x74, 0x15, 0x86, 0x6b, 0x86, 0xcb, 0x17, 0xb3, 0x8f, 0x2d, 0xe6, + 0xd8, 0x7e, 0xbb, 0x38, 0x5c, 0x16, 0x65, 0x38, 0x80, 0xa2, 0x65, 0x98, 0xa6, 0x33, 0xc8, 0xdb, + 0x55, 0x89, 0xee, 0x10, 0x8f, 0x76, 0x6d, 0xa6, 0x9f, 0x75, 0x77, 0x66, 0xbf, 0x5d, 0x9c, 0xbe, + 0x99, 0x02, 0xc7, 0xa9, 0xad, 0xd4, 0x25, 0x18, 0x9e, 0x37, 0x89, 0x43, 0x19, 0x02, 0x7a, 0x0c, + 0x26, 0x48, 0x43, 0x33, 0x4c, 0x4c, 0x74, 0x62, 0xec, 0x10, 0xc7, 0x9d, 0x51, 0xae, 0xf4, 0x5d, + 0x1d, 0x59, 0x40, 0xfb, 0xed, 0xe2, 0xc4, 0x62, 0x04, 0x82, 0x63, 0x35, 0xd5, 0x8f, 0x2a, 0x30, + 0x3a, 0xdf, 0xaa, 0x19, 0x1e, 0x1f, 0x17, 0x72, 0x60, 0x54, 0xa3, 0x3f, 0xd7, 0x6c, 0xd3, 0xd0, + 0xf7, 0xc4, 0xe6, 0x7a, 0x32, 0xd7, 0xe7, 0x16, 0xa2, 0x59, 0x38, 0xb5, 0xdf, 0x2e, 0x8e, 0x4a, + 0x05, 0x58, 0x26, 0xa2, 0x6e, 0x83, 0x0c, 0x43, 0xef, 0x87, 0x31, 0x3e, 0xdc, 0x15, 0xad, 0x89, + 0xc9, 0x96, 0xe8, 0xc3, 0x7d, 0xd2, 0x5a, 0xf9, 0x84, 0xe6, 0x6e, 0x6d, 0xbe, 0x44, 0x74, 0x0f, + 0x93, 0x2d, 0xe2, 0x10, 0x4b, 0x27, 0x7c, 0xdb, 0x94, 0xa4, 0xc6, 0x38, 0x82, 0x4a, 0xfd, 0x7f, + 0x14, 0xb8, 0x34, 0xdf, 0xf2, 0xb6, 0x6d, 0xc7, 0xb8, 0x4b, 0x9c, 0x70, 0xba, 0x03, 0x0c, 0xe8, + 0x09, 0x98, 0xd0, 0x82, 0x0a, 0xab, 0xe1, 0x76, 0x3a, 0x2b, 0xb6, 0xd3, 0xc4, 0x7c, 0x04, 0x8a, + 0x63, 0xb5, 0xd1, 0x43, 0x00, 0x6e, 0xb8, 0xb6, 0x8c, 0x07, 0x2c, 0x20, 0xd1, 0x16, 0xa4, 0x55, + 0x95, 0x6a, 0xa9, 0x7f, 0x49, 0x8f, 0xc2, 0x1d, 0xcd, 0x30, 0xb5, 0x4d, 0xc3, 0x34, 0xbc, 0xbd, + 0xe7, 0x6d, 0x8b, 0x74, 0xb1, 0x9b, 0x37, 0xe0, 0x5c, 0xcb, 0xd2, 0x78, 0x3b, 0x93, 0xac, 0xf0, + 0xfd, 0xbb, 0xbe, 0xd7, 0x24, 0x9c, 0x4b, 0x8e, 0x2c, 0x5c, 0xd8, 0x6f, 0x17, 0xcf, 0x6d, 0xa4, + 0x57, 0xc1, 0x59, 0x6d, 0xe9, 0xa9, 0x27, 0x81, 0x9e, 0xb1, 0xcd, 0x56, 0x43, 0x60, 0xed, 0x63, + 0x58, 0xd9, 0xa9, 0xb7, 0x91, 0x5a, 0x03, 0x67, 0xb4, 0x54, 0xbf, 0x50, 0x80, 0xc1, 0x05, 0x4d, + 0xbf, 0xdd, 0x6a, 0xa2, 0xb7, 0xc3, 0x70, 0xd3, 0xb1, 0x77, 0x8c, 0x1a, 0x71, 0xc4, 0xd8, 0x26, + 0xc5, 0xd8, 0x86, 0xd7, 0x44, 0x39, 0x0e, 0x6a, 0x20, 0x03, 0x26, 0xfc, 0xff, 0x4b, 0x3d, 0x7c, + 0xb9, 0xec, 0x4b, 0x58, 0x8b, 0x20, 0xc2, 0x31, 0xc4, 0x48, 0x85, 0x41, 0x87, 0xd4, 0xe9, 0x51, + 0xd7, 0xc7, 0xba, 0xc5, 0xbe, 0x76, 0xcc, 0x4a, 0xb0, 0x80, 0xa0, 0x0f, 0xc0, 0x84, 0xee, 0x90, + 0x1a, 0xb1, 0x3c, 0x43, 0x33, 0x5d, 0xba, 0x39, 0x07, 0xba, 0xdf, 0x9c, 0xac, 0x13, 0xa5, 0x48, + 0x73, 0x1c, 0x43, 0xa7, 0x7e, 0xa9, 0x00, 0x63, 0x7c, 0xa2, 0x16, 0x5a, 0xfa, 0x6d, 0xe2, 0xa1, + 0xef, 0x82, 0x61, 0x2a, 0xdf, 0xd5, 0x34, 0x4f, 0x13, 0x1f, 0xc2, 0xb7, 0x65, 0x0e, 0x9d, 0x7d, + 0x83, 0xb4, 0x76, 0x48, 0x7d, 0x85, 0x78, 0x5a, 0xb8, 0xff, 0xc2, 0x32, 0x1c, 0x60, 0x45, 0x5b, + 0xd0, 0xef, 0x36, 0x89, 0x2e, 0x26, 0xb6, 0x9c, 0xe7, 0x53, 0x97, 0x7b, 0x5c, 0x6d, 0x12, 0x3d, + 0xdc, 0xae, 0xf4, 0x17, 0x66, 0xf8, 0x91, 0x05, 0x83, 0xae, 0xa7, 0x79, 0x2d, 0x97, 0xcd, 0xef, + 0xe8, 0x43, 0x4b, 0x3d, 0x53, 0x62, 0xd8, 0x16, 0x26, 0x04, 0xad, 0x41, 0xfe, 0x1b, 0x0b, 0x2a, + 0xea, 0xbf, 0x51, 0x60, 0x52, 0xae, 0xbe, 0x6c, 0xb8, 0x1e, 0xfa, 0x8e, 0xc4, 0x74, 0xce, 0x75, + 0x37, 0x9d, 0xb4, 0x35, 0x9b, 0xcc, 0x60, 0xb7, 0xfa, 0x25, 0xd2, 0x54, 0x12, 0x18, 0x30, 0x3c, + 0xd2, 0xf0, 0xa5, 0x94, 0xa7, 0x7a, 0x1d, 0xe1, 0xc2, 0xb8, 0x20, 0x36, 0x50, 0xa1, 0x68, 0x31, + 0xc7, 0xae, 0x7e, 0x17, 0x4c, 0xcb, 0xb5, 0xfc, 0x7d, 0x4d, 0x59, 0x86, 0xb7, 0xd7, 0x4c, 0xb0, + 0x0c, 0xfa, 0x09, 0x62, 0x06, 0x41, 0x6f, 0x0b, 0xf6, 0x38, 0xe7, 0x4c, 0xc1, 0xdc, 0x45, 0xf7, + 0xb9, 0xfa, 0x91, 0xbe, 0xe8, 0xdc, 0xd1, 0x65, 0x44, 0x3b, 0xb1, 0x2f, 0x77, 0xf4, 0xa1, 0x1b, + 0xbd, 0x0e, 0xd0, 0xef, 0xfa, 0xeb, 0x85, 0x07, 0x5c, 0x85, 0x61, 0x97, 0x90, 0x9a, 0x74, 0x2e, + 0xb3, 0xd3, 0xbc, 0x2a, 0xca, 0x70, 0x00, 0x3d, 0x7e, 0x4e, 0xf0, 0x5a, 0x3f, 0xa0, 0xe4, 0x6e, + 0x97, 0x27, 0x83, 0x97, 0x88, 0xa5, 0xe8, 0x65, 0x32, 0xc4, 0x87, 0x13, 0x43, 0x8c, 0xee, 0xc2, + 0xb8, 0xa9, 0xb9, 0xde, 0xad, 0x26, 0xbd, 0xb7, 0xf9, 0x7b, 0x66, 0xf4, 0xa1, 0xf9, 0x3c, 0x8b, + 0xbe, 0x2c, 0x23, 0x5a, 0x98, 0xda, 0x6f, 0x17, 0xc7, 0x23, 0x45, 0x38, 0x4a, 0x0a, 0xbd, 0x04, + 0x23, 0xb4, 0x60, 0xd1, 0x71, 0x6c, 0x47, 0xf0, 0x8b, 0xc7, 0xf3, 0xd2, 0x65, 0x48, 0xf8, 0x3d, + 0x32, 0xf8, 0x89, 0x43, 0xf4, 0xe8, 0x7d, 0x80, 0xec, 0x4d, 0x76, 0x93, 0xaf, 0x5d, 0xe7, 0x97, + 0x54, 0x3a, 0x58, 0xba, 0xfc, 0x7d, 0x0b, 0xb3, 0x62, 0x5f, 0xa2, 0x5b, 0x89, 0x1a, 0x38, 0xa5, + 0x15, 0xba, 0x0d, 0x28, 0xb8, 0xe8, 0xf2, 0xd3, 0xfe, 0x80, 0xad, 0x11, 0x54, 0x12, 0x5b, 0xe3, + 0x2c, 0x25, 0x76, 0x3d, 0x81, 0x02, 0xa7, 0xa0, 0x55, 0x7f, 0xbb, 0x00, 0xa3, 0x7c, 0x8b, 0xf0, + 0xcb, 0xc8, 0xf1, 0x9f, 0x15, 0x24, 0x72, 0x56, 0x94, 0xf2, 0x7f, 0xfe, 0xac, 0xc3, 0x99, 0x47, + 0x45, 0x23, 0x76, 0x54, 0x2c, 0xf6, 0x4a, 0xa8, 0xf3, 0x49, 0xf1, 0x27, 0x0a, 0x9c, 0x92, 0x6a, + 0x9f, 0xc0, 0x41, 0x51, 0x8b, 0x1e, 0x14, 0x4f, 0xf6, 0x38, 0xbe, 0x8c, 0x73, 0xc2, 0x8e, 0x0c, + 0x8b, 0xf1, 0xf0, 0x87, 0x00, 0x36, 0x19, 0x3b, 0x91, 0x44, 0xdb, 0x60, 0xc9, 0x17, 0x02, 0x08, + 0x96, 0x6a, 0x45, 0x98, 0x62, 0xa1, 0x13, 0x53, 0x54, 0xff, 0x7d, 0x1f, 0x4c, 0x25, 0xa6, 0x3d, + 0xc9, 0x47, 0x94, 0x6f, 0x10, 0x1f, 0x29, 0x7c, 0x23, 0xf8, 0x48, 0x5f, 0x2e, 0x3e, 0xd2, 0xfd, + 0x41, 0xe4, 0x00, 0x6a, 0x18, 0x75, 0xde, 0xac, 0xea, 0x69, 0x8e, 0xb7, 0x6e, 0x34, 0x88, 0xe0, + 0x38, 0xdf, 0xda, 0xdd, 0x96, 0xa5, 0x2d, 0x38, 0xe3, 0x59, 0x49, 0x60, 0xc2, 0x29, 0xd8, 0xd5, + 0xef, 0x29, 0xc0, 0xd0, 0x82, 0xe6, 0xb2, 0x9e, 0x7e, 0x08, 0xc6, 0x04, 0xea, 0x4a, 0x43, 0xab, + 0x93, 0x5e, 0xd4, 0x11, 0x02, 0xe5, 0x8a, 0x84, 0x8e, 0xdf, 0xe8, 0xe4, 0x12, 0x1c, 0x21, 0x87, + 0xf6, 0x60, 0xb4, 0x11, 0xde, 0x5e, 0xc4, 0x12, 0x2f, 0xf5, 0x4e, 0x9d, 0x62, 0xe3, 0xd7, 0x56, + 0xa9, 0x00, 0xcb, 0xb4, 0xd4, 0x17, 0xe1, 0x74, 0x4a, 0x8f, 0xbb, 0xb8, 0xb8, 0xbd, 0x15, 0x86, + 0xe8, 0xdd, 0x3b, 0x14, 0xc3, 0x46, 0xf7, 0xdb, 0xc5, 0xa1, 0x67, 0x78, 0x11, 0xf6, 0x61, 0xea, + 0xbb, 0xa9, 0x00, 0x10, 0xef, 0x53, 0x37, 0x0a, 0xbe, 0x01, 0x80, 0xd2, 0x3c, 0xb6, 0x3d, 0xbe, + 0x95, 0x9e, 0x84, 0x81, 0xe6, 0xb6, 0xe6, 0xfa, 0x2d, 0xee, 0xf7, 0x59, 0xc5, 0x1a, 0x2d, 0xbc, + 0xd7, 0x2e, 0xce, 0xc8, 0x82, 0x88, 0x68, 0xc4, 0x60, 0x98, 0xb7, 0xa3, 0x3b, 0x8c, 0x6e, 0xf2, + 0x92, 0xdd, 0x68, 0x9a, 0x84, 0x42, 0xd9, 0x0e, 0x2b, 0xe4, 0xdb, 0x61, 0xcb, 0x09, 0x4c, 0x38, + 0x05, 0xbb, 0x4f, 0xb3, 0x62, 0x19, 0x9e, 0xa1, 0x05, 0x34, 0xfb, 0xf2, 0xd3, 0x8c, 0x62, 0xc2, + 0x29, 0xd8, 0xd1, 0xab, 0x0a, 0xcc, 0x46, 0x8b, 0x97, 0x0c, 0xcb, 0x70, 0xb7, 0x49, 0x8d, 0x11, + 0xef, 0x3f, 0x34, 0xf1, 0xcb, 0xfb, 0xed, 0xe2, 0xec, 0x72, 0x26, 0x46, 0xdc, 0x81, 0x1a, 0xfa, + 0x01, 0x05, 0x2e, 0xc4, 0xe6, 0xc5, 0x31, 0xea, 0x75, 0xe2, 0x88, 0xde, 0x1c, 0xfe, 0x03, 0x2f, + 0xee, 0xb7, 0x8b, 0x17, 0x96, 0xb3, 0x51, 0xe2, 0x4e, 0xf4, 0xd0, 0x4f, 0x2b, 0x70, 0xb6, 0x49, + 0xac, 0x9a, 0x61, 0xd5, 0x9f, 0xb5, 0x9d, 0xdb, 0xc4, 0x71, 0xb1, 0x6d, 0x9a, 0x76, 0xcb, 0x73, + 0x67, 0x06, 0xd9, 0x19, 0x56, 0xc9, 0xf3, 0xcd, 0xad, 0xa5, 0x61, 0x5c, 0xb8, 0x2c, 0xb6, 0xe8, + 0xd9, 0x54, 0xb0, 0x8b, 0x33, 0x3a, 0xa2, 0xfe, 0x96, 0x02, 0x7d, 0x25, 0x5c, 0x41, 0x0f, 0x44, + 0x3e, 0x91, 0x73, 0xf2, 0x27, 0x72, 0xaf, 0x5d, 0x1c, 0x2a, 0xe1, 0x8a, 0xf4, 0x31, 0xfe, 0x80, + 0x02, 0x53, 0xba, 0x6d, 0x79, 0x1a, 0x9d, 0x3b, 0xcc, 0x65, 0x65, 0xff, 0x5c, 0xce, 0x75, 0x19, + 0x2e, 0xc5, 0x90, 0x85, 0xca, 0xee, 0x38, 0xc4, 0xc5, 0x49, 0xca, 0xea, 0xe7, 0x15, 0x98, 0x2e, + 0x69, 0x4d, 0xa1, 0x0a, 0x2a, 0x93, 0x2d, 0x83, 0x6e, 0x90, 0x6e, 0x34, 0xfb, 0x68, 0x1b, 0x06, + 0x99, 0xb6, 0xd9, 0xed, 0xe5, 0x2e, 0x1f, 0xd2, 0x7e, 0x86, 0xe1, 0xe2, 0x7a, 0x10, 0xfe, 0x3f, + 0x16, 0xf8, 0xd5, 0xc7, 0x61, 0x32, 0x5e, 0x0f, 0x15, 0x7d, 0x99, 0x86, 0x2b, 0x1f, 0x47, 0xe2, + 0xe2, 0xc8, 0x63, 0xc3, 0xff, 0xef, 0x6b, 0xc5, 0x37, 0x7d, 0xe4, 0xcf, 0xaf, 0xbc, 0x49, 0xfd, + 0xaa, 0x02, 0x63, 0x25, 0xd3, 0x6e, 0xd5, 0xd6, 0x1c, 0x7b, 0xcb, 0x30, 0xc9, 0x1b, 0x43, 0xcb, + 0x21, 0xf7, 0x38, 0x4b, 0x74, 0x65, 0x5a, 0x07, 0xb9, 0xe2, 0x1b, 0x44, 0xeb, 0x20, 0x77, 0x39, + 0x43, 0x9a, 0x7c, 0x01, 0xce, 0xc8, 0xb5, 0x42, 0x95, 0xe9, 0x15, 0xe8, 0xbf, 0x6d, 0x58, 0xb5, + 0xf8, 0xc6, 0xbc, 0x69, 0x58, 0x35, 0xcc, 0x20, 0xc1, 0xd6, 0x2d, 0x64, 0x9e, 0x59, 0x7f, 0x3b, + 0x12, 0x9d, 0x36, 0x26, 0xac, 0x5e, 0x85, 0x61, 0x5d, 0x5b, 0x68, 0x59, 0x35, 0x33, 0xd8, 0xf5, + 0x74, 0x0a, 0x4a, 0xf3, 0xbc, 0x0c, 0x07, 0x50, 0x74, 0x17, 0x20, 0x7c, 0x9d, 0xe8, 0x45, 0x08, + 0x08, 0x1f, 0x3e, 0xaa, 0xc4, 0xf3, 0x0c, 0xab, 0xee, 0x86, 0xfb, 0x2a, 0x84, 0x61, 0x89, 0x1a, + 0xfa, 0x10, 0x8c, 0xcb, 0x12, 0x09, 0x57, 0x93, 0xe6, 0x5c, 0x86, 0x88, 0xe8, 0x73, 0x46, 0x10, + 0x1e, 0x97, 0x4b, 0x5d, 0x1c, 0xa5, 0x86, 0xf6, 0x02, 0xf9, 0x8b, 0x2b, 0x69, 0xfb, 0xf3, 0xdf, + 0x28, 0x64, 0xd1, 0x67, 0x5a, 0x10, 0x1f, 0x8b, 0x28, 0x8d, 0x23, 0xa4, 0x52, 0x14, 0x33, 0x03, + 0xc7, 0xa5, 0x98, 0x21, 0x30, 0xc4, 0x55, 0x53, 0xfe, 0x71, 0xf3, 0x58, 0x9e, 0x01, 0x72, 0x2d, + 0x57, 0xf8, 0xdc, 0xc6, 0x7f, 0xbb, 0xd8, 0xc7, 0x8d, 0x76, 0x60, 0x8c, 0x0a, 0xd6, 0x55, 0x62, + 0x12, 0xdd, 0xb3, 0x9d, 0x99, 0xa1, 0xfc, 0xcf, 0x59, 0x55, 0x09, 0x0f, 0x97, 0x62, 0xe5, 0x12, + 0x1c, 0xa1, 0x13, 0x68, 0xee, 0x86, 0x33, 0x35, 0x77, 0x2d, 0x18, 0xdd, 0x91, 0x54, 0xf1, 0x23, + 0x6c, 0x12, 0x9e, 0xc8, 0xd3, 0xb1, 0x50, 0x2f, 0xbf, 0x70, 0x5a, 0x10, 0x1a, 0x95, 0x75, 0xf8, + 0x32, 0x1d, 0xb4, 0x09, 0x43, 0x9b, 0x5c, 0x06, 0x9d, 0x01, 0x36, 0x17, 0xef, 0xed, 0x41, 0xb4, + 0xe6, 0x72, 0xae, 0xf8, 0x81, 0x7d, 0xc4, 0xe8, 0x45, 0x18, 0x34, 0x8d, 0x86, 0xe1, 0xb9, 0x33, + 0xa3, 0x8c, 0x44, 0xae, 0xa5, 0x5d, 0x66, 0x18, 0xf8, 0x61, 0xc5, 0xff, 0xc7, 0x02, 0x2b, 0xfa, + 0xb4, 0x02, 0xa7, 0xc5, 0x36, 0x0c, 0x0e, 0x2d, 0x83, 0xb8, 0x33, 0x63, 0x6c, 0x0e, 0x6f, 0xf4, + 0x76, 0x48, 0x86, 0x07, 0xf4, 0xc2, 0x05, 0x31, 0x9b, 0xa7, 0x57, 0x92, 0xc4, 0x70, 0x5a, 0x0f, + 0xd4, 0x57, 0xa7, 0x60, 0xaa, 0x64, 0xb6, 0x5c, 0x8f, 0x38, 0xf3, 0xc2, 0x92, 0x85, 0x38, 0xe8, + 0x63, 0x0a, 0x9c, 0x65, 0xff, 0x96, 0xed, 0x3b, 0x56, 0x99, 0x98, 0xda, 0xde, 0xfc, 0x16, 0xad, + 0x51, 0xab, 0x1d, 0xee, 0xf0, 0x28, 0xb7, 0xc4, 0x35, 0x99, 0xbd, 0xd8, 0x54, 0x53, 0x31, 0xe2, + 0x0c, 0x4a, 0xe8, 0xfb, 0x15, 0x38, 0x9f, 0x02, 0x2a, 0x13, 0x93, 0x78, 0xbe, 0xf0, 0x7f, 0xd8, + 0x7e, 0x5c, 0xda, 0x6f, 0x17, 0xcf, 0x57, 0xb3, 0x90, 0xe2, 0x6c, 0x7a, 0xe8, 0x07, 0x15, 0x98, + 0x4d, 0x81, 0x2e, 0x69, 0x86, 0xd9, 0x72, 0xfc, 0x7b, 0xc1, 0x61, 0xbb, 0xc3, 0xc4, 0xf3, 0x6a, + 0x26, 0x56, 0xdc, 0x81, 0x22, 0xfa, 0x30, 0x9c, 0x09, 0xa0, 0x1b, 0x96, 0x45, 0x48, 0x2d, 0x72, + 0x4b, 0x38, 0x6c, 0x57, 0xce, 0xef, 0xb7, 0x8b, 0x67, 0xaa, 0x69, 0x08, 0x71, 0x3a, 0x1d, 0x54, + 0x87, 0x4b, 0x21, 0xc0, 0x33, 0x4c, 0xe3, 0x2e, 0xbf, 0xc8, 0x6c, 0x3b, 0xc4, 0xdd, 0xb6, 0xcd, + 0x1a, 0x63, 0xc5, 0xca, 0xc2, 0x9b, 0xf7, 0xdb, 0xc5, 0x4b, 0xd5, 0x4e, 0x15, 0x71, 0x67, 0x3c, + 0xa8, 0x06, 0x63, 0xae, 0xae, 0x59, 0x15, 0xcb, 0x23, 0xce, 0x8e, 0x66, 0xce, 0x0c, 0xe6, 0x1a, + 0x20, 0x67, 0x80, 0x12, 0x1e, 0x1c, 0xc1, 0x8a, 0xde, 0x03, 0xc3, 0x64, 0xb7, 0xa9, 0x59, 0x35, + 0xc2, 0x99, 0xee, 0xc8, 0xc2, 0x45, 0x7a, 0xd4, 0x2f, 0x8a, 0xb2, 0x7b, 0xed, 0xe2, 0x98, 0xff, + 0xff, 0x8a, 0x5d, 0x23, 0x38, 0xa8, 0x8d, 0x3e, 0x08, 0xd3, 0xcc, 0xd4, 0xa6, 0x46, 0xd8, 0x11, + 0xe2, 0xfa, 0x77, 0xc5, 0xe1, 0x5c, 0xfd, 0x64, 0xcf, 0xf0, 0x2b, 0x29, 0xf8, 0x70, 0x2a, 0x15, + 0xba, 0x0c, 0x0d, 0x6d, 0xf7, 0xba, 0xa3, 0xe9, 0x64, 0xab, 0x65, 0xae, 0x13, 0xa7, 0x61, 0x58, + 0x5c, 0x59, 0x42, 0x74, 0xdb, 0xaa, 0x51, 0x46, 0xad, 0x5c, 0x1d, 0xe0, 0xcb, 0xb0, 0xd2, 0xa9, + 0x22, 0xee, 0x8c, 0x07, 0x3d, 0x0c, 0x63, 0x46, 0xdd, 0xb2, 0x1d, 0xb2, 0xae, 0x19, 0x96, 0xe7, + 0xce, 0x00, 0x13, 0xb2, 0xd9, 0xb4, 0x56, 0xa4, 0x72, 0x1c, 0xa9, 0x85, 0x76, 0x00, 0x59, 0xe4, + 0xce, 0x9a, 0x5d, 0x63, 0x5b, 0x60, 0xa3, 0xc9, 0x36, 0xb2, 0x60, 0xb3, 0x87, 0x9d, 0x1a, 0x76, + 0x95, 0x5e, 0x4d, 0x60, 0xc3, 0x29, 0x14, 0xd0, 0x12, 0xa0, 0x86, 0xb6, 0xbb, 0xd8, 0x68, 0x7a, + 0x7b, 0x0b, 0x2d, 0xf3, 0xb6, 0xe0, 0x1a, 0x63, 0x6c, 0x2e, 0xb8, 0xa2, 0x29, 0x01, 0xc5, 0x29, + 0x2d, 0x90, 0x06, 0x17, 0xf8, 0x78, 0xca, 0x1a, 0x69, 0xd8, 0x96, 0x4b, 0x3c, 0x57, 0xda, 0xa4, + 0x33, 0xe3, 0xcc, 0xe0, 0x82, 0x5d, 0x6c, 0x2b, 0xd9, 0xd5, 0x70, 0x27, 0x1c, 0x51, 0x93, 0xb3, + 0x89, 0x03, 0x4c, 0xce, 0x1e, 0x81, 0x71, 0xd7, 0xd3, 0x1c, 0xaf, 0xd5, 0x14, 0xcb, 0x70, 0x8a, + 0x2d, 0x03, 0xd3, 0x43, 0x56, 0x65, 0x00, 0x8e, 0xd6, 0xa3, 0xcb, 0xc7, 0x95, 0xcd, 0xa2, 0xdd, + 0x64, 0xb8, 0x7c, 0x55, 0xa9, 0x1c, 0x47, 0x6a, 0xa1, 0x0d, 0x38, 0xd7, 0xd0, 0x76, 0x83, 0xcf, + 0x77, 0x4d, 0x73, 0x34, 0xd3, 0x24, 0xa6, 0xe1, 0x36, 0x66, 0xa6, 0x58, 0x4f, 0xd9, 0x0b, 0xff, + 0x4a, 0x7a, 0x15, 0x9c, 0xd5, 0x56, 0xd8, 0xa0, 0x95, 0x1d, 0xcd, 0x88, 0xa0, 0x44, 0x11, 0x1b, + 0xb4, 0x38, 0x18, 0xa7, 0xb5, 0x41, 0x3f, 0xa9, 0x40, 0x91, 0x9d, 0x7f, 0x9a, 0x49, 0x3f, 0x8e, + 0xeb, 0x8e, 0xdd, 0x6a, 0x2e, 0x68, 0xfa, 0x6d, 0x7b, 0x6b, 0xcb, 0xdf, 0x38, 0x33, 0xa7, 0x73, + 0x6d, 0xb7, 0xfb, 0xf6, 0xdb, 0xc5, 0x62, 0xa5, 0x33, 0x6a, 0x7c, 0x10, 0x6d, 0xf4, 0xc3, 0x0a, + 0x5c, 0x10, 0x1f, 0x6e, 0x6a, 0xdf, 0xa6, 0x73, 0xf5, 0x8d, 0xed, 0xb8, 0x95, 0x6c, 0xb4, 0xb8, + 0x13, 0x4d, 0xf4, 0xa3, 0x0a, 0x5c, 0xb4, 0x62, 0x40, 0x4c, 0x5c, 0xc2, 0x74, 0xab, 0x76, 0xcb, + 0x9b, 0x39, 0x93, 0xab, 0x53, 0x57, 0xf6, 0xdb, 0xc5, 0x8b, 0xab, 0x1d, 0xf0, 0xe2, 0x8e, 0x54, + 0xd5, 0xff, 0xde, 0x0f, 0x33, 0x09, 0x61, 0xc4, 0xb7, 0x09, 0x3c, 0xf0, 0xb8, 0x51, 0x8e, 0xe8, + 0xb8, 0x69, 0xc2, 0x95, 0xa0, 0xc2, 0xf5, 0x66, 0x2b, 0x95, 0x56, 0x81, 0xd1, 0x7a, 0xcb, 0x7e, + 0xbb, 0x78, 0xa5, 0x7a, 0x40, 0x5d, 0x7c, 0x20, 0xb6, 0xec, 0xa3, 0xbc, 0xef, 0x84, 0x8e, 0xf2, + 0x0f, 0xc2, 0xb4, 0x04, 0x70, 0x88, 0x56, 0xdb, 0xeb, 0x41, 0x94, 0x60, 0x27, 0x58, 0x35, 0x05, + 0x1f, 0x4e, 0xa5, 0x92, 0x79, 0x7e, 0x0e, 0x9c, 0xc4, 0xf9, 0xa9, 0xb6, 0xfb, 0x60, 0xa4, 0x64, + 0x5b, 0x35, 0xae, 0xe2, 0x7a, 0x30, 0x62, 0xc0, 0x70, 0x49, 0xbe, 0x06, 0xdd, 0x6b, 0x17, 0xc7, + 0x83, 0x8a, 0xd2, 0xbd, 0xe8, 0xd1, 0xe0, 0xa9, 0x90, 0x2b, 0x17, 0xde, 0x1c, 0x7d, 0xe3, 0xbb, + 0xd7, 0x2e, 0x9e, 0x0a, 0x9a, 0x45, 0x9f, 0xfd, 0xe8, 0xe1, 0x68, 0x6a, 0xae, 0xb7, 0xee, 0x68, + 0x96, 0x6b, 0xf4, 0xa0, 0x63, 0x0e, 0xde, 0x76, 0x96, 0x13, 0xd8, 0x70, 0x0a, 0x05, 0xf4, 0x12, + 0x4c, 0xd0, 0xd2, 0x8d, 0x66, 0x4d, 0xf3, 0x48, 0x4e, 0xd5, 0x72, 0x60, 0x8e, 0xb6, 0x1c, 0xc1, + 0x84, 0x63, 0x98, 0xb9, 0xc1, 0x87, 0xe6, 0xda, 0x16, 0x5b, 0xcf, 0x88, 0xc1, 0x07, 0x2d, 0xc5, + 0x02, 0x8a, 0xee, 0x87, 0xa1, 0x06, 0x71, 0x5d, 0xad, 0x4e, 0x98, 0x80, 0x37, 0x12, 0xde, 0x91, + 0x57, 0x78, 0x31, 0xf6, 0xe1, 0xe8, 0xed, 0x30, 0xa0, 0xdb, 0x35, 0xe2, 0xce, 0x0c, 0xb1, 0x33, + 0x8c, 0x1e, 0xe7, 0x03, 0x25, 0x5a, 0x70, 0xaf, 0x5d, 0x1c, 0x61, 0x2f, 0x61, 0xf4, 0x17, 0xe6, + 0x95, 0xd4, 0x9f, 0x52, 0x60, 0x32, 0xae, 0xf7, 0xec, 0xc2, 0x50, 0xe5, 0xe4, 0x6c, 0x3e, 0xd4, + 0x4f, 0x2b, 0x30, 0x46, 0x7b, 0xe8, 0xd8, 0xe6, 0x9a, 0xa9, 0x59, 0x04, 0x7d, 0x9f, 0x02, 0x93, + 0xdb, 0x46, 0x7d, 0x5b, 0x36, 0xc9, 0x13, 0x37, 0xaf, 0x5c, 0x7a, 0xc3, 0x1b, 0x31, 0x5c, 0x0b, + 0xd3, 0xfb, 0xed, 0xe2, 0x64, 0xbc, 0x14, 0x27, 0x68, 0xaa, 0x7f, 0x55, 0x80, 0x73, 0x72, 0xcf, + 0xe6, 0x43, 0x6f, 0x07, 0xf4, 0x27, 0x0a, 0x40, 0xc3, 0xb0, 0xe6, 0x4d, 0xd3, 0xbe, 0xc3, 0xec, + 0x88, 0xe9, 0x5d, 0xf6, 0x85, 0xbc, 0xfa, 0xea, 0x14, 0x0a, 0x73, 0x2b, 0x01, 0x76, 0xfe, 0xc6, + 0xfc, 0x9c, 0xaf, 0x07, 0x0b, 0x01, 0xf7, 0xda, 0xc5, 0x62, 0xd2, 0xc5, 0x62, 0x0e, 0x0b, 0x3f, + 0x86, 0x65, 0xc3, 0xf5, 0x3e, 0xf6, 0x97, 0x1d, 0xab, 0xf0, 0xc7, 0xe6, 0x70, 0x20, 0xb3, 0x0d, + 0x38, 0x15, 0x23, 0x9c, 0x62, 0x63, 0x5d, 0x96, 0x6d, 0xac, 0x0f, 0x60, 0x52, 0x73, 0xbe, 0x57, + 0xc5, 0xdc, 0xd3, 0x2d, 0xcd, 0xf2, 0xe8, 0x4c, 0x4b, 0x36, 0xd9, 0x7f, 0x51, 0x80, 0x69, 0x31, + 0x01, 0x26, 0xbd, 0x6d, 0x36, 0x4d, 0x7b, 0xaf, 0x41, 0xac, 0x93, 0xb0, 0xbb, 0xf3, 0x3f, 0x82, + 0x42, 0xe6, 0x47, 0xd0, 0x48, 0x7c, 0x04, 0x7d, 0x79, 0x3e, 0x82, 0x80, 0x57, 0x1c, 0xa0, 0x63, + 0xc3, 0x70, 0xd6, 0xb0, 0x68, 0x47, 0xaf, 0xb3, 0x0d, 0x13, 0x5a, 0xc7, 0x32, 0xfe, 0x34, 0xcc, + 0xd5, 0x08, 0x95, 0xd4, 0x1a, 0x38, 0xa3, 0xa5, 0xfa, 0x37, 0x0a, 0xcc, 0xa4, 0xcd, 0xef, 0x09, + 0xa8, 0xc5, 0x1b, 0x51, 0xb5, 0xf8, 0x8d, 0x1e, 0xbe, 0x8d, 0x48, 0xd7, 0x33, 0xd4, 0xe3, 0x7f, + 0x5d, 0x80, 0xb3, 0x61, 0xf5, 0x8a, 0xe5, 0x7a, 0x9a, 0x69, 0x72, 0x81, 0xef, 0xf8, 0xf7, 0x52, + 0x33, 0xf2, 0xba, 0xb1, 0xda, 0xdb, 0x50, 0xe5, 0xbe, 0x67, 0x9a, 0xe8, 0xec, 0xc6, 0x4c, 0x74, + 0xd6, 0x8e, 0x90, 0x66, 0x67, 0x6b, 0x9d, 0xff, 0xa8, 0xc0, 0x6c, 0x7a, 0xc3, 0x13, 0xd8, 0x54, + 0x76, 0x74, 0x53, 0xbd, 0xef, 0xe8, 0x46, 0x9d, 0xb1, 0xad, 0x7e, 0xb1, 0x90, 0x35, 0x5a, 0xf6, + 0x44, 0xb2, 0x05, 0xa7, 0x1c, 0x52, 0x37, 0x5c, 0x4f, 0x5c, 0x3c, 0x0e, 0x67, 0x2e, 0xef, 0x3f, + 0x8d, 0x9e, 0xc2, 0x51, 0x1c, 0x38, 0x8e, 0x14, 0xad, 0xc2, 0x90, 0x4b, 0x48, 0x8d, 0xe2, 0x2f, + 0x74, 0x8f, 0x3f, 0x10, 0x22, 0xaa, 0xbc, 0x2d, 0xf6, 0x91, 0xa0, 0xef, 0x80, 0xf1, 0x5a, 0xf0, + 0x45, 0x51, 0xac, 0x7d, 0xdd, 0x63, 0x65, 0xb7, 0xed, 0xb2, 0xdc, 0x1a, 0x47, 0x91, 0xa9, 0x7f, + 0xaf, 0xc0, 0xc5, 0x4e, 0x7b, 0x0b, 0xbd, 0x0c, 0xa0, 0xfb, 0x52, 0xa1, 0x2b, 0x0e, 0xcf, 0xc7, + 0x73, 0xae, 0x25, 0xc7, 0x12, 0x7e, 0xa0, 0x41, 0x91, 0x8b, 0x25, 0x22, 0x29, 0x86, 0x9b, 0x85, + 0x63, 0x32, 0xdc, 0x54, 0xff, 0x93, 0x22, 0xb3, 0x22, 0x79, 0x6d, 0xdf, 0x68, 0xac, 0x48, 0xee, + 0x7b, 0xe6, 0x93, 0xeb, 0x57, 0x0a, 0x70, 0x25, 0xbd, 0x89, 0x74, 0x9e, 0x3f, 0x05, 0x83, 0x4d, + 0xee, 0xd2, 0xc2, 0xad, 0xfb, 0xaf, 0x52, 0xce, 0xc2, 0x1d, 0x4e, 0xee, 0xb5, 0x8b, 0xb3, 0x69, + 0x8c, 0x5e, 0xb8, 0xaa, 0x88, 0x76, 0xc8, 0x88, 0xbd, 0x0d, 0x71, 0xa1, 0xfd, 0x9d, 0x5d, 0x32, + 0x17, 0x6d, 0x93, 0x98, 0x5d, 0x3f, 0x07, 0x7d, 0x54, 0x81, 0x89, 0xc8, 0x8e, 0x76, 0x67, 0x06, + 0xd8, 0x1e, 0xcd, 0x65, 0x33, 0x17, 0xf9, 0x54, 0x42, 0x69, 0x20, 0x52, 0xec, 0xe2, 0x18, 0xc1, + 0x18, 0x9b, 0x95, 0x67, 0xf5, 0x0d, 0xc7, 0x66, 0xe5, 0xce, 0x67, 0xb0, 0xd9, 0x9f, 0x28, 0x64, + 0x8d, 0x96, 0xb1, 0xd9, 0x3b, 0x30, 0xe2, 0x8b, 0x91, 0x3e, 0xbb, 0x58, 0xea, 0xb5, 0x4f, 0x1c, + 0xdd, 0xc2, 0x94, 0xe8, 0xcf, 0x88, 0x5f, 0xe2, 0xe2, 0x90, 0x16, 0xfa, 0x5e, 0x05, 0x20, 0x5c, + 0x18, 0xf1, 0x51, 0xad, 0x1f, 0xdd, 0x74, 0x48, 0x62, 0xcd, 0x04, 0xfd, 0xa4, 0xa5, 0x4d, 0x21, + 0xd1, 0x55, 0xff, 0x75, 0x3f, 0xa0, 0x64, 0xdf, 0xbb, 0x7b, 0xf9, 0x3f, 0x40, 0xc8, 0x6d, 0xc2, + 0xa4, 0x43, 0xa5, 0x45, 0xdd, 0x30, 0x89, 0xaf, 0x00, 0xcb, 0xa7, 0xf9, 0x60, 0xd7, 0x2a, 0x1c, + 0xc3, 0x85, 0x13, 0xd8, 0xd1, 0x5b, 0x61, 0xa8, 0xe9, 0x18, 0x0d, 0xcd, 0xd9, 0x63, 0x97, 0xe2, + 0x61, 0xfe, 0x2c, 0xb9, 0xc6, 0x8b, 0xb0, 0x0f, 0x43, 0x1f, 0x84, 0x11, 0xd3, 0xd8, 0x22, 0xfa, + 0x9e, 0x6e, 0x12, 0xf1, 0xea, 0x71, 0xeb, 0x68, 0xd6, 0x7c, 0xd9, 0x47, 0x2b, 0x8c, 0x49, 0xfd, + 0x9f, 0x38, 0x24, 0x88, 0x2a, 0x70, 0xfa, 0x0e, 0x33, 0x6f, 0x32, 0x89, 0xeb, 0x56, 0x5b, 0xcd, + 0xa6, 0xed, 0x78, 0xa4, 0xc6, 0xde, 0x46, 0x86, 0xb9, 0x8e, 0xf6, 0xd9, 0x24, 0x18, 0xa7, 0xb5, + 0x41, 0x8f, 0x03, 0x68, 0x2d, 0xcf, 0xe6, 0xbe, 0xa5, 0x33, 0xc3, 0xec, 0xd6, 0x7e, 0x89, 0x2e, + 0xf6, 0x7c, 0x50, 0x7a, 0xaf, 0x5d, 0x1c, 0x15, 0xba, 0x3f, 0xb6, 0x34, 0x52, 0x03, 0xf4, 0x02, + 0x4c, 0xeb, 0x1c, 0x54, 0xb2, 0x1b, 0x4d, 0xcd, 0x33, 0xc4, 0x8d, 0x78, 0x84, 0x21, 0xfa, 0x96, + 0xfd, 0x76, 0x71, 0xba, 0x94, 0x02, 0x8f, 0xa3, 0x4c, 0x45, 0xa2, 0xbe, 0x5a, 0x80, 0x0b, 0x1d, + 0x26, 0x08, 0x61, 0xfa, 0xe1, 0x89, 0xf5, 0x13, 0xdb, 0xec, 0x61, 0xfe, 0xb1, 0x88, 0xc2, 0x7b, + 0xed, 0xe2, 0x7d, 0x1d, 0x10, 0x54, 0xe9, 0x3e, 0x27, 0xf5, 0x3d, 0x1c, 0xa2, 0x41, 0x15, 0x18, + 0xac, 0x85, 0xcf, 0x98, 0x23, 0x0b, 0x0f, 0xd2, 0xa3, 0x80, 0x3f, 0x38, 0x74, 0x8b, 0x4d, 0x20, + 0x40, 0xcb, 0x30, 0xc4, 0xcd, 0x63, 0x89, 0x38, 0x56, 0x1e, 0x62, 0x2a, 0x13, 0x5e, 0xd4, 0x2d, + 0x32, 0x1f, 0x85, 0xfa, 0x77, 0x0a, 0x0c, 0x95, 0x6c, 0x87, 0x94, 0x57, 0xab, 0x68, 0x0f, 0x46, + 0xa5, 0xe0, 0x06, 0x82, 0xc5, 0xe6, 0xe4, 0x39, 0x0c, 0xa3, 0x74, 0xb5, 0xf7, 0xdd, 0x31, 0x83, + 0x02, 0x2c, 0xd3, 0x42, 0x2f, 0xd3, 0x39, 0xbf, 0xe3, 0x18, 0x1e, 0x25, 0xdc, 0x8b, 0xbd, 0x14, + 0x27, 0x8c, 0x7d, 0x5c, 0x7c, 0xb7, 0x07, 0x3f, 0x71, 0x48, 0x45, 0x5d, 0xa3, 0xec, 0x25, 0xde, + 0x4d, 0xf4, 0x18, 0xf4, 0x37, 0xec, 0x9a, 0xbf, 0xee, 0x6f, 0xf3, 0x99, 0xc7, 0x8a, 0x5d, 0xa3, + 0x73, 0x7b, 0x36, 0xd9, 0x82, 0x3d, 0x0d, 0xb2, 0x36, 0xea, 0x2a, 0x4c, 0xc6, 0xe9, 0xa3, 0xc7, + 0x60, 0x42, 0xb7, 0x1b, 0x0d, 0xdb, 0xaa, 0xb6, 0xb6, 0xb6, 0x8c, 0x5d, 0x12, 0xf1, 0x93, 0x2d, + 0x45, 0x20, 0x38, 0x56, 0x53, 0xfd, 0x9c, 0x02, 0x7d, 0x74, 0x5d, 0x54, 0x18, 0xac, 0xd9, 0x0d, + 0xcd, 0xb0, 0x44, 0xaf, 0x98, 0xc1, 0x41, 0x99, 0x95, 0x60, 0x01, 0x41, 0x4d, 0x18, 0xf1, 0x25, + 0xb2, 0x9e, 0x2c, 0xfc, 0xcb, 0xab, 0xd5, 0xc0, 0x41, 0x2a, 0x38, 0x26, 0xfc, 0x12, 0x17, 0x87, + 0x44, 0x54, 0x0d, 0xa6, 0xca, 0xab, 0xd5, 0x8a, 0xa5, 0x9b, 0xad, 0x1a, 0x59, 0xdc, 0x65, 0x7f, + 0x28, 0x9f, 0x33, 0x78, 0x89, 0x18, 0x27, 0xe3, 0x73, 0xa2, 0x12, 0xf6, 0x61, 0xb4, 0x1a, 0xe1, + 0x2d, 0x84, 0xdb, 0x28, 0xab, 0x26, 0x90, 0x60, 0x1f, 0xa6, 0x7e, 0xb5, 0x00, 0xa3, 0x52, 0x87, + 0x90, 0x09, 0x43, 0x7c, 0xb8, 0x6e, 0x2f, 0xa1, 0x01, 0x12, 0xbd, 0xe6, 0xd4, 0xf9, 0x84, 0xba, + 0xd8, 0x27, 0x21, 0xf3, 0xec, 0x42, 0x07, 0x9e, 0x3d, 0x17, 0xf1, 0xbe, 0xe5, 0x9f, 0xe4, 0x44, + 0xb6, 0xe7, 0x2d, 0xba, 0x28, 0x8e, 0x27, 0x6e, 0x62, 0x3f, 0x1c, 0x3b, 0x9a, 0xb6, 0x60, 0xe0, + 0xae, 0x6d, 0x11, 0x57, 0xe8, 0xc2, 0x8f, 0x68, 0x80, 0xcc, 0x30, 0xf2, 0x79, 0x8a, 0x17, 0x73, + 0xf4, 0xea, 0x4f, 0x2b, 0x00, 0x65, 0xcd, 0xd3, 0xb8, 0x15, 0x4e, 0x17, 0x86, 0x9e, 0x17, 0x23, + 0xa7, 0xea, 0x70, 0xc2, 0xc9, 0xaf, 0xdf, 0x35, 0xee, 0xfa, 0xc3, 0x0f, 0xa4, 0x75, 0x8e, 0xbd, + 0x6a, 0xdc, 0x25, 0x98, 0xc1, 0xd1, 0x03, 0x30, 0x42, 0x2c, 0xdd, 0xd9, 0x6b, 0xd2, 0x83, 0x85, + 0xab, 0x78, 0xd8, 0x17, 0xba, 0xe8, 0x17, 0xe2, 0x10, 0xae, 0x3e, 0x08, 0xd1, 0x2b, 0x57, 0x17, + 0x76, 0xe8, 0xff, 0xa0, 0xc0, 0xb9, 0x72, 0x4b, 0x33, 0xe7, 0x9b, 0x74, 0xa3, 0x6a, 0xe6, 0x92, + 0xcd, 0xcd, 0x39, 0xe8, 0x3d, 0xe4, 0xed, 0x30, 0xec, 0x0b, 0x39, 0x71, 0x2f, 0x60, 0x9f, 0x51, + 0xe2, 0xa0, 0x06, 0xd2, 0x60, 0xd8, 0xf5, 0xc5, 0xee, 0x42, 0x0f, 0x62, 0xb7, 0x4f, 0x22, 0x10, + 0xbb, 0x03, 0xb4, 0x5c, 0xf9, 0xc5, 0x16, 0xa8, 0x4a, 0x9c, 0x1d, 0x43, 0x27, 0xf3, 0xba, 0x6e, + 0xb7, 0x2c, 0xcf, 0x15, 0x5e, 0xfd, 0x42, 0xf9, 0x95, 0x56, 0x03, 0x67, 0xb4, 0x54, 0xbf, 0xa4, + 0x40, 0xff, 0xe2, 0x7a, 0xa9, 0x8c, 0xbe, 0x03, 0xfa, 0x03, 0x96, 0x91, 0xd3, 0x6a, 0x8b, 0xe2, + 0xe1, 0x7a, 0x3a, 0xbe, 0xde, 0x2b, 0x94, 0xe1, 0x30, 0xac, 0x68, 0x13, 0x06, 0xc9, 0x0e, 0xa1, + 0x5d, 0x2d, 0x1c, 0x09, 0x7e, 0xc6, 0xd2, 0x16, 0x19, 0x46, 0x2c, 0x30, 0xab, 0x9f, 0x52, 0x00, + 0xc2, 0x2a, 0xe8, 0xbb, 0xd3, 0x4e, 0xa7, 0x9b, 0x47, 0xa8, 0x7d, 0xee, 0x7c, 0x44, 0xa9, 0xbf, + 0x31, 0x00, 0xe7, 0x69, 0x77, 0xc4, 0x56, 0x35, 0x6c, 0xeb, 0x26, 0xd9, 0xfb, 0xa6, 0xc7, 0xc3, + 0x37, 0x3d, 0x1e, 0x8e, 0xd0, 0xe3, 0xe1, 0x05, 0x38, 0x4f, 0xb7, 0x9c, 0x00, 0x13, 0x66, 0xfe, + 0xb5, 0xe6, 0x90, 0xa6, 0xe6, 0x90, 0x1a, 0xbb, 0x0f, 0x0c, 0x73, 0x83, 0xb6, 0xf9, 0xac, 0x4a, + 0x38, 0xbb, 0xbd, 0xfa, 0x24, 0x4c, 0x86, 0x7b, 0x57, 0x7c, 0x54, 0x0f, 0xc4, 0x2f, 0x99, 0x23, + 0xbe, 0xc4, 0x94, 0xbc, 0x18, 0xaa, 0xf7, 0x14, 0x98, 0x5c, 0xdc, 0x6d, 0x1a, 0x0e, 0x0b, 0xb5, + 0xc0, 0x3d, 0x86, 0xd0, 0xfd, 0xa1, 0x63, 0x91, 0x12, 0x7d, 0xc5, 0x8b, 0x3b, 0x17, 0xa1, 0x2d, + 0x98, 0x20, 0xac, 0x39, 0xbb, 0x05, 0x6a, 0x5e, 0x9e, 0xed, 0xcd, 0xe3, 0x8b, 0x44, 0xb0, 0xe0, + 0x18, 0x56, 0x54, 0x85, 0x09, 0xdd, 0xd4, 0x5c, 0xd7, 0xd8, 0x32, 0xf4, 0xd0, 0x21, 0x6e, 0x64, + 0xe1, 0x01, 0x26, 0x73, 0x45, 0x20, 0xf7, 0xda, 0xc5, 0x33, 0xa2, 0x9f, 0x51, 0x00, 0x8e, 0xa1, + 0x50, 0x3f, 0x53, 0x80, 0xf1, 0xc5, 0xdd, 0xa6, 0xed, 0xb6, 0x1c, 0xc2, 0xaa, 0x9e, 0x80, 0x5e, + 0xeb, 0x7e, 0x18, 0xda, 0xd6, 0xac, 0x9a, 0x49, 0x1c, 0x71, 0xec, 0x06, 0x73, 0x7b, 0x83, 0x17, + 0x63, 0x1f, 0x8e, 0x5e, 0x01, 0x70, 0xf5, 0x6d, 0x52, 0x6b, 0x31, 0xe6, 0xd8, 0x97, 0x9f, 0x39, + 0x46, 0xc6, 0x58, 0x0d, 0x50, 0x0a, 0x91, 0x26, 0xf8, 0x8d, 0x25, 0x72, 0xea, 0x9f, 0x2a, 0x30, + 0x15, 0x69, 0x77, 0x02, 0xea, 0x9a, 0xad, 0xa8, 0xba, 0x66, 0xbe, 0xe7, 0xb1, 0x66, 0x68, 0x69, + 0x3e, 0x51, 0x80, 0x73, 0x19, 0x73, 0x92, 0x30, 0xdd, 0x56, 0x4e, 0xc8, 0x74, 0xbb, 0x05, 0xa3, + 0x9e, 0x6d, 0x0a, 0xbf, 0x4d, 0x7f, 0x06, 0x72, 0x1d, 0xc1, 0xeb, 0x01, 0x9a, 0xd0, 0x30, 0x3b, + 0x2c, 0x73, 0xb1, 0x4c, 0x47, 0xfd, 0x2d, 0x05, 0x46, 0x02, 0xad, 0xf0, 0xeb, 0xea, 0x41, 0xbd, + 0xfb, 0x90, 0x48, 0xea, 0xef, 0x17, 0xe0, 0x6c, 0x80, 0xdb, 0x67, 0x73, 0x55, 0x8f, 0xf2, 0x8d, + 0x83, 0x55, 0x4b, 0x17, 0x23, 0x4e, 0x25, 0xc3, 0x49, 0x1f, 0xcb, 0x66, 0xcb, 0x69, 0xda, 0xae, + 0x2f, 0x07, 0xf3, 0x0b, 0x03, 0x2f, 0xc2, 0x3e, 0x0c, 0xad, 0xc2, 0x80, 0x4b, 0xe9, 0x89, 0xb3, + 0xee, 0x90, 0xb3, 0xc1, 0x44, 0x79, 0xd6, 0x5f, 0xcc, 0xd1, 0xa0, 0x57, 0x64, 0x1e, 0x3e, 0x90, + 0x5f, 0x79, 0x49, 0x47, 0x52, 0x0b, 0x24, 0xe1, 0xf0, 0xe9, 0xa3, 0x93, 0xb2, 0x50, 0x5d, 0x86, + 0x49, 0x61, 0x9f, 0xcc, 0xb7, 0x8d, 0xa5, 0x13, 0xf4, 0x9e, 0xc8, 0xce, 0x78, 0x4b, 0xcc, 0xa4, + 0x66, 0x3a, 0x5e, 0x3f, 0xdc, 0x31, 0xaa, 0x0b, 0xc3, 0xd7, 0x45, 0x27, 0xd1, 0x2c, 0x14, 0x0c, + 0x7f, 0x2d, 0x40, 0xe0, 0x28, 0x54, 0xca, 0xb8, 0x60, 0x74, 0xe1, 0xdc, 0x23, 0x1f, 0x4b, 0x7d, + 0x9d, 0x8f, 0x25, 0xf5, 0xeb, 0x05, 0x98, 0xf6, 0xa9, 0xfa, 0x63, 0x2c, 0x8b, 0xd7, 0xf2, 0x03, + 0x2e, 0x45, 0x07, 0xab, 0x1a, 0x6f, 0x41, 0x3f, 0x63, 0x80, 0xb9, 0x5e, 0xd1, 0x03, 0x84, 0xb4, + 0x3b, 0x98, 0x21, 0x42, 0x1f, 0x84, 0x41, 0x93, 0xde, 0x30, 0x7c, 0xaf, 0x9b, 0x5c, 0x8a, 0xd9, + 0xb4, 0xe1, 0xf2, 0x8b, 0x8b, 0x88, 0x46, 0x17, 0x3c, 0x84, 0xf2, 0x42, 0x2c, 0x68, 0xce, 0x3e, + 0x0a, 0xa3, 0x52, 0xb5, 0x43, 0x85, 0xa2, 0xfb, 0x5c, 0x01, 0x66, 0x6e, 0x10, 0xb3, 0x91, 0x6a, + 0xfa, 0x50, 0x84, 0x01, 0x7d, 0x5b, 0x73, 0x78, 0x94, 0xc3, 0x31, 0xbe, 0xc9, 0x4b, 0xb4, 0x00, + 0xf3, 0x72, 0x7a, 0xe1, 0x88, 0xf8, 0x19, 0x3e, 0x21, 0xcd, 0x64, 0x18, 0xfe, 0xf2, 0x03, 0x41, + 0x7c, 0xcc, 0x70, 0xe0, 0x91, 0x0a, 0xf4, 0x78, 0x79, 0x5f, 0xf5, 0xd6, 0x6a, 0x9a, 0x87, 0x21, + 0xba, 0x0b, 0xe3, 0xb6, 0x6e, 0x60, 0xd2, 0xb4, 0x5d, 0xc3, 0xb3, 0x9d, 0x3d, 0xb1, 0x68, 0xb9, + 0x8e, 0x96, 0x5b, 0xa5, 0x4a, 0x88, 0x88, 0x3f, 0x1f, 0x46, 0x8a, 0x70, 0x94, 0x94, 0xfa, 0x45, + 0x05, 0x46, 0x6f, 0x18, 0x9b, 0xc4, 0xe1, 0x26, 0xd8, 0x4c, 0x43, 0x12, 0x89, 0xd7, 0x37, 0x9a, + 0x16, 0xab, 0x0f, 0xed, 0xc2, 0x88, 0x38, 0x87, 0x03, 0x07, 0xd2, 0xeb, 0xf9, 0xec, 0x85, 0x02, + 0xd2, 0xe2, 0x7c, 0x93, 0x3e, 0x7c, 0xbf, 0xc4, 0xc5, 0x21, 0x31, 0xf5, 0x15, 0x38, 0x9d, 0xd2, + 0x88, 0x2e, 0x24, 0xb3, 0x42, 0x16, 0x1f, 0x8d, 0xcf, 0xad, 0xe8, 0x42, 0xb2, 0x72, 0x74, 0x1e, + 0xfa, 0x88, 0x55, 0x13, 0x5f, 0xcc, 0xd0, 0x7e, 0xbb, 0xd8, 0xb7, 0x68, 0xd5, 0x30, 0x2d, 0xa3, + 0x4c, 0xdc, 0xb4, 0x23, 0x12, 0x1b, 0x63, 0xe2, 0xcb, 0xa2, 0x0c, 0x07, 0x50, 0x66, 0xe1, 0x15, + 0x37, 0x66, 0xa2, 0x37, 0x8b, 0xc9, 0xad, 0x18, 0x6f, 0xe9, 0xc5, 0x86, 0x2a, 0xce, 0xa7, 0x16, + 0x66, 0xc4, 0x84, 0x24, 0x38, 0x1e, 0x4e, 0xd0, 0x55, 0x7f, 0xb5, 0x1f, 0x2e, 0xdd, 0xb0, 0x1d, + 0xe3, 0xae, 0x6d, 0x79, 0x9a, 0xb9, 0x66, 0xd7, 0x42, 0xfb, 0x56, 0x71, 0x64, 0x7d, 0x5c, 0x81, + 0x73, 0x7a, 0xb3, 0x25, 0xcc, 0x8d, 0x85, 0x89, 0xe8, 0x1a, 0x71, 0x0c, 0x3b, 0xaf, 0xcf, 0x0d, + 0xb3, 0xcc, 0x2e, 0xad, 0x6d, 0xa4, 0xa1, 0xc4, 0x59, 0xb4, 0x98, 0xeb, 0x4f, 0xcd, 0xbe, 0x63, + 0xb1, 0xce, 0x55, 0x3d, 0x36, 0x9b, 0x77, 0xc3, 0x45, 0xc8, 0xe9, 0xfa, 0x53, 0x4e, 0xc5, 0x88, + 0x33, 0x28, 0xa1, 0x0f, 0xc3, 0x19, 0x61, 0x56, 0x8d, 0x89, 0x56, 0x33, 0x2c, 0xe2, 0xba, 0xdc, + 0x6f, 0xa0, 0x07, 0xdf, 0x96, 0x4a, 0x1a, 0x42, 0x9c, 0x4e, 0x07, 0xbd, 0x08, 0xe0, 0xee, 0x59, + 0xba, 0x98, 0xff, 0x7c, 0x86, 0xa8, 0x5c, 0x44, 0x0e, 0xb0, 0x60, 0x09, 0x23, 0xbd, 0x68, 0x79, + 0xc1, 0xa6, 0x1c, 0x64, 0xc6, 0xc4, 0xec, 0xa2, 0x15, 0xee, 0xa1, 0x10, 0xae, 0x7e, 0x4a, 0x81, + 0x89, 0x8a, 0xb5, 0x66, 0x6a, 0x3a, 0xe1, 0x76, 0x95, 0x2e, 0xba, 0x06, 0x23, 0x6e, 0xf0, 0x22, + 0xc3, 0x39, 0x42, 0xf8, 0x7d, 0x06, 0x6f, 0x31, 0x61, 0x1d, 0x66, 0x70, 0x6f, 0x58, 0xe2, 0xb0, + 0x5b, 0xb2, 0x1d, 0x8e, 0x48, 0x7c, 0x77, 0xdc, 0xe0, 0x3e, 0x09, 0xc6, 0x69, 0x6d, 0xd4, 0x5f, + 0x50, 0x60, 0x3a, 0xda, 0x1d, 0x61, 0xd2, 0xf0, 0xe3, 0x0a, 0x4c, 0x47, 0xfc, 0xe2, 0x05, 0xb8, + 0x97, 0x50, 0x5d, 0x6b, 0x29, 0xf8, 0xb8, 0x89, 0x6f, 0x1a, 0x04, 0xa7, 0xd2, 0x57, 0xff, 0xa9, + 0x02, 0x43, 0x22, 0x6c, 0x27, 0x7a, 0x5b, 0x4c, 0x7b, 0x1e, 0x1c, 0x6d, 0x31, 0x0d, 0xfa, 0x1e, + 0xb3, 0xcf, 0x10, 0x47, 0x93, 0x38, 0x65, 0x72, 0xa9, 0x5f, 0x05, 0xe1, 0xf0, 0x9c, 0x8b, 0xd8, + 0x69, 0xf8, 0x4f, 0x33, 0x12, 0x31, 0xf5, 0x35, 0x05, 0xa6, 0x12, 0xad, 0xba, 0x10, 0x47, 0x4f, + 0xd0, 0x62, 0xf5, 0x2b, 0xfd, 0x74, 0x4b, 0x7a, 0x94, 0xdf, 0x9b, 0x5c, 0xb1, 0x7d, 0x02, 0xf7, + 0xdf, 0x07, 0x60, 0xc4, 0x68, 0x34, 0x5a, 0x1e, 0x7b, 0x44, 0x1c, 0x08, 0xb5, 0xc5, 0x15, 0xbf, + 0x10, 0x87, 0x70, 0x64, 0x09, 0x49, 0x8b, 0x9f, 0x82, 0xcb, 0xf9, 0x56, 0x4e, 0x1e, 0xe0, 0x1c, + 0x95, 0x8a, 0xb8, 0x38, 0x94, 0x26, 0x88, 0x7d, 0x9f, 0x02, 0xe0, 0x7a, 0x8e, 0x61, 0xd5, 0x69, + 0xa1, 0x90, 0xc6, 0xf0, 0x11, 0x90, 0xad, 0x06, 0x48, 0x39, 0xf1, 0x30, 0x94, 0x67, 0x00, 0xc0, + 0x12, 0x65, 0x34, 0x2f, 0x84, 0x50, 0x7e, 0x64, 0xbe, 0x23, 0x26, 0x6e, 0x5f, 0x4a, 0xb1, 0x84, + 0xe5, 0x84, 0x42, 0x29, 0x75, 0xf6, 0x11, 0x18, 0x09, 0xe8, 0x1d, 0x24, 0xd4, 0x8d, 0x49, 0x42, + 0xdd, 0xec, 0xe3, 0x70, 0x2a, 0xd6, 0xdd, 0x43, 0xc9, 0x84, 0x7f, 0xa6, 0x00, 0x8a, 0x8e, 0xfe, + 0x04, 0x34, 0x07, 0xf5, 0xa8, 0xe6, 0x60, 0xa1, 0xf7, 0x25, 0xcb, 0x50, 0x1d, 0xfc, 0x1c, 0x02, + 0x16, 0xd5, 0x38, 0x88, 0xf2, 0x2d, 0x4e, 0x7e, 0x2a, 0xa8, 0x84, 0xce, 0xfc, 0xe2, 0xcb, 0xed, + 0x41, 0x50, 0xb9, 0x19, 0xc3, 0x15, 0x0a, 0x2a, 0x71, 0x08, 0x4e, 0xd0, 0x45, 0x9f, 0x54, 0x60, + 0x52, 0x8b, 0x46, 0x35, 0xf6, 0x67, 0x26, 0x57, 0xac, 0xb5, 0x58, 0x84, 0xe4, 0xb0, 0x2f, 0x31, + 0x80, 0x8b, 0x13, 0x64, 0xd1, 0xc3, 0x30, 0xa6, 0x35, 0x8d, 0xf9, 0x56, 0xcd, 0xa0, 0x37, 0x4f, + 0x3f, 0xf8, 0x2b, 0xd3, 0x86, 0xcc, 0xaf, 0x55, 0x82, 0x72, 0x1c, 0xa9, 0x15, 0x84, 0x0f, 0x2e, + 0x85, 0x86, 0xc3, 0xbd, 0x84, 0x0f, 0x16, 0x73, 0x18, 0x86, 0x0f, 0x16, 0x53, 0x27, 0x13, 0x41, + 0x16, 0x80, 0x6d, 0xd4, 0x74, 0x41, 0x72, 0x30, 0xff, 0x1b, 0xc8, 0xad, 0x4a, 0xb9, 0x24, 0x28, + 0x32, 0xf1, 0x21, 0xfc, 0x8d, 0x25, 0x0a, 0xe8, 0xd3, 0x0a, 0x8c, 0x0b, 0xde, 0x2d, 0x68, 0x0e, + 0xb1, 0x25, 0x7a, 0x3e, 0xef, 0x7e, 0x89, 0xed, 0xc9, 0x39, 0x2c, 0x23, 0xe7, 0x7c, 0x27, 0x88, + 0x05, 0x11, 0x81, 0xe1, 0x68, 0x3f, 0x98, 0x0c, 0xe0, 0x46, 0x1e, 0xa1, 0x44, 0x07, 0x87, 0xf3, + 0xcb, 0x00, 0xd5, 0x14, 0x7c, 0xc2, 0xc9, 0x28, 0x05, 0x82, 0x53, 0xe9, 0x53, 0xb9, 0xf6, 0xd4, + 0x1d, 0xcd, 0xd3, 0xb7, 0x4b, 0x9a, 0xbe, 0xcd, 0xde, 0x20, 0xb9, 0x67, 0x6c, 0xce, 0x7d, 0xfd, + 0x6c, 0x14, 0xd5, 0xc2, 0xe9, 0xfd, 0x76, 0xf1, 0x54, 0xac, 0x10, 0xc7, 0x09, 0x22, 0x1b, 0x86, + 0x1d, 0x11, 0xda, 0x5f, 0x04, 0x33, 0xc8, 0x17, 0xcd, 0x3e, 0x9e, 0x27, 0x80, 0xdf, 0x8c, 0xfc, + 0x5f, 0x38, 0x20, 0x82, 0xea, 0x70, 0x89, 0xdf, 0x0d, 0xe7, 0x2d, 0xdb, 0xda, 0x6b, 0xd8, 0x2d, + 0x77, 0xbe, 0xe5, 0x6d, 0x13, 0xcb, 0xf3, 0x55, 0xe1, 0xa3, 0xec, 0x18, 0x65, 0x4e, 0x73, 0x8b, + 0x9d, 0x2a, 0xe2, 0xce, 0x78, 0xd0, 0x73, 0x30, 0xcc, 0xde, 0xe9, 0xd6, 0xd7, 0x97, 0x99, 0x93, + 0xed, 0xe1, 0xc5, 0x65, 0x36, 0x84, 0x45, 0x81, 0x03, 0x07, 0xd8, 0xd0, 0x6d, 0x18, 0x32, 0x79, + 0x6e, 0x06, 0xe6, 0x6c, 0x9b, 0x93, 0x29, 0xc6, 0xf3, 0x3c, 0xf0, 0x0b, 0xb4, 0xf8, 0x81, 0x7d, + 0x0a, 0xa8, 0x09, 0x57, 0x6a, 0x64, 0x4b, 0x6b, 0x99, 0xde, 0xaa, 0xed, 0x61, 0xe6, 0xa1, 0x16, + 0x68, 0x3c, 0x7d, 0x7f, 0xea, 0x09, 0x16, 0x4e, 0x8f, 0xf9, 0xfe, 0x95, 0x0f, 0xa8, 0x8b, 0x0f, + 0xc4, 0x86, 0xf6, 0xe0, 0x3e, 0x51, 0x87, 0xb9, 0xc4, 0xe9, 0xdb, 0x74, 0x96, 0x93, 0x44, 0x4f, + 0x31, 0xa2, 0xdf, 0xb2, 0xdf, 0x2e, 0xde, 0x57, 0x3e, 0xb8, 0x3a, 0xee, 0x06, 0x27, 0xf3, 0x32, + 0x22, 0xb1, 0x27, 0xa0, 0x99, 0xc9, 0xfc, 0x73, 0x1c, 0x7f, 0x4e, 0xe2, 0xe6, 0x70, 0xf1, 0x52, + 0x9c, 0xa0, 0x89, 0xbe, 0xa0, 0xc0, 0x8c, 0xeb, 0x39, 0x2d, 0xdd, 0x6b, 0x39, 0xa4, 0x16, 0xdb, + 0xa1, 0x53, 0xac, 0x43, 0xb9, 0x04, 0xb8, 0x6a, 0x06, 0x4e, 0xe6, 0xd9, 0x3f, 0x93, 0x05, 0xc5, + 0x99, 0x7d, 0x41, 0x9f, 0x57, 0xe0, 0x5c, 0x14, 0x48, 0xef, 0xf4, 0xbc, 0x9f, 0x28, 0xff, 0x23, + 0x4b, 0x35, 0x1d, 0x25, 0xbf, 0xc1, 0x67, 0x00, 0x71, 0x56, 0x47, 0xe2, 0x2f, 0xe3, 0xa7, 0x4f, + 0xf8, 0x65, 0x7c, 0xf6, 0x29, 0x40, 0xc9, 0xe3, 0xe3, 0x20, 0x39, 0x70, 0x58, 0x96, 0x03, 0x3f, + 0x3b, 0x00, 0x17, 0xe8, 0xa9, 0x14, 0xde, 0x7e, 0x56, 0x34, 0x4b, 0xab, 0xbf, 0x3e, 0x25, 0xa6, + 0x2f, 0x2a, 0x70, 0x6e, 0x3b, 0x5d, 0xb5, 0x23, 0xee, 0x5f, 0x4f, 0xe7, 0x52, 0xc1, 0x75, 0xd2, + 0x16, 0x71, 0x86, 0xdd, 0xb1, 0x0a, 0xce, 0xea, 0x14, 0x7a, 0x0a, 0x26, 0x2d, 0xbb, 0x46, 0x4a, + 0x95, 0x32, 0x5e, 0xd1, 0xdc, 0xdb, 0x55, 0xdf, 0x50, 0x67, 0x80, 0x7f, 0xaf, 0xab, 0x31, 0x18, + 0x4e, 0xd4, 0x46, 0x3b, 0x80, 0x9a, 0x76, 0x6d, 0x71, 0x87, 0xe7, 0x30, 0xe9, 0xcd, 0x64, 0x96, + 0x19, 0x0b, 0xac, 0x25, 0xb0, 0xe1, 0x14, 0x0a, 0x4c, 0x37, 0x45, 0x3b, 0xb3, 0x62, 0x5b, 0x86, + 0x67, 0x3b, 0x2c, 0x56, 0x45, 0x4f, 0x2a, 0x1a, 0xa6, 0x9b, 0x5a, 0x4d, 0xc5, 0x88, 0x33, 0x28, + 0xa9, 0xff, 0x4d, 0x81, 0x53, 0x74, 0x5b, 0xac, 0x39, 0xf6, 0xee, 0xde, 0xeb, 0x71, 0x43, 0xde, + 0x2f, 0x6c, 0x16, 0xb9, 0x6e, 0xe7, 0x8c, 0x64, 0xaf, 0x38, 0xc2, 0xfa, 0x1c, 0x9a, 0x28, 0xca, + 0x6a, 0xe5, 0xbe, 0x6c, 0xb5, 0xb2, 0xfa, 0xe9, 0x02, 0xbf, 0xb9, 0xf8, 0x6a, 0xdd, 0xd7, 0xe5, + 0x77, 0xf8, 0x08, 0x8c, 0xd3, 0xb2, 0x15, 0x6d, 0x77, 0xad, 0xfc, 0x8c, 0x6d, 0xfa, 0xde, 0xd8, + 0x4c, 0xd7, 0x7e, 0x53, 0x06, 0xe0, 0x68, 0x3d, 0xf4, 0x18, 0x0c, 0x35, 0x79, 0xc8, 0x37, 0x71, + 0x67, 0xbe, 0xc2, 0x0d, 0xfb, 0x58, 0xd1, 0xbd, 0x76, 0x71, 0x2a, 0x7c, 0xe2, 0xf5, 0x03, 0xcf, + 0xf9, 0x0d, 0xd4, 0x5f, 0x3d, 0x0b, 0x0c, 0xb9, 0x49, 0xbc, 0xd7, 0xe3, 0x9c, 0x3c, 0x08, 0xa3, + 0x7a, 0xb3, 0x55, 0x5a, 0xaa, 0x3e, 0xdd, 0xb2, 0x99, 0x2e, 0x84, 0xa5, 0xfa, 0xa1, 0xdc, 0xbb, + 0xb4, 0xb6, 0xe1, 0x17, 0x63, 0xb9, 0x0e, 0xe5, 0x0e, 0x7a, 0xb3, 0x25, 0xf8, 0xed, 0x9a, 0xec, + 0xaf, 0xc2, 0xb8, 0x43, 0x69, 0x6d, 0x23, 0x02, 0xc3, 0x89, 0xda, 0xe8, 0xc3, 0x30, 0x46, 0xc4, + 0x87, 0x7b, 0x43, 0x73, 0x6a, 0x82, 0x2f, 0x54, 0xf2, 0x0e, 0x3e, 0x98, 0x5a, 0x9f, 0x1b, 0xf0, + 0x1b, 0xe0, 0xa2, 0x44, 0x02, 0x47, 0x08, 0xa2, 0x17, 0xe0, 0xbc, 0xff, 0x9b, 0xae, 0xb2, 0x5d, + 0x8b, 0x33, 0x8a, 0x01, 0x6e, 0x36, 0xb3, 0x98, 0x55, 0x09, 0x67, 0xb7, 0x47, 0x3f, 0xaf, 0xc0, + 0xd9, 0x00, 0x6a, 0x58, 0x46, 0xa3, 0xd5, 0xc0, 0x44, 0x37, 0x35, 0xa3, 0x21, 0xee, 0x7d, 0xcf, + 0x1e, 0xd9, 0x40, 0xa3, 0xe8, 0x39, 0xb3, 0x4a, 0x87, 0xe1, 0x8c, 0x2e, 0xa1, 0xd7, 0x14, 0xb8, + 0xe2, 0x83, 0xd6, 0x1c, 0xe2, 0xba, 0x2d, 0x87, 0x84, 0xb1, 0x00, 0xc4, 0x94, 0x0c, 0xe5, 0xe2, + 0x9d, 0x4c, 0x00, 0x5e, 0x3c, 0x00, 0x37, 0x3e, 0x90, 0xba, 0xbc, 0x5d, 0xaa, 0xf6, 0x96, 0x27, + 0x2e, 0x8a, 0xc7, 0xb5, 0x5d, 0x28, 0x09, 0x1c, 0x21, 0x88, 0x7e, 0x41, 0x81, 0x73, 0x72, 0x81, + 0xbc, 0x5b, 0xf8, 0x0d, 0xf1, 0xb9, 0x23, 0xeb, 0x4c, 0x0c, 0x3f, 0x97, 0xf0, 0x32, 0x80, 0x38, + 0xab, 0x57, 0x94, 0x6d, 0x37, 0xd8, 0xc6, 0xe4, 0xb7, 0xc8, 0x01, 0xce, 0xb6, 0xf9, 0x5e, 0x75, + 0xb1, 0x0f, 0x43, 0x0f, 0xc3, 0x58, 0xd3, 0xae, 0xad, 0x19, 0x35, 0x97, 0x85, 0xa3, 0x63, 0x77, + 0xbd, 0x3e, 0x3e, 0x1d, 0x6b, 0x76, 0x6d, 0xad, 0x52, 0xe6, 0xe5, 0x38, 0x52, 0x0b, 0xcd, 0x01, + 0x6c, 0x69, 0x86, 0x59, 0xbd, 0xa3, 0x35, 0x6f, 0xf9, 0xf1, 0x8d, 0x98, 0x2e, 0x62, 0x29, 0x28, + 0xc5, 0x52, 0x0d, 0xba, 0x7e, 0x94, 0xef, 0x60, 0xc2, 0x23, 0x88, 0xb3, 0xeb, 0xd1, 0x51, 0xac, + 0x9f, 0x8f, 0x90, 0x77, 0xf8, 0xa6, 0x44, 0x02, 0x47, 0x08, 0xa2, 0x8f, 0x2b, 0x30, 0xe1, 0xee, + 0xb9, 0x1e, 0x69, 0x04, 0x7d, 0x38, 0x75, 0xd4, 0x7d, 0x60, 0x3a, 0xf1, 0x6a, 0x84, 0x08, 0x8e, + 0x11, 0x65, 0x91, 0xa2, 0x1a, 0x5a, 0x9d, 0x5c, 0x2f, 0xdd, 0x30, 0xea, 0xdb, 0x41, 0x74, 0x97, + 0x35, 0xe2, 0xe8, 0xc4, 0xf2, 0xd8, 0xc5, 0x6a, 0x40, 0x44, 0x8a, 0xca, 0xae, 0x86, 0x3b, 0xe1, + 0x40, 0x2f, 0xc2, 0xac, 0x00, 0x2f, 0xdb, 0x77, 0x12, 0x14, 0x78, 0x40, 0x26, 0x66, 0x00, 0x59, + 0xc9, 0xac, 0x85, 0x3b, 0x60, 0x40, 0x15, 0x38, 0xed, 0x12, 0x87, 0xbd, 0x09, 0xf2, 0xe0, 0x9e, + 0x6b, 0x2d, 0xd3, 0x74, 0xd9, 0xd5, 0x46, 0xb8, 0xfc, 0x54, 0x93, 0x60, 0x9c, 0xd6, 0x06, 0x3d, + 0x1e, 0xb8, 0x05, 0xef, 0xd1, 0x82, 0xa7, 0xd7, 0xaa, 0xec, 0x26, 0x32, 0xc0, 0x35, 0x25, 0x38, + 0x0a, 0xc2, 0xf1, 0xba, 0xf4, 0x34, 0xf7, 0x8b, 0x16, 0x5a, 0x8e, 0xeb, 0xb1, 0x30, 0x49, 0x03, + 0xfc, 0x34, 0xc7, 0x32, 0x00, 0x47, 0xeb, 0xa1, 0xc7, 0x60, 0xc2, 0x25, 0xba, 0x6e, 0x37, 0x9a, + 0xe2, 0x9e, 0xcc, 0x62, 0x19, 0x0d, 0x8b, 0x15, 0x8c, 0x40, 0x70, 0xac, 0x26, 0xda, 0x83, 0xd3, + 0x41, 0x34, 0xe4, 0x65, 0xbb, 0xbe, 0xa2, 0xed, 0x32, 0xe1, 0xf8, 0x6c, 0x9e, 0x10, 0x0f, 0x7c, + 0xba, 0x4a, 0x49, 0x74, 0x38, 0x8d, 0x06, 0x5a, 0x86, 0xe9, 0x58, 0xf1, 0x92, 0x61, 0x12, 0x77, + 0xe6, 0x1c, 0x1b, 0x36, 0x53, 0x76, 0x95, 0x52, 0xe0, 0x38, 0xb5, 0x15, 0xba, 0x05, 0x67, 0x9a, + 0x8e, 0xed, 0x11, 0xdd, 0xbb, 0x49, 0x05, 0x02, 0x53, 0x0c, 0xd0, 0x9d, 0x99, 0x61, 0x73, 0xc1, + 0xde, 0x43, 0xd7, 0xd2, 0x2a, 0xe0, 0xf4, 0x76, 0xe8, 0xb3, 0x0a, 0x5c, 0x76, 0x3d, 0x87, 0x68, + 0x0d, 0xc3, 0xaa, 0x97, 0x6c, 0xcb, 0x22, 0x8c, 0x31, 0x55, 0x6a, 0xa1, 0xc7, 0xdc, 0xf9, 0x5c, + 0xa7, 0x88, 0xba, 0xdf, 0x2e, 0x5e, 0xae, 0x76, 0xc4, 0x8c, 0x0f, 0xa0, 0x8c, 0x5e, 0x01, 0x68, + 0x90, 0x86, 0xed, 0xec, 0x51, 0x8e, 0x34, 0x33, 0x9b, 0xff, 0xbe, 0xbb, 0x12, 0x60, 0xe1, 0x9f, + 0x7f, 0xe4, 0x25, 0x37, 0x04, 0x62, 0x89, 0x1c, 0x9d, 0x6a, 0xca, 0x6f, 0x45, 0x40, 0x32, 0xe9, + 0xa3, 0xb9, 0xc0, 0x56, 0x8e, 0x4d, 0xf5, 0x4a, 0x5a, 0x05, 0x9c, 0xde, 0x0e, 0xb9, 0x30, 0xc5, + 0xbe, 0x50, 0x71, 0x92, 0x5f, 0x2f, 0xcd, 0xd7, 0xc9, 0xcc, 0xc5, 0x5c, 0x93, 0x4b, 0x65, 0xff, + 0xa9, 0x4a, 0x1c, 0x19, 0x4e, 0xe2, 0x0f, 0x89, 0x6a, 0xbb, 0x21, 0xd1, 0x4b, 0xbd, 0x12, 0x95, + 0x90, 0xe1, 0x24, 0x7e, 0xb5, 0x5d, 0x80, 0x33, 0xa9, 0xa7, 0x24, 0x65, 0x1e, 0x7c, 0x8a, 0xe7, + 0xfd, 0x54, 0x6e, 0xe2, 0xd9, 0x93, 0x31, 0x8f, 0x95, 0x28, 0x08, 0xc7, 0xeb, 0x52, 0x19, 0x96, + 0x51, 0x5b, 0xaa, 0x86, 0xed, 0x0b, 0xa1, 0x0c, 0x5b, 0x89, 0xc1, 0x70, 0xa2, 0x36, 0x2a, 0x89, + 0xf9, 0x58, 0xaa, 0x56, 0xe8, 0x35, 0xd0, 0x5d, 0x72, 0x88, 0x7f, 0x3b, 0x08, 0xc7, 0x27, 0x03, + 0x71, 0xb2, 0x3e, 0x1d, 0x05, 0xfd, 0x21, 0xf7, 0xa2, 0x3f, 0x1c, 0xc5, 0x6a, 0x14, 0x84, 0xe3, + 0x75, 0xfd, 0x7b, 0x7a, 0xa4, 0x0b, 0x03, 0xe1, 0x28, 0x56, 0x63, 0x30, 0x9c, 0xa8, 0xad, 0xfe, + 0x79, 0x3f, 0xdc, 0xd7, 0x85, 0x64, 0x89, 0x1a, 0xe9, 0xd3, 0x7d, 0x78, 0x9e, 0xd7, 0xdd, 0xf2, + 0x34, 0x33, 0x96, 0xe7, 0xf0, 0xf4, 0xba, 0x5d, 0x4e, 0x37, 0x6b, 0x39, 0x0f, 0x4f, 0xb2, 0xfb, + 0xe5, 0x6f, 0xa4, 0x2f, 0x7f, 0xce, 0x59, 0x3d, 0x70, 0xbb, 0x34, 0x33, 0xb6, 0x4b, 0xce, 0x59, + 0xed, 0x62, 0x7b, 0xfd, 0x45, 0x3f, 0xbc, 0xa5, 0x1b, 0x29, 0x37, 0xe7, 0xfe, 0x4a, 0xe1, 0x2d, + 0xc7, 0xba, 0xbf, 0xb2, 0xfc, 0xb9, 0x8f, 0x71, 0x7f, 0x75, 0x64, 0x9f, 0xc7, 0xb3, 0xbf, 0xb2, + 0x66, 0xf5, 0xb8, 0xf6, 0x57, 0xd6, 0xac, 0x76, 0xb1, 0xbf, 0xfe, 0x36, 0x7e, 0x3e, 0x04, 0xa2, + 0x76, 0x05, 0xfa, 0xf4, 0x66, 0x2b, 0x27, 0x93, 0x62, 0x56, 0x86, 0xa5, 0xb5, 0x0d, 0x4c, 0x71, + 0x20, 0x0c, 0x83, 0x7c, 0xff, 0xe4, 0x64, 0x41, 0xcc, 0x72, 0x94, 0x6f, 0x49, 0x2c, 0x30, 0xd1, + 0xa9, 0x22, 0xcd, 0x6d, 0xd2, 0x20, 0x8e, 0x66, 0x56, 0x3d, 0xdb, 0xd1, 0xea, 0x79, 0xb9, 0x0d, + 0x7f, 0x41, 0x89, 0xe1, 0xc2, 0x09, 0xec, 0x74, 0x42, 0x9a, 0x46, 0x2d, 0x27, 0x7f, 0x61, 0x13, + 0xb2, 0x56, 0x29, 0x63, 0x8a, 0x43, 0xfd, 0x8d, 0x61, 0x90, 0xf2, 0x0c, 0xa0, 0x57, 0x15, 0x98, + 0xd2, 0xe3, 0x31, 0x39, 0x7b, 0xb1, 0x87, 0x4a, 0x04, 0xf8, 0xe4, 0x5b, 0x3e, 0x51, 0x8c, 0x93, + 0x64, 0xd1, 0x47, 0x14, 0xae, 0xe4, 0x0b, 0x5e, 0xf3, 0xc4, 0xb4, 0x5e, 0x3f, 0xa2, 0x77, 0xef, + 0x50, 0x5b, 0x18, 0x3e, 0xb1, 0x46, 0x09, 0xa2, 0xd7, 0x14, 0x38, 0x73, 0x3b, 0xed, 0x6d, 0x42, + 0x4c, 0xfe, 0xad, 0xbc, 0x5d, 0xc9, 0x78, 0xec, 0xe0, 0x12, 0x64, 0x6a, 0x05, 0x9c, 0xde, 0x91, + 0x60, 0x96, 0x02, 0x75, 0xad, 0xf8, 0x4e, 0x73, 0xcf, 0x52, 0x4c, 0xef, 0x1b, 0xce, 0x52, 0x00, + 0xc0, 0x51, 0x82, 0xa8, 0x09, 0x23, 0xb7, 0x7d, 0x1d, 0xb9, 0xd0, 0x8b, 0x95, 0xf2, 0x52, 0x97, + 0x14, 0xed, 0xdc, 0xde, 0x2b, 0x28, 0xc4, 0x21, 0x11, 0xb4, 0x0d, 0x43, 0xb7, 0x39, 0xaf, 0x10, + 0xfa, 0xac, 0xf9, 0x9e, 0x6f, 0xff, 0x5c, 0xad, 0x22, 0x8a, 0xb0, 0x8f, 0x5e, 0xf6, 0x25, 0x18, + 0x3e, 0xc0, 0xc5, 0xed, 0xb3, 0x0a, 0x9c, 0xd9, 0x21, 0x8e, 0x67, 0xe8, 0xf1, 0x97, 0xa1, 0x91, + 0xfc, 0x1a, 0x8a, 0x67, 0xd2, 0x10, 0xf2, 0x6d, 0x92, 0x0a, 0xc2, 0xe9, 0x5d, 0x40, 0xcf, 0x40, + 0x3f, 0xf1, 0xf4, 0x9a, 0x88, 0xc5, 0xfd, 0x9e, 0xbc, 0x2e, 0xbb, 0xdc, 0xef, 0x85, 0xfe, 0x87, + 0x19, 0x3e, 0xf5, 0xaf, 0x15, 0x48, 0xe8, 0xa6, 0xd1, 0x0f, 0x29, 0x30, 0xb6, 0x45, 0x34, 0xaf, + 0xe5, 0x90, 0xeb, 0xc2, 0x26, 0xb4, 0xef, 0xea, 0xe8, 0x43, 0xcf, 0x1c, 0x85, 0x4a, 0x7c, 0x6e, + 0x49, 0x42, 0xcc, 0x8d, 0x55, 0x82, 0xdc, 0x21, 0x32, 0x08, 0x47, 0x7a, 0x30, 0xfb, 0x24, 0x4c, + 0x25, 0x1a, 0x1e, 0xea, 0x99, 0xf2, 0xd7, 0x15, 0x48, 0x4b, 0x53, 0x8f, 0x5e, 0x84, 0x01, 0xad, + 0x56, 0x0b, 0xb2, 0x95, 0x3e, 0x9a, 0xcf, 0x6e, 0xaa, 0x26, 0x47, 0x0a, 0x62, 0x3f, 0x31, 0x47, + 0x8b, 0x96, 0x00, 0x69, 0x91, 0x77, 0xe9, 0x95, 0x30, 0x44, 0x05, 0x7b, 0x4e, 0x9b, 0x4f, 0x40, + 0x71, 0x4a, 0x0b, 0xf5, 0x13, 0x0a, 0xa0, 0x64, 0xb6, 0x19, 0xe4, 0xc0, 0xb0, 0xd8, 0xbf, 0xfe, + 0x2a, 0x95, 0x73, 0x7a, 0xd3, 0x45, 0x5c, 0x43, 0x43, 0x23, 0x3c, 0x51, 0xe0, 0xe2, 0x80, 0x8e, + 0xfa, 0xf7, 0x0a, 0x84, 0x19, 0x0d, 0xd1, 0xbb, 0x60, 0xb4, 0x46, 0x5c, 0xdd, 0x31, 0x9a, 0x5e, + 0xe8, 0x48, 0x1a, 0x38, 0xa4, 0x95, 0x43, 0x10, 0x96, 0xeb, 0x21, 0x15, 0x06, 0x3d, 0xcd, 0xbd, + 0x5d, 0x29, 0x8b, 0xcb, 0x1e, 0x3b, 0x9a, 0xd7, 0x59, 0x09, 0x16, 0x90, 0x30, 0x74, 0x6c, 0x5f, + 0x17, 0xa1, 0x63, 0xd1, 0xd6, 0x11, 0xc4, 0xc9, 0x45, 0x07, 0xc7, 0xc8, 0x55, 0x7f, 0xb6, 0x00, + 0xa7, 0x68, 0x95, 0x15, 0xcd, 0xb0, 0x3c, 0x62, 0x31, 0xb7, 0xa9, 0x9c, 0x93, 0x50, 0x87, 0x71, + 0x2f, 0xe2, 0xb4, 0x7c, 0x78, 0xa7, 0xda, 0xc0, 0xd2, 0x2b, 0xea, 0xaa, 0x1c, 0xc5, 0x8b, 0x1e, + 0xf5, 0xfd, 0xd6, 0xf8, 0xb5, 0xf8, 0x3e, 0x7f, 0xab, 0x32, 0x67, 0xb4, 0x7b, 0xc2, 0x03, 0x3c, + 0x48, 0x83, 0x19, 0x71, 0x51, 0x7b, 0x04, 0xc6, 0x85, 0x87, 0x04, 0x8f, 0x01, 0x2c, 0xae, 0xc5, + 0xec, 0x58, 0x59, 0x92, 0x01, 0x38, 0x5a, 0x4f, 0xfd, 0xa3, 0x02, 0x44, 0x93, 0x6d, 0xe6, 0x9d, + 0xa5, 0x64, 0x00, 0xe4, 0xc2, 0xb1, 0x05, 0x40, 0xe6, 0xe9, 0xe6, 0x99, 0x39, 0xb7, 0x78, 0x67, + 0x97, 0x53, 0x4d, 0xb3, 0x72, 0x1c, 0xd4, 0x08, 0xa7, 0xb5, 0xff, 0xd0, 0xd3, 0xfa, 0x2e, 0x61, + 0xf9, 0x3b, 0x10, 0x09, 0x43, 0xed, 0x5b, 0xfe, 0x4e, 0x45, 0x1a, 0x4a, 0x5e, 0x76, 0xf3, 0x20, + 0xd2, 0xd5, 0xd0, 0x75, 0x11, 0x21, 0xb2, 0xdd, 0x75, 0xdb, 0xd3, 0x4c, 0x36, 0x9d, 0x42, 0xe9, + 0xba, 0x22, 0x03, 0x70, 0xb4, 0x9e, 0xba, 0x0a, 0x6f, 0x5e, 0xb6, 0xb5, 0xda, 0x82, 0x66, 0xd2, + 0xad, 0xeb, 0x08, 0xb3, 0x3c, 0x97, 0x9d, 0xcc, 0x6b, 0x8e, 0xed, 0xd9, 0xba, 0x6d, 0xd2, 0x73, + 0x53, 0x0b, 0x62, 0x05, 0xb3, 0x87, 0x48, 0xff, 0xdc, 0x14, 0xb1, 0x76, 0xb1, 0x0f, 0x57, 0x7f, + 0x57, 0x81, 0x21, 0x91, 0xc2, 0xa6, 0x0b, 0xc7, 0xd2, 0x2d, 0x18, 0x60, 0xb7, 0xa3, 0x5e, 0xa4, + 0xd2, 0xea, 0xb6, 0x6d, 0x7b, 0x91, 0xdc, 0x57, 0x3c, 0x7b, 0x1c, 0xcb, 0xf7, 0xc9, 0xd1, 0x33, + 0x7b, 0x54, 0x47, 0xdf, 0x36, 0x3c, 0xc2, 0xcc, 0x6e, 0xc4, 0xc6, 0xe7, 0xf6, 0xa8, 0x52, 0x39, + 0x8e, 0xd4, 0x52, 0xff, 0xeb, 0x00, 0x5c, 0xf1, 0xd3, 0xf1, 0xc4, 0x45, 0xb5, 0x80, 0xe7, 0xee, + 0x05, 0xf9, 0x81, 0x58, 0x7e, 0x03, 0x5f, 0xa7, 0x9a, 0xef, 0x96, 0x7c, 0x4e, 0xca, 0x00, 0x24, + 0xa3, 0xc3, 0x69, 0x34, 0x78, 0xf4, 0x75, 0x56, 0x7c, 0x83, 0x68, 0xa6, 0xb7, 0xed, 0xd3, 0x2e, + 0xf4, 0x12, 0x7d, 0x3d, 0x89, 0x0f, 0xa7, 0x52, 0x61, 0x26, 0x1d, 0x7e, 0x5e, 0x22, 0x87, 0x68, + 0xb2, 0x3d, 0x49, 0x0f, 0xee, 0x46, 0x2b, 0xa9, 0x18, 0x71, 0x06, 0x25, 0xa6, 0x6e, 0xd4, 0x76, + 0x99, 0xf6, 0x02, 0x13, 0xcf, 0x31, 0x58, 0x0e, 0xb3, 0xe0, 0xad, 0x62, 0x25, 0x0a, 0xc2, 0xf1, + 0xba, 0xe8, 0x31, 0x98, 0x60, 0x26, 0x32, 0x61, 0x38, 0xcf, 0x81, 0x30, 0xa8, 0xd3, 0x6a, 0x04, + 0x82, 0x63, 0x35, 0xd1, 0xf7, 0x28, 0x30, 0x6d, 0xc8, 0xce, 0x34, 0xfe, 0xe8, 0x07, 0x7b, 0x49, + 0x0b, 0xc1, 0xb7, 0x71, 0x0a, 0x5a, 0x9c, 0x4a, 0x0c, 0x2d, 0xc3, 0xb4, 0xf0, 0x88, 0x8e, 0xee, + 0x01, 0x1e, 0xeb, 0x8d, 0xad, 0x69, 0x39, 0x05, 0x8e, 0x53, 0x5b, 0xa9, 0x1f, 0x2d, 0xc0, 0xd8, + 0x21, 0xf3, 0xd1, 0xb6, 0x24, 0x99, 0xa3, 0x07, 0xbf, 0x45, 0x99, 0x6a, 0x17, 0x62, 0x07, 0x7a, + 0x0e, 0x26, 0x5a, 0x6c, 0x22, 0xfc, 0x48, 0x68, 0xe2, 0x9b, 0xfe, 0x36, 0xba, 0x72, 0x1b, 0x11, + 0xc8, 0xbd, 0x76, 0x71, 0x56, 0x46, 0x1f, 0x85, 0xe2, 0x18, 0x1e, 0xf5, 0x37, 0x0b, 0x80, 0xe4, + 0xea, 0x4b, 0xa6, 0xb6, 0x63, 0x3b, 0xe8, 0xf3, 0x0a, 0x8c, 0xe9, 0x72, 0x06, 0x30, 0x2e, 0x60, + 0x3d, 0xd7, 0xeb, 0x60, 0x39, 0xfa, 0x39, 0x39, 0xb5, 0x17, 0x17, 0x84, 0x7d, 0x77, 0xeb, 0x31, + 0x19, 0x74, 0x2f, 0xf6, 0x1b, 0x47, 0xfa, 0x34, 0xfb, 0x71, 0x05, 0xa6, 0x12, 0x98, 0x52, 0x24, + 0xe3, 0xe7, 0xa3, 0x31, 0xd0, 0x8f, 0x24, 0xd7, 0xa7, 0x2c, 0x5f, 0xff, 0xd4, 0x00, 0x9c, 0x4e, + 0x59, 0x51, 0x66, 0x62, 0x43, 0x62, 0xd2, 0x65, 0x2f, 0x26, 0x36, 0x09, 0x49, 0x35, 0x30, 0xb1, + 0x89, 0x43, 0x70, 0x82, 0x2e, 0x7a, 0x06, 0xfa, 0x74, 0xc7, 0x10, 0x9b, 0xf6, 0x91, 0x5c, 0x53, + 0x80, 0x2b, 0x0b, 0xa3, 0x82, 0x62, 0x5f, 0x09, 0x57, 0x30, 0x45, 0x48, 0xcf, 0x62, 0xf9, 0x18, + 0xf1, 0x05, 0x56, 0x76, 0x16, 0xcb, 0xa7, 0x8d, 0x8b, 0xa3, 0xf5, 0xd0, 0x73, 0x30, 0x23, 0x6e, + 0xaa, 0x7e, 0x24, 0x13, 0xdb, 0x72, 0x3d, 0xca, 0xf1, 0x3d, 0x21, 0x53, 0x30, 0xeb, 0xd7, 0x9b, + 0x19, 0x75, 0x70, 0x66, 0x6b, 0xf4, 0xdd, 0x30, 0x11, 0xe1, 0x1e, 0x7e, 0x54, 0xb2, 0x9c, 0x2e, + 0x33, 0x32, 0x26, 0xce, 0x2b, 0xa3, 0x65, 0x38, 0x46, 0x0d, 0xfd, 0xdf, 0x0a, 0x4c, 0x05, 0x1b, + 0x75, 0x8f, 0x6f, 0x79, 0x3f, 0x19, 0xe3, 0xd2, 0xd1, 0x7c, 0x41, 0x52, 0xa6, 0xdc, 0x38, 0x21, + 0x9c, 0xa4, 0xad, 0xfe, 0xe8, 0x20, 0xc8, 0xc9, 0xb9, 0xd1, 0x4a, 0x2f, 0x7a, 0xc9, 0x70, 0x0f, + 0xf8, 0xba, 0xc9, 0x15, 0xe8, 0xab, 0x37, 0x5b, 0x39, 0x15, 0x93, 0x01, 0xba, 0xeb, 0x14, 0x5d, + 0xbd, 0xd9, 0x42, 0xcf, 0x04, 0xaa, 0xce, 0x7c, 0xca, 0xc8, 0xc0, 0x55, 0x32, 0xa6, 0xee, 0xf4, + 0xd9, 0x7b, 0x7f, 0x26, 0x7b, 0x6f, 0xc0, 0x90, 0x2b, 0xf4, 0xa0, 0x03, 0xf9, 0xc3, 0x48, 0x4a, + 0x33, 0x2d, 0xf4, 0x9e, 0x5c, 0x43, 0xe3, 0xab, 0x45, 0x7d, 0x1a, 0xf4, 0x22, 0xd8, 0x62, 0xe7, + 0x92, 0x08, 0x92, 0xc4, 0x2e, 0x82, 0x1b, 0xac, 0x04, 0x0b, 0x48, 0x42, 0x98, 0x1b, 0xea, 0x46, + 0x98, 0x43, 0x9f, 0x8b, 0xf3, 0xef, 0x61, 0xb6, 0xfb, 0x9e, 0xee, 0x71, 0x38, 0xff, 0x87, 0x31, + 0xee, 0xff, 0x2b, 0x3c, 0xfc, 0xa4, 0xc5, 0x42, 0xf7, 0xc1, 0x00, 0x8b, 0xa2, 0x24, 0xe4, 0x80, + 0x40, 0xb9, 0xc1, 0xe2, 0xe8, 0x60, 0x0e, 0x43, 0x55, 0x11, 0x3a, 0x30, 0xdf, 0xa6, 0x67, 0xb6, + 0x8d, 0x82, 0x9e, 0x14, 0x67, 0xf0, 0x4a, 0xc4, 0x27, 0x32, 0xed, 0x0e, 0xb1, 0x01, 0x43, 0x0d, + 0xc3, 0x62, 0xe6, 0x1e, 0xf9, 0x94, 0xe8, 0xdc, 0x04, 0x8b, 0xa3, 0xc0, 0x3e, 0x2e, 0xf5, 0x2f, + 0x0a, 0x94, 0x41, 0x84, 0x97, 0xfa, 0x3d, 0x1e, 0x08, 0x57, 0x78, 0x5f, 0x2b, 0xf9, 0x95, 0x80, + 0x12, 0xd2, 0xf9, 0x00, 0x21, 0x37, 0x54, 0x08, 0x7f, 0x63, 0x89, 0x18, 0x25, 0xed, 0x19, 0x0d, + 0xf2, 0xac, 0x61, 0xd5, 0xec, 0x3b, 0x62, 0x7a, 0x7b, 0x25, 0xbd, 0x1e, 0x20, 0xe4, 0xa4, 0xc3, + 0xdf, 0x58, 0x22, 0x46, 0x8f, 0x24, 0x96, 0x8c, 0xc3, 0x62, 0x49, 0x95, 0x45, 0xdf, 0x78, 0xca, + 0x74, 0x61, 0x77, 0xcc, 0x8e, 0xa4, 0x52, 0x46, 0x1d, 0x9c, 0xd9, 0x5a, 0xfd, 0x79, 0x05, 0xce, + 0xa4, 0x4e, 0x05, 0xba, 0x0e, 0x53, 0xa1, 0x39, 0xac, 0x2c, 0x24, 0x0c, 0x87, 0x3c, 0xfe, 0x66, + 0xbc, 0x02, 0x4e, 0xb6, 0xe1, 0xa9, 0xea, 0x12, 0x42, 0x88, 0xb0, 0xa5, 0x95, 0xaf, 0x5a, 0x32, + 0x18, 0xa7, 0xb5, 0x51, 0x5f, 0x88, 0x74, 0x36, 0x9c, 0x2c, 0xfa, 0x65, 0x6c, 0x92, 0x7a, 0xe0, + 0x93, 0x1e, 0x7c, 0x19, 0x0b, 0xb4, 0x10, 0x73, 0x18, 0xba, 0x24, 0x87, 0xca, 0x08, 0xb8, 0xbb, + 0x1f, 0x2e, 0x43, 0xfd, 0x75, 0x05, 0xce, 0xad, 0x68, 0x56, 0x4b, 0x33, 0xb9, 0xf3, 0xfb, 0x9a, + 0x6d, 0x9b, 0x62, 0x9a, 0x3a, 0xa5, 0xce, 0x57, 0x5e, 0x2f, 0xa9, 0xf3, 0x3f, 0x00, 0xe7, 0x32, + 0xec, 0x6f, 0x50, 0x19, 0xc6, 0xdc, 0x3b, 0x5a, 0x73, 0x81, 0x6c, 0x6b, 0x3b, 0x86, 0x08, 0xac, + 0xc5, 0xcd, 0xb4, 0xc7, 0xaa, 0x52, 0xf9, 0xbd, 0xd8, 0x6f, 0x1c, 0x69, 0xa5, 0x7a, 0x00, 0xc2, + 0x9c, 0xdf, 0xb0, 0xea, 0x68, 0x0b, 0x86, 0x35, 0x93, 0x38, 0x5e, 0x18, 0x3c, 0xf2, 0xdb, 0x73, + 0xe9, 0x69, 0x05, 0x0e, 0xee, 0xbe, 0xe6, 0xff, 0xc2, 0x01, 0x6e, 0xf5, 0xff, 0x57, 0xe0, 0x6c, + 0x7a, 0x28, 0xa5, 0x2e, 0xae, 0x45, 0x0d, 0x18, 0x75, 0xc2, 0x66, 0xe2, 0xa3, 0x7d, 0xb7, 0x9c, + 0xa1, 0x42, 0x72, 0xbc, 0xa1, 0x17, 0xc1, 0x92, 0x63, 0xbb, 0xfe, 0xce, 0x8d, 0x27, 0xad, 0x08, + 0xb4, 0x62, 0x52, 0x4f, 0xb0, 0x8c, 0x9f, 0x25, 0x90, 0xa1, 0xd4, 0xdd, 0xa6, 0xa6, 0x93, 0xda, + 0x09, 0xa7, 0xc7, 0x3f, 0x82, 0xac, 0x0d, 0xe9, 0x7d, 0x3f, 0xde, 0x04, 0x32, 0x19, 0x34, 0x0f, + 0x4e, 0x20, 0x93, 0xde, 0xf0, 0x0d, 0x92, 0xd9, 0x20, 0xbd, 0xf3, 0x19, 0x8e, 0xef, 0xff, 0x79, + 0x30, 0x6b, 0xb4, 0x87, 0xcc, 0xb1, 0xbf, 0x73, 0x8c, 0x39, 0xf6, 0x27, 0xbe, 0x99, 0x5f, 0x3f, + 0x25, 0xbf, 0x7e, 0x2c, 0xe7, 0xfb, 0xe0, 0x09, 0xe5, 0x7c, 0x7f, 0x19, 0x06, 0x9b, 0x9a, 0x43, + 0x2c, 0xff, 0x2d, 0xb7, 0x92, 0xcf, 0xd0, 0x20, 0xdc, 0x68, 0x21, 0x17, 0x0c, 0x3e, 0xc9, 0x35, + 0x46, 0x00, 0x0b, 0x42, 0x29, 0xc1, 0x53, 0x86, 0x8f, 0x2b, 0x3a, 0x61, 0x98, 0x6d, 0x7e, 0xe4, + 0x38, 0xb2, 0xcd, 0xab, 0x7f, 0xa7, 0xc0, 0xc5, 0x4e, 0x6c, 0x89, 0x29, 0x50, 0xf4, 0xd8, 0x67, + 0xd8, 0x8b, 0x02, 0x25, 0xc1, 0x6d, 0x03, 0x05, 0x4a, 0x1c, 0x82, 0x13, 0x74, 0xd1, 0xfb, 0x00, + 0xd9, 0x9b, 0xdc, 0xae, 0xe7, 0x3a, 0xa5, 0xc1, 0xbd, 0x69, 0x0b, 0xcc, 0x57, 0x21, 0xc8, 0x6b, + 0x79, 0x2b, 0x51, 0x03, 0xa7, 0xb4, 0x52, 0x7f, 0xb5, 0x00, 0xb0, 0x4a, 0xbc, 0x3b, 0xb6, 0x73, + 0x9b, 0x9e, 0xf1, 0x17, 0x23, 0x4f, 0x07, 0xc3, 0xdf, 0xb8, 0x78, 0x94, 0x17, 0xa1, 0xbf, 0x69, + 0xd7, 0x5c, 0x71, 0xff, 0x60, 0x1d, 0x61, 0xae, 0x1a, 0xac, 0x14, 0x15, 0x61, 0x80, 0x19, 0x3d, + 0x89, 0x0b, 0x34, 0x7b, 0x78, 0x60, 0x8f, 0x2b, 0x98, 0x97, 0x53, 0x0e, 0x29, 0x62, 0x1a, 0xb8, + 0xe2, 0x71, 0x67, 0x8c, 0xc7, 0x10, 0xe7, 0x65, 0x38, 0x80, 0xa2, 0xc7, 0x00, 0x8c, 0xe6, 0x92, + 0xd6, 0x30, 0x4c, 0x43, 0x7c, 0xae, 0x23, 0x4c, 0x23, 0x0e, 0x95, 0x35, 0xbf, 0xf4, 0x5e, 0xbb, + 0x38, 0x2c, 0x7e, 0xed, 0x61, 0xa9, 0xb6, 0xfa, 0x45, 0x05, 0x26, 0xc3, 0xc9, 0x13, 0x5b, 0xc5, + 0xef, 0x39, 0x0f, 0x06, 0x9c, 0xd9, 0x73, 0x1e, 0xb6, 0xbf, 0x73, 0xcf, 0xb9, 0x02, 0x2b, 0xab, + 0xe7, 0x0f, 0xc2, 0x28, 0xe1, 0x21, 0x8f, 0x2a, 0x65, 0xcc, 0x79, 0xdc, 0x08, 0xbf, 0xce, 0x2d, + 0x86, 0xc5, 0x58, 0xae, 0xa3, 0xfe, 0x43, 0x1f, 0x8c, 0xad, 0xd6, 0x0d, 0x6b, 0xd7, 0x8f, 0xed, + 0x14, 0x3c, 0xbc, 0x2b, 0xc7, 0xf3, 0xf0, 0xfe, 0x1c, 0xcc, 0x98, 0xf2, 0x33, 0x17, 0x17, 0x9c, + 0x34, 0xab, 0x1e, 0xcc, 0x00, 0xbb, 0xc7, 0x2c, 0x67, 0xd4, 0xc1, 0x99, 0xad, 0x91, 0x07, 0x83, + 0xba, 0x9f, 0x5f, 0x31, 0x77, 0xbc, 0x22, 0x79, 0x2e, 0xe6, 0xe4, 0xd0, 0x1d, 0x01, 0xcf, 0x13, + 0xdb, 0x53, 0xd0, 0x42, 0x1f, 0x53, 0xe0, 0x0c, 0xd9, 0xe5, 0xa1, 0x6b, 0xd6, 0x1d, 0x6d, 0x6b, + 0xcb, 0xd0, 0x85, 0xc7, 0x1f, 0xdf, 0x89, 0xcb, 0xfb, 0xed, 0xe2, 0x99, 0xc5, 0xb4, 0x0a, 0xf7, + 0xda, 0xc5, 0x6b, 0xa9, 0x91, 0x84, 0xd8, 0x6a, 0xa6, 0x36, 0xc1, 0xe9, 0xa4, 0x66, 0x1f, 0x85, + 0xd1, 0x43, 0xf8, 0x89, 0x47, 0xe2, 0x05, 0xfd, 0x5a, 0x01, 0xc6, 0xe8, 0x76, 0x5b, 0xb6, 0x75, + 0xcd, 0x2c, 0xaf, 0x56, 0xd1, 0xfd, 0xf1, 0x30, 0x89, 0xc1, 0x13, 0x63, 0x22, 0x54, 0xe2, 0x32, + 0x4c, 0x6f, 0xd9, 0x8e, 0x4e, 0xd6, 0x4b, 0x6b, 0xeb, 0xb6, 0x30, 0x3e, 0x2b, 0xaf, 0x56, 0xc5, + 0xbd, 0x8e, 0x3d, 0x79, 0x2c, 0xa5, 0xc0, 0x71, 0x6a, 0x2b, 0x74, 0x0b, 0xce, 0x84, 0xe5, 0x1b, + 0x4d, 0xee, 0xb0, 0x40, 0xd1, 0xf5, 0x85, 0x0e, 0x17, 0x4b, 0x69, 0x15, 0x70, 0x7a, 0x3b, 0xa4, + 0xc1, 0x05, 0xf1, 0xb6, 0xb2, 0x64, 0x3b, 0x77, 0x34, 0xa7, 0x16, 0x45, 0xdb, 0x1f, 0xa6, 0x9d, + 0x2f, 0x67, 0x57, 0xc3, 0x9d, 0x70, 0xa8, 0x7f, 0xaf, 0x40, 0x34, 0x08, 0x25, 0x3a, 0x0f, 0x7d, + 0x8e, 0x48, 0xdf, 0x27, 0x82, 0x31, 0xd2, 0x2b, 0x02, 0x2d, 0x43, 0x73, 0x00, 0x4e, 0x18, 0x09, + 0xb3, 0x10, 0xa6, 0xb5, 0x90, 0x62, 0x58, 0x4a, 0x35, 0x28, 0x2a, 0x4f, 0xab, 0x0b, 0x86, 0xc7, + 0x50, 0xad, 0x6b, 0x75, 0x4c, 0xcb, 0x58, 0xfe, 0x12, 0xa3, 0x4e, 0x5c, 0x5f, 0x1d, 0xcd, 0xf3, + 0x97, 0xb0, 0x12, 0x2c, 0x20, 0x48, 0x83, 0xf1, 0x66, 0xcb, 0x14, 0xf1, 0x94, 0xe8, 0xd5, 0x87, + 0xab, 0x0d, 0xaf, 0xa6, 0x25, 0xe7, 0x63, 0xab, 0x9f, 0x9a, 0xa1, 0x6f, 0x4d, 0x46, 0x81, 0xa3, + 0x18, 0xd5, 0x9f, 0x18, 0x04, 0x29, 0xbc, 0xce, 0x21, 0xa4, 0xd0, 0x9f, 0x51, 0x60, 0x5a, 0x37, + 0x0d, 0x62, 0x79, 0xb1, 0x48, 0x15, 0xfc, 0xf8, 0xd8, 0xc8, 0x15, 0xf7, 0xa7, 0x49, 0xac, 0x4a, + 0x59, 0xb8, 0xb7, 0x94, 0x52, 0x90, 0x0b, 0x17, 0xa0, 0x14, 0x08, 0x4e, 0xed, 0x0c, 0x1b, 0x0f, + 0x2b, 0xaf, 0x94, 0xe5, 0xe8, 0x99, 0x25, 0x51, 0x86, 0x03, 0x28, 0xe5, 0xbc, 0x75, 0xc7, 0x6e, + 0x35, 0xdd, 0x12, 0xf3, 0x62, 0xe5, 0x8b, 0xc2, 0x38, 0xef, 0xf5, 0xb0, 0x18, 0xcb, 0x75, 0xd0, + 0xc3, 0x30, 0xc6, 0x7f, 0xae, 0x39, 0x64, 0xcb, 0xd8, 0x15, 0x87, 0x12, 0x53, 0x9e, 0x5e, 0x97, + 0xca, 0x71, 0xa4, 0x16, 0x8b, 0xdf, 0xe6, 0xba, 0x2d, 0xe2, 0x6c, 0xe0, 0x65, 0x91, 0xe3, 0x99, + 0xc7, 0x6f, 0xf3, 0x0b, 0x71, 0x08, 0x47, 0x3f, 0xa2, 0xc0, 0x84, 0x43, 0x5e, 0x6e, 0x19, 0x0e, + 0x15, 0x61, 0x34, 0xa3, 0xe1, 0x8a, 0x18, 0x47, 0xb8, 0xb7, 0xb8, 0x4a, 0x73, 0x38, 0x82, 0x94, + 0x33, 0xc8, 0xc0, 0x94, 0x23, 0x0a, 0xc4, 0xb1, 0x1e, 0xd0, 0xa9, 0x72, 0x8d, 0xba, 0x65, 0x58, + 0xf5, 0x79, 0xb3, 0xee, 0x8a, 0x44, 0x56, 0x5c, 0xe7, 0x18, 0x16, 0x63, 0xb9, 0x0e, 0x7a, 0x04, + 0xc6, 0x5b, 0x2e, 0x65, 0x7b, 0x0d, 0xc2, 0xe7, 0x77, 0x24, 0xb4, 0x75, 0xd9, 0x90, 0x01, 0x38, + 0x5a, 0x0f, 0x3d, 0x06, 0x13, 0x7e, 0x81, 0x98, 0x65, 0xe0, 0xd9, 0x54, 0xd8, 0xdb, 0x64, 0x04, + 0x82, 0x63, 0x35, 0x67, 0xe7, 0xe1, 0x74, 0xca, 0x30, 0x0f, 0xc5, 0x5b, 0xff, 0x97, 0x02, 0x67, + 0xb8, 0xd4, 0xe5, 0x67, 0x87, 0xf6, 0x73, 0x5b, 0xa4, 0xa7, 0x89, 0x50, 0x8e, 0x35, 0x4d, 0xc4, + 0x37, 0x20, 0x1d, 0x86, 0xfa, 0x73, 0x05, 0x78, 0xf3, 0x81, 0xdf, 0x25, 0xfa, 0x49, 0x05, 0x46, + 0xc9, 0xae, 0xe7, 0x68, 0x81, 0xab, 0x3f, 0xdd, 0xa4, 0x5b, 0xc7, 0xc2, 0x04, 0xe6, 0x16, 0x43, + 0x42, 0x7c, 0xe3, 0x06, 0x57, 0x29, 0x09, 0x82, 0xe5, 0xfe, 0x50, 0x6e, 0xcb, 0xb3, 0x0d, 0xc9, + 0x46, 0x71, 0x82, 0x0b, 0x0a, 0xc8, 0xec, 0x13, 0x30, 0x19, 0xc7, 0x7c, 0xa8, 0xbd, 0xf2, 0x05, + 0x05, 0x52, 0xc3, 0x71, 0xa2, 0x12, 0x4c, 0x69, 0x2d, 0xcf, 0x8e, 0xbc, 0xeb, 0x09, 0x51, 0x92, + 0x19, 0x7d, 0xcf, 0xc7, 0x81, 0x38, 0x59, 0x9f, 0x2b, 0x60, 0xad, 0x96, 0x66, 0x46, 0xd1, 0x70, + 0x81, 0x4b, 0x28, 0x60, 0x13, 0x60, 0x9c, 0xd6, 0x46, 0xfd, 0x67, 0x0a, 0x9c, 0x49, 0x55, 0x4b, + 0x76, 0xa1, 0x8b, 0x4b, 0xdf, 0xf6, 0x85, 0xe3, 0xdc, 0xf6, 0xea, 0xaf, 0x14, 0x60, 0x68, 0xcd, + 0xb1, 0xe9, 0xd1, 0x76, 0x02, 0x1a, 0x38, 0x2d, 0xa2, 0x81, 0xcb, 0xa5, 0x5f, 0x10, 0x9d, 0xcd, + 0x54, 0xb9, 0x19, 0x31, 0x95, 0xdb, 0x7c, 0x2f, 0x44, 0x3a, 0xeb, 0xd8, 0xbe, 0xac, 0xc0, 0xa8, + 0xa8, 0x79, 0x02, 0x4a, 0xb5, 0xef, 0x8a, 0x2a, 0xd5, 0xde, 0xdb, 0xc3, 0xb8, 0x32, 0xb4, 0x68, + 0x9f, 0x55, 0x60, 0x5c, 0xd4, 0x58, 0x21, 0x8d, 0x4d, 0xe2, 0xa0, 0x25, 0x18, 0x72, 0x5b, 0x6c, + 0x21, 0xc5, 0x80, 0x2e, 0xc8, 0xe2, 0x91, 0xb3, 0xa9, 0xe9, 0xb4, 0xfb, 0x55, 0x5e, 0x45, 0xca, + 0x84, 0xcc, 0x0b, 0xb0, 0xdf, 0x98, 0xee, 0x7d, 0xc7, 0x36, 0x13, 0x61, 0xeb, 0xb1, 0x6d, 0x12, + 0xcc, 0x20, 0xf4, 0x9e, 0x47, 0xff, 0xfa, 0x77, 0x38, 0x76, 0xcf, 0xa3, 0x60, 0x17, 0xf3, 0x72, + 0xf5, 0x8b, 0x03, 0xc1, 0x64, 0xb3, 0x4b, 0xfd, 0x0d, 0x18, 0xd1, 0x1d, 0xa2, 0x79, 0xa4, 0xb6, + 0xb0, 0xd7, 0x4d, 0xe7, 0x98, 0x1c, 0x50, 0xf2, 0x5b, 0xe0, 0xb0, 0x31, 0x3d, 0x72, 0x65, 0x03, + 0xcf, 0x42, 0x28, 0x9d, 0x64, 0x1a, 0x77, 0x7e, 0x3b, 0x0c, 0xd8, 0x77, 0xac, 0xc0, 0x39, 0xa4, + 0x23, 0x61, 0x36, 0x94, 0x5b, 0xb4, 0x36, 0xe6, 0x8d, 0xe4, 0xb4, 0x0d, 0xfd, 0x1d, 0xd2, 0x36, + 0x98, 0x30, 0xd4, 0x60, 0xcb, 0xd0, 0x53, 0x62, 0xdc, 0xc8, 0x82, 0x86, 0x4b, 0xc4, 0x7f, 0xbb, + 0xd8, 0x27, 0x41, 0x45, 0x27, 0xcb, 0xd7, 0xe8, 0xc8, 0xa2, 0x53, 0xa0, 0xe6, 0xc1, 0x21, 0x1c, + 0xed, 0x45, 0xf3, 0x81, 0x0c, 0xe5, 0xd7, 0x93, 0x8a, 0xee, 0x49, 0x29, 0x40, 0xf8, 0xd4, 0x67, + 0xe5, 0x04, 0x41, 0x5f, 0x50, 0xe0, 0x5c, 0x2d, 0x3d, 0xe1, 0x9a, 0x78, 0x2a, 0xcf, 0xe5, 0x98, + 0x9d, 0x91, 0xc3, 0x6d, 0xa1, 0x28, 0x26, 0x2c, 0x2b, 0xc9, 0x1b, 0xce, 0xea, 0x8c, 0xfa, 0xfd, + 0xfd, 0xc1, 0xd7, 0x24, 0x34, 0x1d, 0xe9, 0x7a, 0x28, 0x25, 0x8f, 0x1e, 0x0a, 0xbd, 0xd3, 0x4f, + 0xff, 0xc5, 0xb7, 0xeb, 0xa5, 0x78, 0xfa, 0xaf, 0x31, 0x41, 0x3a, 0x92, 0xf2, 0xab, 0x05, 0xa7, + 0x5d, 0x4f, 0x33, 0x49, 0xd5, 0x10, 0x8f, 0x83, 0xae, 0xa7, 0x35, 0x9a, 0x39, 0xf2, 0x6f, 0xf1, + 0x40, 0x0d, 0x49, 0x54, 0x38, 0x0d, 0x3f, 0xfa, 0x5e, 0x16, 0x7c, 0x4f, 0x33, 0xd9, 0xe3, 0x29, + 0xcf, 0x41, 0x1a, 0x12, 0x3f, 0xbc, 0xb9, 0xbb, 0x08, 0xad, 0x97, 0x8e, 0x0f, 0x67, 0x52, 0x42, + 0xaf, 0xc0, 0x19, 0x7a, 0x00, 0xce, 0xeb, 0x9e, 0xb1, 0x63, 0x78, 0x7b, 0x61, 0x17, 0x0e, 0x9f, + 0x74, 0x8b, 0xdd, 0xb6, 0x97, 0xd3, 0x90, 0xe1, 0x74, 0x1a, 0xea, 0xdf, 0x2a, 0x80, 0x92, 0x7b, + 0x1d, 0x99, 0x30, 0x5c, 0xf3, 0x23, 0x27, 0x28, 0x47, 0x92, 0x55, 0x27, 0x38, 0x42, 0x82, 0x80, + 0x0b, 0x01, 0x05, 0x64, 0xc3, 0xc8, 0x9d, 0x6d, 0xc3, 0x23, 0xa6, 0xe1, 0x7a, 0x47, 0x94, 0xc4, + 0x27, 0x88, 0x09, 0xff, 0xac, 0x8f, 0x18, 0x87, 0x34, 0xd4, 0x4f, 0xf5, 0xc3, 0x70, 0x90, 0x4c, + 0xf3, 0x60, 0x33, 0xeb, 0x16, 0x20, 0x5d, 0x0a, 0x08, 0xd8, 0x8b, 0xce, 0x94, 0xc9, 0x40, 0xa5, + 0x04, 0x32, 0x9c, 0x42, 0x00, 0xbd, 0x02, 0xd3, 0x86, 0xb5, 0xe5, 0x68, 0x41, 0xb8, 0xc3, 0x92, + 0xaf, 0x28, 0xcb, 0x41, 0x98, 0xdd, 0xa2, 0x2b, 0x29, 0xe8, 0x70, 0x2a, 0x11, 0x44, 0x60, 0x88, + 0xe7, 0x33, 0xf6, 0x5f, 0x5d, 0x72, 0xe9, 0xea, 0xb9, 0xac, 0x19, 0xb2, 0x77, 0x5f, 0xf6, 0xf4, + 0x71, 0xf3, 0xe0, 0xb4, 0xfc, 0x7f, 0xff, 0x41, 0x4a, 0xec, 0xfb, 0x52, 0x7e, 0x7a, 0xe1, 0xdb, + 0x16, 0x0f, 0x4e, 0x1b, 0x2d, 0xc4, 0x71, 0x82, 0xea, 0xef, 0x29, 0x30, 0xc0, 0x63, 0x80, 0x1d, + 0xbf, 0xa8, 0xf9, 0x81, 0x88, 0xa8, 0xf9, 0x78, 0x9e, 0x41, 0xb2, 0xae, 0x66, 0x66, 0xe4, 0xff, + 0x5d, 0x05, 0x46, 0x58, 0x8d, 0x13, 0x90, 0xfd, 0x5e, 0x8c, 0xca, 0x7e, 0x8f, 0xe6, 0x1e, 0x4d, + 0x86, 0xe4, 0xf7, 0x7b, 0x7d, 0x62, 0x2c, 0x4c, 0xb4, 0xaa, 0xc0, 0x69, 0xe1, 0x18, 0xbb, 0x6c, + 0x6c, 0x11, 0xba, 0xc5, 0xcb, 0xda, 0x9e, 0x2b, 0x5c, 0x35, 0x78, 0xd0, 0x99, 0x24, 0x18, 0xa7, + 0xb5, 0x41, 0xbf, 0xa6, 0x50, 0x21, 0xc6, 0x73, 0x0c, 0xbd, 0xa7, 0xc7, 0xe0, 0xa0, 0x6f, 0x73, + 0x2b, 0x1c, 0x19, 0xbf, 0x9b, 0x6e, 0x84, 0xd2, 0x0c, 0x2b, 0xbd, 0xd7, 0x2e, 0x16, 0x53, 0x74, + 0xc6, 0x61, 0x56, 0x6a, 0xd7, 0xfb, 0xd8, 0x5f, 0x76, 0xac, 0xc2, 0x6e, 0x63, 0x7e, 0x8f, 0xd1, + 0x0d, 0x18, 0x70, 0x75, 0xbb, 0xe9, 0xbb, 0x56, 0xdf, 0x97, 0xa6, 0x1b, 0x8c, 0xab, 0x05, 0x83, + 0x09, 0xae, 0xd2, 0x96, 0x98, 0x23, 0x98, 0x7d, 0x09, 0xc6, 0xe4, 0x9e, 0xa7, 0xdc, 0x7d, 0xcb, + 0x51, 0x8b, 0xb9, 0x43, 0x1a, 0x87, 0xc9, 0x77, 0xe5, 0x3f, 0xee, 0x83, 0x41, 0x4c, 0xea, 0x22, + 0x65, 0xda, 0x01, 0x77, 0x4e, 0xc3, 0x4f, 0xff, 0x5b, 0xc8, 0xef, 0x87, 0x27, 0x27, 0xc5, 0x79, + 0xde, 0xb6, 0xa4, 0x39, 0x90, 0x33, 0x00, 0x23, 0x2b, 0x48, 0x24, 0xd5, 0x97, 0xdf, 0xa0, 0x96, + 0x0f, 0xac, 0x9b, 0xd4, 0x51, 0xe8, 0x87, 0x15, 0x40, 0x9a, 0xae, 0x13, 0xd7, 0xc5, 0xc4, 0xa5, + 0x73, 0xcf, 0x85, 0x55, 0xce, 0x65, 0xf3, 0x45, 0xc5, 0x8e, 0x63, 0x0b, 0xc5, 0xb6, 0x04, 0xc8, + 0xc5, 0x29, 0xc4, 0x7b, 0x49, 0x67, 0xf5, 0x07, 0x0a, 0x8c, 0x45, 0xb2, 0x85, 0x35, 0x42, 0x5d, + 0x7a, 0x7e, 0x93, 0x1d, 0xdf, 0xfb, 0xeb, 0x42, 0x87, 0x4a, 0x5c, 0x3f, 0x7f, 0x2b, 0x48, 0x77, + 0x71, 0x34, 0x89, 0xc5, 0xd4, 0x4f, 0x2b, 0x70, 0xd6, 0x1f, 0x50, 0x34, 0xae, 0x39, 0xba, 0x0a, + 0xc3, 0x5a, 0xd3, 0x60, 0x8a, 0x5e, 0x59, 0x55, 0x3e, 0xbf, 0x56, 0x61, 0x65, 0x38, 0x80, 0x46, + 0x72, 0x2c, 0x17, 0x0e, 0xcc, 0xb1, 0xfc, 0x56, 0x29, 0x6b, 0xf4, 0x40, 0x28, 0xbb, 0x04, 0x84, + 0xb9, 0x31, 0xa7, 0xfa, 0x6e, 0x18, 0xa9, 0x56, 0x6f, 0xf0, 0x25, 0x3d, 0xc4, 0x8b, 0x8f, 0xfa, + 0xc9, 0x3e, 0x18, 0x17, 0x09, 0x1a, 0x0c, 0xa6, 0x02, 0x3a, 0x81, 0x73, 0x6e, 0x1d, 0x46, 0xdc, + 0xe0, 0x0d, 0xa3, 0x90, 0xcd, 0xa7, 0x82, 0x67, 0x89, 0x78, 0x96, 0xbd, 0xf0, 0xf9, 0x22, 0x44, + 0x84, 0x6e, 0xc2, 0xe0, 0xcb, 0x94, 0xe7, 0xfa, 0xdf, 0x6a, 0x57, 0xac, 0x2f, 0xf8, 0x10, 0x19, + 0xbb, 0x76, 0xb1, 0x40, 0x81, 0x5c, 0xe6, 0x9e, 0xc8, 0x84, 0xc0, 0x5e, 0x42, 0x75, 0x46, 0x66, + 0x36, 0xc8, 0x19, 0x3f, 0x26, 0xbc, 0x1c, 0xd9, 0x2f, 0x1c, 0x10, 0x62, 0x29, 0x42, 0x23, 0x2d, + 0xde, 0x20, 0x29, 0x42, 0x23, 0x7d, 0xce, 0x38, 0xae, 0x1f, 0x85, 0x33, 0xa9, 0x93, 0x71, 0xb0, + 0x88, 0xad, 0xfe, 0xf3, 0x02, 0xf4, 0x57, 0x09, 0xa9, 0x9d, 0xc0, 0xce, 0x7c, 0x31, 0x22, 0x81, + 0x7d, 0x7b, 0xee, 0x24, 0xa5, 0x59, 0x9a, 0xbe, 0xad, 0x98, 0xa6, 0xef, 0x89, 0xdc, 0x14, 0x3a, + 0xab, 0xf9, 0xfe, 0xba, 0x00, 0x43, 0xb4, 0x5a, 0x79, 0xb5, 0x8a, 0x1a, 0xd2, 0x56, 0x2e, 0xe4, + 0x17, 0x9f, 0x05, 0xba, 0x83, 0x36, 0x31, 0xfd, 0x72, 0x0c, 0x91, 0xf8, 0x45, 0x0c, 0xb2, 0x72, + 0x04, 0xe4, 0x44, 0xec, 0x09, 0x46, 0xd4, 0xcf, 0x2b, 0x83, 0x03, 0x42, 0xe8, 0x8e, 0x74, 0x27, + 0xed, 0xcf, 0x6f, 0xbb, 0x9b, 0x4e, 0xb4, 0xc3, 0xf5, 0x54, 0xfd, 0xa4, 0x02, 0xa7, 0x62, 0xad, + 0xba, 0xb8, 0x34, 0x1e, 0x0b, 0x03, 0x54, 0xbf, 0xae, 0xd0, 0x6f, 0x2c, 0x65, 0x04, 0x5d, 0xf4, + 0x28, 0x4c, 0x00, 0x56, 0xe8, 0x98, 0x00, 0xec, 0x22, 0xf4, 0x53, 0xc9, 0x48, 0xb6, 0xd9, 0xa1, + 0x02, 0x13, 0x66, 0xa5, 0x48, 0x87, 0x09, 0x5d, 0x4a, 0xe1, 0x4e, 0xb6, 0x04, 0xef, 0xec, 0x8a, + 0x15, 0x07, 0xcf, 0x8d, 0xa5, 0x08, 0x0a, 0x1c, 0x43, 0xa9, 0xfe, 0x8e, 0x02, 0xc3, 0x74, 0x98, + 0x27, 0xc0, 0x1c, 0xbf, 0x33, 0xca, 0x1c, 0xdf, 0x93, 0x77, 0x4f, 0x65, 0xf0, 0xc4, 0xbf, 0x29, + 0x00, 0xcb, 0x60, 0x2c, 0xec, 0x8b, 0x24, 0xcb, 0x21, 0x25, 0xc3, 0xe6, 0xe9, 0x8a, 0x30, 0x3c, + 0x8a, 0x29, 0xa5, 0x25, 0xe3, 0xa3, 0xb7, 0x47, 0x6c, 0x8b, 0x22, 0xc2, 0x45, 0x8a, 0x7d, 0xd1, + 0x5d, 0x18, 0x77, 0xb7, 0x6d, 0xdb, 0x2b, 0x87, 0x1f, 0x4f, 0xee, 0x07, 0x08, 0xe6, 0x2c, 0xee, + 0x0f, 0x85, 0x3f, 0xe5, 0x56, 0x65, 0xdc, 0x38, 0x4a, 0x0a, 0xcd, 0x01, 0x6c, 0x9a, 0xb6, 0x7e, + 0x9b, 0x9b, 0x36, 0x71, 0xe7, 0x60, 0x66, 0x3c, 0xb1, 0x10, 0x94, 0x62, 0xa9, 0x46, 0x4f, 0x56, + 0x5c, 0x5f, 0x57, 0xf8, 0x4c, 0x1f, 0xe2, 0x1b, 0x3d, 0x41, 0x43, 0xb8, 0xb7, 0xc1, 0xa0, 0xc3, + 0xee, 0x08, 0x62, 0xc5, 0x82, 0x8f, 0x8f, 0xdf, 0x1c, 0xb0, 0x80, 0xd2, 0xed, 0xc1, 0x2f, 0x3e, + 0xfd, 0xe1, 0x83, 0x83, 0x7c, 0x5d, 0x51, 0x7f, 0x45, 0x0c, 0x33, 0x48, 0x82, 0xdd, 0x84, 0x71, + 0x76, 0xb3, 0x88, 0x65, 0xdf, 0x7e, 0x67, 0x97, 0xdf, 0x88, 0xdc, 0x34, 0x34, 0xcb, 0x8d, 0x14, + 0xe3, 0x28, 0x01, 0xf4, 0x08, 0x8c, 0xfb, 0xa3, 0xe3, 0xd6, 0xb1, 0x85, 0xd0, 0x43, 0x73, 0x4d, + 0x06, 0xe0, 0x68, 0x3d, 0xf5, 0x33, 0x05, 0xb8, 0xc4, 0xfb, 0xce, 0x34, 0x2f, 0x65, 0xd2, 0x24, + 0x56, 0x8d, 0x58, 0xfa, 0x1e, 0x93, 0xb3, 0x6b, 0x76, 0x1d, 0xbd, 0x02, 0x83, 0x77, 0x08, 0xa9, + 0x05, 0x4f, 0x18, 0xcf, 0xe6, 0xcf, 0x21, 0x9e, 0x41, 0xe2, 0x59, 0x86, 0x9e, 0xbf, 0x06, 0xf3, + 0xff, 0xb1, 0x20, 0x49, 0x89, 0x37, 0x1d, 0x7b, 0x33, 0x10, 0x07, 0x8f, 0x9e, 0xf8, 0x1a, 0x43, + 0xcf, 0x89, 0xf3, 0xff, 0xb1, 0x20, 0xa9, 0xae, 0xc1, 0x7d, 0x5d, 0x34, 0x3d, 0x8c, 0xd8, 0x7f, + 0x10, 0x46, 0x3e, 0xfa, 0xc3, 0x60, 0xfc, 0x53, 0x05, 0xde, 0x22, 0xa1, 0x5c, 0xdc, 0xa5, 0x37, + 0x91, 0x92, 0xd6, 0xd4, 0x74, 0x7a, 0xd7, 0x67, 0x31, 0xea, 0x0e, 0x95, 0xb5, 0xf7, 0x93, 0x0a, + 0x0c, 0x71, 0x8b, 0x3c, 0x9f, 0xfd, 0xbe, 0xd8, 0xe3, 0x94, 0x67, 0x76, 0xc9, 0xcf, 0x66, 0xe6, + 0x8f, 0x8d, 0xff, 0x76, 0xb1, 0x4f, 0x5f, 0xfd, 0x57, 0x03, 0xf0, 0xad, 0xdd, 0x23, 0x42, 0x5f, + 0x57, 0xe4, 0x6c, 0xe3, 0x5c, 0x47, 0xde, 0x38, 0xde, 0xce, 0x07, 0xda, 0x20, 0xa1, 0x60, 0x78, + 0x36, 0x91, 0x90, 0xfc, 0x88, 0x14, 0x4d, 0xe1, 0xc0, 0xd0, 0x3f, 0x51, 0x60, 0x8c, 0x1e, 0x4b, + 0x01, 0x73, 0xe1, 0xcb, 0xd4, 0x3c, 0xe6, 0x91, 0xae, 0x4a, 0x24, 0x63, 0x71, 0xad, 0x64, 0x10, + 0x8e, 0xf4, 0x0d, 0x6d, 0x44, 0x9f, 0xff, 0xf8, 0x15, 0xf1, 0x72, 0x9a, 0x5c, 0x72, 0x98, 0x74, + 0xff, 0xb3, 0x26, 0x4c, 0x44, 0x67, 0xfe, 0x38, 0xd5, 0x64, 0xb3, 0x4f, 0xc2, 0x54, 0x62, 0xf4, + 0x87, 0x52, 0xc8, 0x7c, 0x79, 0x10, 0x8a, 0xd2, 0x54, 0xa7, 0x85, 0xa7, 0x41, 0x9f, 0x53, 0x60, + 0x54, 0xb3, 0x2c, 0x61, 0xd8, 0xe4, 0xef, 0xdf, 0x5a, 0x8f, 0xab, 0x9a, 0x46, 0x6a, 0x6e, 0x3e, + 0x24, 0x13, 0xb3, 0xdc, 0x91, 0x20, 0x58, 0xee, 0x4d, 0x07, 0xeb, 0xdc, 0xc2, 0x89, 0x59, 0xe7, + 0xa2, 0x0f, 0xf9, 0x07, 0x71, 0x5f, 0xfe, 0x40, 0x15, 0x07, 0xcc, 0x0d, 0x3b, 0xd7, 0x33, 0xb4, + 0x92, 0x3f, 0xa8, 0xb0, 0x43, 0x36, 0x8c, 0x22, 0x24, 0xce, 0xa4, 0x5c, 0x46, 0x96, 0x07, 0x86, + 0x28, 0x0a, 0xce, 0xee, 0xb0, 0x08, 0x47, 0xc9, 0xa3, 0x1f, 0x53, 0x60, 0xec, 0xae, 0x6d, 0x69, + 0xa6, 0xb0, 0xb3, 0x16, 0xcf, 0x34, 0x1f, 0x38, 0x9e, 0x79, 0x09, 0xc8, 0x70, 0xdb, 0x4a, 0xb9, + 0x04, 0x47, 0xba, 0xc1, 0x32, 0xe6, 0x33, 0xdf, 0xea, 0xc1, 0x50, 0x9e, 0x96, 0xfd, 0xaa, 0x67, + 0x9f, 0x80, 0xc9, 0xf8, 0x1e, 0x3c, 0xd4, 0xf7, 0xb4, 0x0e, 0x0f, 0x1c, 0xa2, 0xbf, 0x5d, 0x1e, + 0x7d, 0xea, 0x6f, 0xf6, 0x47, 0x8e, 0xd2, 0xcc, 0xed, 0xd1, 0x85, 0xae, 0xfc, 0xb5, 0xd8, 0xc7, + 0xcc, 0x59, 0xb4, 0x71, 0x5c, 0x1b, 0xf6, 0x68, 0xbf, 0xe8, 0xbe, 0x93, 0xfb, 0xa2, 0x5f, 0x6f, + 0x9f, 0x54, 0xcf, 0x3b, 0x73, 0x81, 0xeb, 0x02, 0xc4, 0x82, 0x85, 0xc9, 0x9a, 0x58, 0xa0, 0x4e, + 0xc3, 0x35, 0xfc, 0x70, 0xd3, 0x92, 0x48, 0xf7, 0x0c, 0x2f, 0xc6, 0x3e, 0x5c, 0x5d, 0x8e, 0x1c, + 0x16, 0xeb, 0x76, 0xd3, 0x36, 0xed, 0xfa, 0xde, 0xfc, 0x1d, 0xcd, 0x21, 0xd8, 0x6e, 0x79, 0x02, + 0x5b, 0xb7, 0x02, 0xe2, 0x47, 0x07, 0xe0, 0x8a, 0x84, 0x2e, 0x35, 0x2a, 0xe7, 0x61, 0x7c, 0x15, + 0x5e, 0x8b, 0x07, 0xcf, 0x2c, 0xe4, 0x37, 0x32, 0x3d, 0xa8, 0x5f, 0xf9, 0x82, 0x69, 0xa2, 0xb6, + 0x02, 0xd0, 0xd0, 0x76, 0x45, 0x24, 0x37, 0x71, 0x5a, 0xd4, 0x8e, 0xa5, 0x83, 0x2b, 0x01, 0x19, + 0xde, 0xbd, 0xe7, 0x7c, 0x3d, 0x67, 0x08, 0x38, 0x22, 0x09, 0x50, 0x1a, 0x51, 0xcf, 0xd1, 0x42, + 0x67, 0x1b, 0x70, 0x2a, 0xd6, 0xf3, 0x63, 0x7d, 0x67, 0xfc, 0xf2, 0x90, 0x7f, 0x41, 0x16, 0x21, + 0xe6, 0x7e, 0x59, 0x81, 0xf3, 0x24, 0x4b, 0xe0, 0x14, 0xb7, 0xe5, 0xe7, 0x8e, 0x4b, 0xa0, 0x15, + 0x09, 0xb7, 0xb2, 0xc0, 0x38, 0xbb, 0x67, 0x68, 0x0f, 0xc0, 0x0d, 0xbe, 0xe9, 0x5e, 0xe2, 0x56, + 0xa4, 0x32, 0x09, 0xae, 0x87, 0x09, 0x7f, 0x63, 0x89, 0x18, 0xfa, 0x69, 0x05, 0xa6, 0xcd, 0x14, + 0x0e, 0x27, 0x38, 0x66, 0xf5, 0x18, 0xce, 0x16, 0x6e, 0xa1, 0x92, 0x06, 0xc1, 0xa9, 0x5d, 0x41, + 0x3f, 0x9b, 0x19, 0x62, 0x98, 0x4b, 0x26, 0xeb, 0xc7, 0xf1, 0x0d, 0xe6, 0x88, 0x36, 0xfc, 0x19, + 0x05, 0x50, 0x2d, 0x71, 0xf9, 0x16, 0xc6, 0x89, 0x4f, 0x1f, 0xb9, 0x8a, 0x81, 0x9b, 0x18, 0x25, + 0xcb, 0x71, 0x4a, 0x27, 0xd8, 0x3a, 0x7b, 0x29, 0x3c, 0x5f, 0xb8, 0x00, 0xf7, 0xba, 0xce, 0x69, + 0xc7, 0x09, 0x5f, 0xe7, 0x34, 0x08, 0x4e, 0xed, 0x8a, 0xfa, 0xdb, 0x23, 0x5c, 0x17, 0xcc, 0x6c, + 0x40, 0x5e, 0x84, 0xc1, 0x4d, 0x4d, 0xbf, 0x2d, 0xde, 0x5f, 0x73, 0x5a, 0x25, 0x2d, 0x30, 0x0c, + 0x5c, 0x0b, 0xc3, 0xff, 0xc7, 0x02, 0x2b, 0x7a, 0x1e, 0xfa, 0x6a, 0x96, 0xef, 0x61, 0xff, 0xde, + 0x1e, 0xde, 0x17, 0xc2, 0x38, 0x25, 0xe5, 0xd5, 0x2a, 0xa6, 0x48, 0x91, 0x05, 0xc3, 0x96, 0x50, + 0x9d, 0x0a, 0xed, 0xd6, 0x53, 0x79, 0x09, 0x04, 0x2a, 0xd8, 0x40, 0xf1, 0xeb, 0x97, 0xe0, 0x80, + 0x06, 0xa5, 0x17, 0x7b, 0xdf, 0xcc, 0x4d, 0x2f, 0x78, 0x11, 0x92, 0x03, 0xb8, 0xc6, 0x5f, 0x85, + 0x08, 0x0c, 0x7a, 0x9a, 0x61, 0x79, 0xbe, 0xb7, 0xfc, 0xe3, 0x79, 0xa9, 0xad, 0x53, 0x2c, 0xa1, + 0x86, 0x94, 0xfd, 0x74, 0xb1, 0x40, 0x8e, 0x36, 0x61, 0x90, 0x7b, 0xcc, 0x8b, 0x4f, 0x28, 0xf7, + 0xfb, 0x1a, 0x77, 0xc2, 0xe7, 0xdb, 0x80, 0xff, 0x8f, 0x05, 0x66, 0xf4, 0x12, 0x0c, 0xbb, 0xbe, + 0x39, 0xda, 0x70, 0x6f, 0x53, 0x17, 0xd8, 0xa2, 0x09, 0xff, 0x5f, 0x61, 0x84, 0x16, 0xe0, 0x47, + 0x9b, 0x30, 0x64, 0x88, 0x2b, 0xd5, 0x48, 0xfe, 0x6d, 0xe7, 0x5f, 0x97, 0xd8, 0x6d, 0xc3, 0xbf, + 0x29, 0xf9, 0x88, 0xb3, 0x6c, 0x4e, 0xe0, 0x1b, 0x68, 0x73, 0x82, 0x5e, 0x06, 0x20, 0xbe, 0x0a, + 0xdd, 0x9d, 0x19, 0xcd, 0xbf, 0x65, 0x24, 0x45, 0xbc, 0x2f, 0x10, 0x05, 0x45, 0x2e, 0x96, 0x88, + 0xa0, 0x57, 0x64, 0x9d, 0xdd, 0x58, 0x6f, 0x41, 0x40, 0x92, 0x61, 0x6d, 0xc2, 0xa7, 0xbb, 0x40, + 0x5d, 0x24, 0xa9, 0xd2, 0xd4, 0x2f, 0x03, 0x40, 0xf8, 0xac, 0x8b, 0xb6, 0x60, 0xd8, 0xa7, 0xd2, + 0x4b, 0x18, 0x9e, 0xeb, 0x02, 0xcc, 0xb7, 0x97, 0xff, 0x0b, 0x07, 0xb8, 0x51, 0x29, 0x2d, 0x1c, + 0x54, 0x98, 0x21, 0xb7, 0xbb, 0x50, 0x50, 0x2f, 0x03, 0xe8, 0x61, 0x90, 0xd7, 0xbe, 0xfc, 0x6b, + 0x15, 0x04, 0x80, 0x0d, 0xd7, 0x4a, 0x8a, 0x11, 0x2b, 0x11, 0xc9, 0xb0, 0x4a, 0xef, 0xcf, 0x65, + 0x95, 0xfe, 0x38, 0x9c, 0x12, 0x56, 0x80, 0x15, 0xf6, 0xca, 0xe8, 0xed, 0x09, 0xc7, 0x4d, 0x66, + 0x1f, 0x5a, 0x8a, 0x82, 0x70, 0xbc, 0x2e, 0xfa, 0x97, 0x0a, 0x0c, 0xeb, 0x42, 0x40, 0x13, 0xbc, + 0x6d, 0xb9, 0xb7, 0x47, 0xfd, 0x39, 0x5f, 0xde, 0xe3, 0x82, 0xfc, 0x33, 0x3e, 0x57, 0xf5, 0x8b, + 0x8f, 0x48, 0x8c, 0x0f, 0x7a, 0x8d, 0x7e, 0x5f, 0x81, 0x51, 0xcd, 0x34, 0x6d, 0x5d, 0xf3, 0x58, + 0x78, 0x40, 0xee, 0x51, 0x7a, 0xab, 0xc7, 0x51, 0xcc, 0x87, 0x18, 0xf9, 0x40, 0xde, 0x1f, 0x68, + 0x02, 0x42, 0xc8, 0x11, 0x8d, 0x45, 0xee, 0x3e, 0xfa, 0xff, 0x14, 0x78, 0x0b, 0x77, 0xe3, 0x2d, + 0x51, 0x99, 0x6b, 0xcb, 0xd0, 0x35, 0x8f, 0xf0, 0x98, 0xa5, 0xbe, 0x0b, 0x19, 0xb7, 0xa1, 0x1f, + 0x3e, 0xb4, 0x0d, 0xfd, 0xd5, 0xfd, 0x76, 0xf1, 0x2d, 0xa5, 0x2e, 0x70, 0xe3, 0xae, 0x7a, 0x80, + 0xee, 0xc2, 0xb8, 0x29, 0xc7, 0x1f, 0x17, 0x4c, 0x3e, 0xd7, 0xf3, 0x6b, 0x24, 0x90, 0x39, 0x57, + 0x30, 0x44, 0x8a, 0x70, 0x94, 0xd4, 0xec, 0x6d, 0x18, 0x8f, 0x6c, 0xb4, 0x63, 0x55, 0x5c, 0x5b, + 0x30, 0x19, 0xdf, 0x0f, 0xc7, 0x7a, 0xcf, 0xbb, 0x09, 0x23, 0x81, 0xb0, 0x80, 0x2e, 0x49, 0x84, + 0x42, 0xd1, 0xeb, 0x26, 0xd9, 0xe3, 0x54, 0x8b, 0x11, 0x1d, 0x0a, 0x57, 0x12, 0xb2, 0xe8, 0x8d, + 0x02, 0xa1, 0xfa, 0x87, 0xe2, 0x55, 0x75, 0x9d, 0x34, 0x9a, 0xa6, 0xe6, 0x91, 0x37, 0xbe, 0x1d, + 0x92, 0xfa, 0x1f, 0x14, 0x7e, 0xde, 0x70, 0xd1, 0x06, 0x69, 0x30, 0xda, 0xe0, 0xc9, 0xef, 0x58, + 0xac, 0x47, 0x25, 0x7f, 0x94, 0xc9, 0x95, 0x10, 0x0d, 0x96, 0x71, 0xa2, 0x3b, 0x30, 0xe2, 0x0b, + 0x83, 0xbe, 0xa6, 0x66, 0xa9, 0x37, 0xe1, 0x2c, 0x90, 0x3b, 0x83, 0xa3, 0xd5, 0x2f, 0x71, 0x71, + 0x48, 0x4b, 0xd5, 0x00, 0x25, 0xdb, 0xa0, 0xfb, 0x43, 0x7f, 0x36, 0x25, 0x9a, 0xae, 0x26, 0xe1, + 0xd3, 0xe6, 0x2b, 0x59, 0x0b, 0x59, 0x4a, 0x56, 0xf5, 0xb7, 0x0a, 0x30, 0x2d, 0xae, 0x9e, 0xf3, + 0xba, 0x6e, 0xb7, 0x2c, 0x2f, 0x74, 0x33, 0xe6, 0xbe, 0xfb, 0x82, 0x08, 0x13, 0x27, 0xb9, 0x63, + 0x3f, 0x16, 0x10, 0x74, 0x8b, 0x6b, 0x3f, 0xad, 0xda, 0xba, 0x7d, 0x9b, 0x58, 0x21, 0x97, 0x90, + 0x83, 0x64, 0x2c, 0xa6, 0x55, 0xc0, 0xe9, 0xed, 0xd0, 0x0e, 0xa0, 0x86, 0xb6, 0x1b, 0xc7, 0x96, + 0x2f, 0x09, 0x1a, 0xbb, 0x2f, 0xae, 0x24, 0xb0, 0xe1, 0x14, 0x0a, 0xf4, 0x20, 0xa5, 0x92, 0x5c, + 0xd3, 0x23, 0x35, 0x3e, 0x44, 0xdf, 0xa8, 0x83, 0x1d, 0xa4, 0xf3, 0x51, 0x10, 0x8e, 0xd7, 0x55, + 0x3f, 0x3e, 0x08, 0xe7, 0xa3, 0x93, 0x48, 0xbf, 0x50, 0xdf, 0xbd, 0xfe, 0x49, 0xdf, 0x77, 0x8c, + 0x4f, 0xe4, 0xfd, 0x71, 0xdf, 0xb1, 0x19, 0xd9, 0x68, 0x48, 0x34, 0x8a, 0xf8, 0x91, 0x7d, 0x03, + 0x7c, 0xe5, 0x33, 0x9c, 0xa3, 0xfb, 0x8e, 0x35, 0x26, 0xc0, 0xab, 0x0a, 0xcc, 0x46, 0x8b, 0x97, + 0x0c, 0xcb, 0x70, 0xb7, 0x45, 0xde, 0x93, 0xc3, 0xbb, 0xae, 0xb1, 0xcc, 0xc9, 0xcb, 0x99, 0x18, + 0x71, 0x07, 0x6a, 0xe8, 0x07, 0x14, 0xb8, 0x10, 0x9b, 0x97, 0x48, 0x16, 0x96, 0xc3, 0x7b, 0xb1, + 0xb1, 0xe0, 0x2e, 0xcb, 0xd9, 0x28, 0x71, 0x27, 0x7a, 0x9d, 0x22, 0x7e, 0x0e, 0xbe, 0x5e, 0x22, + 0x7e, 0xfe, 0x52, 0x01, 0x06, 0x98, 0xdd, 0xd4, 0x1b, 0xc3, 0xe1, 0x88, 0x75, 0x35, 0xd3, 0xde, + 0xb5, 0x1e, 0xb3, 0x77, 0x7d, 0x32, 0x3f, 0x89, 0xce, 0x06, 0xaf, 0xef, 0x87, 0xb3, 0xac, 0xda, + 0x7c, 0x8d, 0x29, 0xda, 0x5c, 0x52, 0x9b, 0xaf, 0xd5, 0xd8, 0xf5, 0xf6, 0xe0, 0x37, 0xb2, 0x4b, + 0xd0, 0xd7, 0x72, 0xcc, 0x78, 0x08, 0xd9, 0x0d, 0xbc, 0x8c, 0x69, 0xb9, 0xfa, 0xaa, 0x02, 0x93, + 0x0c, 0xb7, 0xc4, 0x62, 0xd0, 0x0e, 0x0c, 0x3b, 0x82, 0xcd, 0x88, 0xb5, 0x59, 0xce, 0x3d, 0xb4, + 0x14, 0xd6, 0xc5, 0x6f, 0x6c, 0xfe, 0x2f, 0x1c, 0xd0, 0x52, 0xff, 0x6c, 0x00, 0x66, 0xb2, 0x1a, + 0xa1, 0x1f, 0x51, 0xe0, 0xac, 0x1e, 0x4a, 0x9c, 0xf3, 0x2d, 0x6f, 0xdb, 0x76, 0xfc, 0x8c, 0x0a, + 0xb9, 0xd5, 0x21, 0xa5, 0xf9, 0xa0, 0x57, 0x2c, 0x8d, 0x48, 0x29, 0x95, 0x02, 0xce, 0xa0, 0x8c, + 0x5e, 0x01, 0x70, 0xdd, 0xed, 0x9b, 0x64, 0xaf, 0xa9, 0x19, 0xbe, 0xe5, 0xd6, 0xcd, 0xfc, 0xdb, + 0xa0, 0x7a, 0x43, 0xa0, 0x0a, 0x3a, 0xc5, 0x75, 0xd5, 0x61, 0xb9, 0x44, 0x0e, 0x7d, 0x4c, 0x81, + 0x71, 0x5b, 0x8e, 0xd7, 0xd2, 0x8b, 0x31, 0x7f, 0x6a, 0xe0, 0x17, 0x2e, 0x69, 0x47, 0x41, 0x51, + 0x92, 0x74, 0x59, 0xa6, 0xdc, 0xf8, 0xc9, 0x26, 0x78, 0xdf, 0x4a, 0x3e, 0x19, 0x28, 0xe3, 0x98, + 0xe4, 0xb7, 0xf6, 0x24, 0x38, 0x49, 0x9e, 0x75, 0x8a, 0x78, 0x7a, 0x6d, 0xd1, 0xd2, 0x9d, 0x3d, + 0x16, 0x21, 0x80, 0x76, 0x6a, 0x30, 0x7f, 0xa7, 0x16, 0xd7, 0x4b, 0xe5, 0x08, 0xb2, 0x68, 0xa7, + 0x92, 0xe0, 0x24, 0x79, 0xf5, 0xa3, 0x05, 0x38, 0xc7, 0x96, 0xf9, 0x66, 0x6b, 0x93, 0x70, 0xe3, + 0xab, 0x7f, 0x74, 0x01, 0x76, 0x7e, 0x57, 0x81, 0x11, 0x36, 0x07, 0x6f, 0x10, 0x1f, 0x4d, 0xd6, + 0xd7, 0x0c, 0x03, 0xe7, 0xdf, 0x51, 0x60, 0x2a, 0x91, 0x43, 0xaa, 0x2b, 0x0f, 0xbf, 0x13, 0xb3, + 0xbd, 0x7d, 0x6b, 0x98, 0xb7, 0xb2, 0x2f, 0x0c, 0x6c, 0x11, 0xcf, 0x59, 0xa9, 0x3e, 0x0b, 0xe3, + 0x11, 0xfb, 0x66, 0x29, 0x04, 0x64, 0x5a, 0xf0, 0x4a, 0x39, 0xc2, 0x63, 0xa1, 0x53, 0x6c, 0xca, + 0x70, 0xcb, 0x27, 0x39, 0xdb, 0x3f, 0x9e, 0x2d, 0x8f, 0xc4, 0x96, 0xf7, 0x9f, 0x71, 0x58, 0x60, + 0x49, 0xb7, 0x97, 0x67, 0x1c, 0x16, 0xa1, 0x52, 0x04, 0x82, 0xe5, 0xff, 0x63, 0x81, 0x15, 0x3d, + 0x15, 0x0d, 0xf3, 0xba, 0x1a, 0xde, 0xed, 0xa6, 0xe3, 0xc1, 0x59, 0xd9, 0x96, 0x4c, 0xd4, 0x46, + 0x98, 0x3f, 0x04, 0xf1, 0xb3, 0x2c, 0x57, 0x76, 0x9b, 0xf2, 0x6a, 0x95, 0xc7, 0xff, 0x0b, 0x1e, + 0x80, 0xa2, 0x1a, 0xef, 0xfe, 0x93, 0xd0, 0x78, 0x3b, 0x30, 0xba, 0x6d, 0x6c, 0x12, 0xc7, 0xe2, + 0x62, 0xcc, 0x40, 0x7e, 0x09, 0xed, 0x46, 0x88, 0x86, 0xab, 0x01, 0xa4, 0x02, 0x2c, 0x13, 0x41, + 0x4e, 0x24, 0x58, 0xf5, 0x60, 0x7e, 0xa9, 0x24, 0x54, 0x4d, 0x87, 0xe3, 0xcc, 0x08, 0x54, 0x6d, + 0x01, 0x58, 0x41, 0x04, 0xd7, 0x5e, 0x1e, 0x86, 0xc2, 0x38, 0xb0, 0x5c, 0xe8, 0x08, 0x7f, 0x63, + 0x89, 0x02, 0x9d, 0xd7, 0x46, 0x98, 0xd0, 0x40, 0xa8, 0x19, 0x9f, 0xec, 0x31, 0xa9, 0x84, 0x50, + 0xaf, 0x84, 0x05, 0x58, 0x26, 0x42, 0xc7, 0xd8, 0x08, 0xc2, 0xf8, 0x0b, 0x35, 0x62, 0xae, 0x31, + 0x86, 0xc9, 0x00, 0xf8, 0x18, 0xc3, 0xdf, 0x58, 0xa2, 0x80, 0x5e, 0x92, 0xde, 0x0f, 0x21, 0xbf, + 0x92, 0xaa, 0xab, 0xb7, 0xc3, 0x77, 0x85, 0xba, 0x9a, 0x51, 0xf6, 0x9d, 0x5e, 0x90, 0xf4, 0x34, + 0x2c, 0xbd, 0x01, 0xe5, 0x1d, 0x09, 0xbd, 0x4d, 0xe8, 0x55, 0x31, 0xd6, 0xd1, 0xab, 0xa2, 0x44, + 0xa5, 0x33, 0xc9, 0x33, 0x91, 0x31, 0x84, 0xf1, 0xf0, 0x11, 0xa4, 0x1a, 0x07, 0xe2, 0x64, 0x7d, + 0xce, 0xf0, 0x49, 0x8d, 0xb5, 0x9d, 0x90, 0x19, 0x3e, 0x2f, 0xc3, 0x01, 0x14, 0xed, 0xc0, 0x98, + 0x2b, 0xb9, 0x68, 0xcc, 0x9c, 0xea, 0xf5, 0x09, 0x51, 0xb8, 0x67, 0x30, 0x5b, 0x48, 0xb9, 0x04, + 0x47, 0xe8, 0x44, 0xdf, 0xb7, 0x26, 0x4f, 0xf6, 0x7d, 0x0b, 0xb5, 0xa2, 0xd6, 0xd7, 0x53, 0x47, + 0x12, 0xc7, 0xe5, 0x40, 0xeb, 0x6c, 0xba, 0xb4, 0x64, 0xb7, 0x69, 0xbb, 0x2d, 0x87, 0x30, 0xb3, + 0x4f, 0xb6, 0x3c, 0x28, 0x5c, 0xda, 0xc5, 0x38, 0x10, 0x27, 0xeb, 0xa3, 0xef, 0x53, 0x60, 0xd2, + 0xdd, 0x73, 0x3d, 0xd2, 0xa0, 0xc7, 0x96, 0x6d, 0x11, 0xcb, 0x73, 0x67, 0x4e, 0xe7, 0x8f, 0x0b, + 0x5e, 0x8d, 0xe1, 0xe2, 0xc7, 0x4e, 0xbc, 0x14, 0x27, 0x68, 0xd2, 0x9d, 0x23, 0x47, 0x82, 0x99, + 0x99, 0xce, 0xbf, 0x73, 0xe4, 0x28, 0x33, 0x7c, 0xe7, 0xc8, 0x25, 0x38, 0x42, 0x07, 0x3d, 0x02, + 0xe3, 0xae, 0x9f, 0xc3, 0x9c, 0xcd, 0xe0, 0x99, 0x30, 0x58, 0x67, 0x55, 0x06, 0xe0, 0x68, 0x3d, + 0xf4, 0x61, 0x18, 0x93, 0xcf, 0xce, 0x99, 0xb3, 0x47, 0x1d, 0xb6, 0x9e, 0xf7, 0x5c, 0x06, 0x45, + 0x08, 0x22, 0x0c, 0x67, 0x25, 0xe7, 0x41, 0xf9, 0xfb, 0x3e, 0xc7, 0x86, 0xc0, 0xef, 0xb3, 0xa9, + 0x35, 0x70, 0x46, 0x4b, 0xf4, 0x13, 0xe9, 0xcf, 0xe5, 0x33, 0x6c, 0x4b, 0xaf, 0x1d, 0xc9, 0x73, + 0xf9, 0xb3, 0x86, 0xb7, 0x7d, 0xab, 0xc9, 0x23, 0x8b, 0x1d, 0xf2, 0xe5, 0x5c, 0xfd, 0x63, 0x05, + 0x20, 0x50, 0x98, 0x9c, 0xc4, 0x53, 0x45, 0x2d, 0xa2, 0x43, 0x5a, 0xe8, 0x49, 0xc1, 0x93, 0x99, + 0x95, 0x44, 0xfd, 0x8a, 0x02, 0x13, 0x61, 0xb5, 0x13, 0xb8, 0x1a, 0xe9, 0xd1, 0xab, 0xd1, 0x13, + 0xbd, 0x8d, 0x2b, 0xe3, 0x7e, 0xf4, 0x3f, 0x0b, 0xf2, 0xa8, 0x98, 0xf4, 0xbb, 0x13, 0x79, 0xfa, + 0xa7, 0xa4, 0x6f, 0xf4, 0xf2, 0xf4, 0x2f, 0x47, 0xdf, 0x08, 0xc7, 0x9b, 0x62, 0x0a, 0xf0, 0xdd, + 0x11, 0xf9, 0xb3, 0x87, 0xb8, 0x37, 0x81, 0xb0, 0xe9, 0x93, 0xe6, 0x13, 0x70, 0x90, 0x30, 0xfa, + 0xb2, 0x7c, 0x3c, 0xf5, 0x90, 0x49, 0x24, 0x32, 0xe0, 0xce, 0x46, 0x17, 0xbf, 0x34, 0x05, 0xa3, + 0x92, 0x6e, 0x31, 0x66, 0xc8, 0xa0, 0x9c, 0x84, 0x21, 0x83, 0x07, 0xa3, 0x7a, 0x90, 0x4a, 0xd2, + 0x9f, 0xf6, 0x1e, 0x69, 0x06, 0xc7, 0x62, 0x98, 0xa4, 0xd2, 0xc5, 0x32, 0x19, 0x2a, 0xbc, 0x05, + 0x7b, 0xac, 0xef, 0x08, 0xcc, 0x4b, 0x3a, 0xed, 0xab, 0x87, 0x01, 0x7c, 0xf9, 0x9f, 0xd4, 0x44, + 0x84, 0xf6, 0xc0, 0x74, 0xba, 0xe2, 0xde, 0x08, 0x60, 0x58, 0xaa, 0x97, 0x7c, 0x18, 0x1f, 0x38, + 0xb1, 0x87, 0x71, 0xba, 0x0d, 0x4c, 0x3f, 0xe9, 0x7e, 0x4f, 0xe6, 0x6a, 0x41, 0xea, 0xfe, 0x70, + 0x1b, 0x04, 0x45, 0x2e, 0x96, 0x88, 0x64, 0xd8, 0xb3, 0x0c, 0xe5, 0xb2, 0x67, 0x69, 0xc1, 0x69, + 0x87, 0x78, 0xce, 0x5e, 0x69, 0x4f, 0x67, 0xa9, 0x4d, 0x1c, 0x8f, 0xdd, 0xe0, 0x87, 0xf3, 0x05, + 0x4c, 0xc4, 0x49, 0x54, 0x38, 0x0d, 0x7f, 0x44, 0x00, 0x1e, 0xe9, 0x28, 0x00, 0xbf, 0x0b, 0x46, + 0x3d, 0xa2, 0x6f, 0x5b, 0x86, 0xae, 0x99, 0x95, 0xb2, 0x88, 0xdf, 0x1d, 0xca, 0x72, 0x21, 0x08, + 0xcb, 0xf5, 0xd0, 0x02, 0xf4, 0xb5, 0x8c, 0x9a, 0xb8, 0x01, 0x7c, 0x5b, 0xa0, 0xa5, 0xaf, 0x94, + 0xef, 0xb5, 0x8b, 0x6f, 0x0e, 0x0d, 0x44, 0x82, 0x51, 0x5d, 0x6b, 0xde, 0xae, 0x5f, 0xf3, 0xf6, + 0x9a, 0xc4, 0x9d, 0xdb, 0xa8, 0x94, 0x31, 0x6d, 0x9c, 0x66, 0xeb, 0x33, 0x76, 0x08, 0x5b, 0x9f, + 0xcf, 0x28, 0x70, 0x5a, 0x8b, 0x3f, 0x30, 0x10, 0x77, 0x66, 0x3c, 0x3f, 0xb7, 0x4c, 0x7f, 0xb4, + 0x58, 0xb8, 0x20, 0xc6, 0x77, 0x7a, 0x3e, 0x49, 0x0e, 0xa7, 0xf5, 0x01, 0x39, 0x80, 0x1a, 0x46, + 0x3d, 0xc8, 0x7f, 0x2f, 0x56, 0x7d, 0x22, 0x9f, 0xde, 0x66, 0x25, 0x81, 0x09, 0xa7, 0x60, 0x47, + 0x77, 0x60, 0x54, 0x12, 0x92, 0xc4, 0x4d, 0xa6, 0x7c, 0x14, 0xef, 0x20, 0xfc, 0xb6, 0x2b, 0xbf, + 0x71, 0xc8, 0x94, 0x82, 0x47, 0x4e, 0x49, 0xcd, 0x20, 0x1e, 0xfa, 0xd8, 0xa8, 0x27, 0xf3, 0x3f, + 0x72, 0xa6, 0x63, 0xc4, 0x1d, 0xa8, 0xb1, 0x30, 0x85, 0x14, 0x2c, 0xdd, 0xcd, 0x67, 0xa6, 0xf2, + 0xc7, 0x59, 0x59, 0x8e, 0xa2, 0xe2, 0x5b, 0x33, 0x56, 0x88, 0xe3, 0x04, 0xd1, 0x12, 0x20, 0xc2, + 0x55, 0xe9, 0xe1, 0xe5, 0xcc, 0x9d, 0x41, 0xec, 0xfd, 0x9d, 0x2d, 0xe9, 0x62, 0x02, 0x8a, 0x53, + 0x5a, 0x20, 0x2f, 0xa2, 0x2b, 0xe9, 0xe1, 0x96, 0x13, 0xcf, 0x99, 0xd3, 0x51, 0x63, 0xf2, 0xbd, + 0x4a, 0x22, 0x89, 0x32, 0xbf, 0xdc, 0xdc, 0xe8, 0x3d, 0x89, 0xb2, 0x20, 0xdf, 0x4d, 0x2a, 0xe5, + 0xcf, 0x2b, 0x70, 0xae, 0x91, 0x9e, 0x2c, 0x92, 0xdd, 0x79, 0x72, 0x3e, 0x5c, 0x65, 0xe4, 0x9f, + 0x64, 0x6a, 0x8c, 0xac, 0xe4, 0x94, 0x38, 0xab, 0x23, 0xea, 0x1f, 0x29, 0x42, 0x15, 0x7d, 0x82, + 0xe6, 0x48, 0xc7, 0xfd, 0x4e, 0xac, 0x3e, 0x0b, 0x33, 0x55, 0x3f, 0xc8, 0x68, 0x2d, 0x96, 0x4b, + 0xe0, 0xbd, 0x30, 0xce, 0x9f, 0x82, 0x56, 0xb4, 0xe6, 0x6a, 0xf8, 0x6e, 0x10, 0x84, 0xa3, 0x28, + 0xc9, 0x40, 0x1c, 0xad, 0xab, 0x7e, 0x4d, 0x81, 0x73, 0x51, 0xcc, 0xb6, 0x63, 0xdc, 0xed, 0x1d, + 0x31, 0xfa, 0x84, 0x02, 0xa3, 0xb7, 0x83, 0x57, 0x2a, 0x5f, 0x74, 0xcb, 0xe5, 0xb2, 0xe1, 0xf7, + 0x8a, 0x38, 0xd2, 0xb3, 0x57, 0x32, 0x41, 0x65, 0x08, 0x74, 0xb1, 0x4c, 0x5a, 0xfd, 0x2f, 0x0a, + 0x24, 0xd4, 0x07, 0x68, 0x13, 0x86, 0x28, 0x91, 0xf2, 0x6a, 0x55, 0xec, 0x89, 0xf7, 0xe6, 0x93, + 0x2a, 0x19, 0x0a, 0xfe, 0x28, 0x22, 0x7e, 0x60, 0x1f, 0x31, 0xda, 0xe1, 0xfe, 0xff, 0x7e, 0xa2, + 0x21, 0xb1, 0x3d, 0x72, 0x89, 0xed, 0x72, 0xc2, 0x22, 0x7e, 0xad, 0x97, 0x4b, 0x70, 0x84, 0x8e, + 0xba, 0x0c, 0x10, 0xaa, 0x7c, 0x7a, 0x36, 0xef, 0xfb, 0xb7, 0xa7, 0xe1, 0x4c, 0xcf, 0xce, 0x88, + 0x1f, 0x53, 0xe0, 0x2c, 0xd9, 0x31, 0x74, 0x6f, 0x7e, 0xcb, 0x23, 0xce, 0xad, 0x5b, 0x2b, 0xeb, + 0xdb, 0x0e, 0x71, 0xb7, 0x6d, 0xb3, 0xd6, 0x8d, 0x31, 0x63, 0x8a, 0xe5, 0x15, 0x53, 0x4d, 0x2c, + 0xa6, 0x62, 0xc4, 0x19, 0x94, 0x98, 0xba, 0x6b, 0x87, 0x2b, 0x02, 0x30, 0xbd, 0x73, 0xb5, 0x1c, + 0xd7, 0x13, 0x71, 0x03, 0xb9, 0xba, 0x2b, 0x0e, 0xc4, 0xc9, 0xfa, 0x71, 0x24, 0x2c, 0x8b, 0x1e, + 0x93, 0xdb, 0x95, 0x24, 0x12, 0x06, 0xc4, 0xc9, 0xfa, 0x32, 0x12, 0xbe, 0x52, 0xf4, 0x50, 0x1c, + 0x48, 0x22, 0x09, 0x80, 0x38, 0x59, 0x1f, 0xd5, 0xe0, 0xa2, 0x43, 0x74, 0xbb, 0xd1, 0x20, 0x56, + 0x8d, 0x4d, 0xca, 0x8a, 0xe6, 0xd4, 0x0d, 0x6b, 0xc9, 0xd1, 0x58, 0x45, 0xf6, 0x7a, 0xa0, 0xb0, + 0x84, 0xb7, 0x17, 0x71, 0x87, 0x7a, 0xb8, 0x23, 0x16, 0xd4, 0x80, 0x53, 0x2d, 0xc6, 0xff, 0x1d, + 0x16, 0x57, 0x6c, 0x47, 0x33, 0xc5, 0x13, 0xc1, 0x61, 0x57, 0x8c, 0x1d, 0xd4, 0x1b, 0x51, 0x54, + 0x38, 0x8e, 0x1b, 0xed, 0x51, 0xf1, 0x5c, 0x74, 0x47, 0x22, 0x39, 0x9c, 0x8b, 0xa4, 0x10, 0xd1, + 0x13, 0xe8, 0x70, 0x1a, 0x0d, 0x54, 0x81, 0xd3, 0x9e, 0xe6, 0xd4, 0x89, 0x57, 0x5a, 0xdb, 0x58, + 0x23, 0x8e, 0x4e, 0x79, 0xac, 0xc9, 0xa5, 0x75, 0x85, 0xa3, 0x5a, 0x4f, 0x82, 0x71, 0x5a, 0x1b, + 0xf4, 0x61, 0x78, 0x6b, 0x74, 0x52, 0x97, 0xed, 0x3b, 0xc4, 0x59, 0xb0, 0x5b, 0x56, 0x2d, 0x8a, + 0x1c, 0x18, 0xf2, 0xfb, 0xf7, 0xdb, 0xc5, 0xb7, 0xe2, 0x6e, 0x1a, 0xe0, 0xee, 0xf0, 0x26, 0x3b, + 0xb0, 0xd1, 0x6c, 0xa6, 0x76, 0x60, 0x34, 0xab, 0x03, 0x19, 0x0d, 0x70, 0x77, 0x78, 0x11, 0x86, + 0xb3, 0x7c, 0x62, 0x78, 0x7a, 0x66, 0x89, 0xe2, 0x18, 0xa3, 0xc8, 0xbe, 0xdf, 0xf5, 0xd4, 0x1a, + 0x38, 0xa3, 0x25, 0x3d, 0x53, 0xae, 0x66, 0x0d, 0x3f, 0x41, 0x66, 0x9c, 0x91, 0x79, 0xfb, 0x7e, + 0xbb, 0x78, 0x15, 0x77, 0xd9, 0x06, 0x77, 0x8d, 0x3d, 0xa5, 0x2b, 0xe1, 0x44, 0x24, 0xba, 0x32, + 0x91, 0xd5, 0x95, 0xec, 0x36, 0xb8, 0x6b, 0xec, 0xe8, 0xfb, 0x15, 0x38, 0xaf, 0x37, 0x5b, 0x37, + 0x0c, 0xd7, 0xb3, 0xeb, 0x8e, 0xd6, 0x28, 0x13, 0x5d, 0xdb, 0xbb, 0xa1, 0x99, 0x5b, 0xcb, 0xc6, + 0x16, 0x11, 0x97, 0x8e, 0xc3, 0x7e, 0x38, 0xcc, 0xf1, 0xb6, 0xb4, 0xb6, 0x91, 0x8e, 0x14, 0x67, + 0xd3, 0x43, 0x3f, 0xaa, 0xc0, 0xc5, 0x06, 0xeb, 0x62, 0x46, 0x87, 0x26, 0x73, 0x75, 0x88, 0x71, + 0xb1, 0x95, 0x0e, 0x78, 0x71, 0x47, 0xaa, 0x6c, 0x92, 0x78, 0x85, 0xf9, 0x7a, 0xdd, 0x21, 0x75, + 0x86, 0x35, 0xe0, 0x2e, 0x53, 0xf9, 0x27, 0x69, 0x25, 0x0b, 0x29, 0xce, 0xa6, 0x87, 0x5e, 0x82, + 0xcb, 0x99, 0xc0, 0x92, 0xdd, 0xb2, 0x3c, 0xf6, 0x08, 0xd3, 0xb7, 0xa0, 0xee, 0xb7, 0x8b, 0x97, + 0x57, 0x3a, 0xd6, 0xc4, 0x07, 0x60, 0x62, 0xb1, 0x47, 0x22, 0x61, 0x00, 0x4e, 0x33, 0x49, 0xec, + 0x85, 0x5c, 0xe9, 0x78, 0x8f, 0xd0, 0xf7, 0xff, 0x2b, 0x51, 0xdf, 0xff, 0x69, 0xd6, 0xab, 0xf7, + 0x1f, 0x5d, 0xaf, 0xbe, 0xe9, 0xf0, 0xdf, 0x8d, 0x23, 0xc8, 0x67, 0x14, 0x10, 0xfe, 0x9b, 0xe8, + 0x62, 0xc4, 0xec, 0x68, 0x38, 0x66, 0x72, 0xe4, 0x67, 0xc5, 0x2d, 0xa4, 0x66, 0xc5, 0x7d, 0x9b, + 0x14, 0x84, 0x79, 0x24, 0xbc, 0x2c, 0x71, 0xcc, 0x61, 0x14, 0x66, 0xf4, 0x00, 0x8c, 0x04, 0x97, + 0x6a, 0xa1, 0xec, 0x64, 0x19, 0x69, 0xc2, 0xdb, 0x77, 0x08, 0x57, 0xff, 0x40, 0x01, 0x08, 0x93, + 0x3d, 0xa3, 0xfb, 0xfc, 0x60, 0x35, 0xbc, 0x83, 0xc1, 0x8b, 0x81, 0x1c, 0xb0, 0xe6, 0x60, 0x67, + 0x04, 0xa4, 0xc2, 0x60, 0x8b, 0xa5, 0xb7, 0x14, 0x0e, 0x04, 0xcc, 0x04, 0x66, 0x83, 0x95, 0x60, + 0x01, 0x41, 0x1b, 0x30, 0xd4, 0x30, 0x2c, 0xe6, 0xeb, 0xd1, 0x9f, 0xcb, 0xd7, 0x83, 0xdd, 0x07, + 0x56, 0x38, 0x0a, 0xec, 0xe3, 0x52, 0x7f, 0x59, 0x81, 0x53, 0xd1, 0xa8, 0xd8, 0x2c, 0xe4, 0x8d, + 0x08, 0x96, 0x2a, 0x82, 0xf1, 0xb3, 0xa6, 0x22, 0x08, 0x24, 0xf6, 0x61, 0xd1, 0xd7, 0xe9, 0x1e, + 0x5e, 0x1f, 0xd2, 0x83, 0x73, 0x1f, 0xf0, 0x10, 0xf0, 0xe3, 0xa7, 0x61, 0x90, 0x5f, 0xb4, 0xa9, + 0x1c, 0x9f, 0x12, 0xfc, 0xea, 0x66, 0xfe, 0x7c, 0x13, 0x79, 0x22, 0xe2, 0xc8, 0x59, 0x37, 0x0b, + 0x1d, 0xb3, 0x6e, 0x62, 0xe8, 0xd3, 0x1d, 0xa3, 0x17, 0x4b, 0xa4, 0x12, 0xae, 0x70, 0x4b, 0xa4, + 0x12, 0xae, 0x60, 0x8a, 0x0c, 0x79, 0x11, 0x13, 0x9d, 0xfe, 0xfc, 0x2a, 0x20, 0x3e, 0x01, 0x92, + 0xa1, 0xce, 0x44, 0x47, 0x23, 0x1d, 0x3f, 0xd2, 0xfe, 0x40, 0x7e, 0xe7, 0x20, 0x31, 0xe5, 0xdd, + 0x44, 0xda, 0xf7, 0x3f, 0xa4, 0xc1, 0xcc, 0x0f, 0x69, 0x0b, 0x86, 0xc4, 0xa7, 0x20, 0x2e, 0x04, + 0xef, 0xed, 0x21, 0xb7, 0xbc, 0x94, 0xc5, 0x8a, 0x17, 0x60, 0x1f, 0x39, 0xbd, 0x65, 0x36, 0xb4, + 0x5d, 0xa3, 0xd1, 0x6a, 0xb0, 0x5b, 0xc0, 0x80, 0x5c, 0x95, 0x15, 0x63, 0x1f, 0xce, 0xaa, 0x72, + 0x9f, 0x2a, 0x26, 0xb5, 0xcb, 0x55, 0x79, 0x31, 0xf6, 0xe1, 0xe8, 0x79, 0x18, 0x6e, 0x68, 0xbb, + 0xd5, 0x96, 0x53, 0x27, 0xc2, 0x40, 0x27, 0x5b, 0x29, 0xd4, 0xf2, 0x0c, 0x73, 0xce, 0xb0, 0x3c, + 0xd7, 0x73, 0xe6, 0x2a, 0x96, 0x77, 0xcb, 0xa9, 0x7a, 0xcc, 0x00, 0x88, 0xed, 0xba, 0x15, 0x81, + 0x05, 0x07, 0xf8, 0x90, 0x09, 0x13, 0x0d, 0x6d, 0x77, 0xc3, 0xd2, 0x78, 0x12, 0x05, 0x21, 0x65, + 0xe7, 0xa1, 0xc0, 0xb4, 0x72, 0x2b, 0x11, 0x5c, 0x38, 0x86, 0x3b, 0xc5, 0x18, 0x74, 0xec, 0xb8, + 0x8c, 0x41, 0xe7, 0x83, 0x28, 0x05, 0x5c, 0xa5, 0x7f, 0x3e, 0x35, 0x3e, 0x60, 0xc7, 0x08, 0x04, + 0x2f, 0x06, 0x11, 0x08, 0x26, 0xf2, 0x5b, 0x2f, 0x76, 0x88, 0x3e, 0xd0, 0x82, 0xd1, 0x9a, 0xe6, + 0x69, 0xbc, 0xd4, 0x9d, 0x39, 0x95, 0xff, 0x75, 0xba, 0x1c, 0xa0, 0x09, 0x59, 0x52, 0x58, 0xe6, + 0x62, 0x99, 0x0e, 0xba, 0x05, 0x67, 0xe8, 0xc7, 0x6a, 0x12, 0x2f, 0xac, 0xc2, 0x74, 0x66, 0x93, + 0xec, 0xfb, 0x61, 0x5e, 0x6a, 0x37, 0xd3, 0x2a, 0xe0, 0xf4, 0x76, 0x61, 0x2c, 0xdb, 0xa9, 0xf4, + 0x58, 0xb6, 0xe8, 0x53, 0x69, 0x66, 0x37, 0x88, 0xcd, 0xe9, 0xfb, 0xf2, 0xf3, 0x86, 0xdc, 0xc6, + 0x37, 0xff, 0x42, 0x81, 0x19, 0xb1, 0xcb, 0x84, 0xa9, 0x8c, 0x49, 0x9c, 0x15, 0xcd, 0xd2, 0xea, + 0xc4, 0x11, 0x7a, 0xf2, 0xf5, 0x1e, 0xf8, 0x43, 0x02, 0x67, 0x10, 0x1a, 0xe2, 0x2d, 0xfb, 0xed, + 0xe2, 0x95, 0x83, 0x6a, 0xe1, 0xcc, 0xbe, 0x21, 0x07, 0x86, 0xdc, 0x3d, 0x57, 0xf7, 0x4c, 0x57, + 0xc8, 0xa0, 0xd7, 0x7b, 0xe0, 0xac, 0x55, 0x8e, 0x89, 0xb3, 0xd6, 0x30, 0x77, 0x22, 0x2f, 0xc5, + 0x3e, 0x21, 0xf4, 0xc3, 0x0a, 0x4c, 0x89, 0xc7, 0x33, 0x29, 0xf4, 0xce, 0x99, 0xfc, 0x7e, 0x32, + 0xa5, 0x38, 0x32, 0xdf, 0x3c, 0x86, 0x69, 0x93, 0x12, 0x50, 0x9c, 0xa4, 0x4e, 0x0f, 0xd5, 0xa6, + 0x63, 0xd8, 0x8e, 0xe1, 0xed, 0x31, 0x43, 0xa4, 0x01, 0x3f, 0x82, 0x3d, 0x2f, 0xc3, 0x01, 0x14, + 0x55, 0x61, 0x82, 0x6b, 0x6d, 0xaa, 0x9e, 0xa3, 0x79, 0xa4, 0xbe, 0x27, 0xac, 0x85, 0x1e, 0x60, + 0x39, 0x86, 0x23, 0x90, 0x7b, 0xed, 0xe2, 0x19, 0xb1, 0x36, 0x51, 0x00, 0x8e, 0xa1, 0x40, 0x1f, + 0x8c, 0x19, 0x6f, 0xcd, 0xe4, 0xcf, 0x3f, 0xc8, 0xd7, 0xe2, 0x30, 0x26, 0x5c, 0xbd, 0x46, 0x93, + 0xeb, 0x21, 0x07, 0xcc, 0xec, 0x63, 0x30, 0x26, 0xef, 0x9a, 0x43, 0x05, 0xb1, 0xf3, 0x00, 0x25, + 0x07, 0x7b, 0xdc, 0x71, 0x7e, 0xd4, 0x9f, 0x51, 0x60, 0x32, 0x2e, 0xbb, 0xa0, 0x6d, 0x18, 0x12, + 0x8c, 0x4c, 0x50, 0x9d, 0xcf, 0x6b, 0xb5, 0x6c, 0x12, 0xe1, 0x1e, 0xcc, 0x45, 0x61, 0x51, 0x84, + 0x7d, 0xf4, 0xb2, 0x47, 0x42, 0xa1, 0x83, 0x47, 0xc2, 0xe3, 0x70, 0x36, 0x9d, 0xa5, 0xd1, 0x8b, + 0x84, 0x46, 0xef, 0x57, 0x42, 0x69, 0x1d, 0x5c, 0x24, 0xd8, 0xa5, 0x0b, 0x73, 0x98, 0xfa, 0x21, + 0x88, 0xe7, 0x3e, 0x43, 0x2f, 0xc1, 0x88, 0xeb, 0x6e, 0x73, 0x3b, 0x33, 0x31, 0xc8, 0x7c, 0x4f, + 0x3d, 0x7e, 0x1e, 0x1a, 0x7e, 0xf7, 0x09, 0x7e, 0xe2, 0x10, 0xfd, 0xc2, 0x73, 0x5f, 0xfa, 0xda, + 0xe5, 0x37, 0xfd, 0xe1, 0xd7, 0x2e, 0xbf, 0xe9, 0xab, 0x5f, 0xbb, 0xfc, 0xa6, 0x8f, 0xec, 0x5f, + 0x56, 0xbe, 0xb4, 0x7f, 0x59, 0xf9, 0xc3, 0xfd, 0xcb, 0xca, 0x57, 0xf7, 0x2f, 0x2b, 0x7f, 0xb5, + 0x7f, 0x59, 0xf9, 0xa1, 0x7f, 0x77, 0xf9, 0x4d, 0xcf, 0x3f, 0x14, 0x52, 0xbf, 0xe6, 0x13, 0x0d, + 0xff, 0x69, 0xde, 0xae, 0xd3, 0x4b, 0xab, 0xeb, 0xdf, 0x5a, 0x19, 0xf5, 0xff, 0x1d, 0x00, 0x00, + 0xff, 0xff, 0x89, 0x6e, 0x7a, 0xd1, 0x2e, 0x27, 0x01, 0x00, } func (m *APIServerLogging) Marshal() (dAtA []byte, err error) { @@ -8513,6 +8518,48 @@ func (m *ClusterAutoscaler) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.NodeGroupBackoffResetTimeout != nil { + { + size, err := m.NodeGroupBackoffResetTimeout.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xaa + } + if m.MaxNodeGroupBackoffDuration != nil { + { + size, err := m.MaxNodeGroupBackoffDuration.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa2 + } + if m.InitialNodeGroupBackoffDuration != nil { + { + size, err := m.InitialNodeGroupBackoffDuration.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x9a + } if m.MaxDrainParallelism != nil { i = encodeVarintGenerated(dAtA, i, uint64(*m.MaxDrainParallelism)) i-- @@ -19844,6 +19891,18 @@ func (m *ClusterAutoscaler) Size() (n int) { if m.MaxDrainParallelism != nil { n += 2 + sovGenerated(uint64(*m.MaxDrainParallelism)) } + if m.InitialNodeGroupBackoffDuration != nil { + l = m.InitialNodeGroupBackoffDuration.Size() + n += 2 + l + sovGenerated(uint64(l)) + } + if m.MaxNodeGroupBackoffDuration != nil { + l = m.MaxNodeGroupBackoffDuration.Size() + n += 2 + l + sovGenerated(uint64(l)) + } + if m.NodeGroupBackoffResetTimeout != nil { + l = m.NodeGroupBackoffResetTimeout.Size() + n += 2 + l + sovGenerated(uint64(l)) + } return n } @@ -24216,6 +24275,9 @@ func (this *ClusterAutoscaler) String() string { `StatusTaints:` + fmt.Sprintf("%v", this.StatusTaints) + `,`, `MaxScaleDownParallelism:` + valueToStringGenerated(this.MaxScaleDownParallelism) + `,`, `MaxDrainParallelism:` + valueToStringGenerated(this.MaxDrainParallelism) + `,`, + `InitialNodeGroupBackoffDuration:` + strings.Replace(fmt.Sprintf("%v", this.InitialNodeGroupBackoffDuration), "Duration", "v11.Duration", 1) + `,`, + `MaxNodeGroupBackoffDuration:` + strings.Replace(fmt.Sprintf("%v", this.MaxNodeGroupBackoffDuration), "Duration", "v11.Duration", 1) + `,`, + `NodeGroupBackoffResetTimeout:` + strings.Replace(fmt.Sprintf("%v", this.NodeGroupBackoffResetTimeout), "Duration", "v11.Duration", 1) + `,`, `}`, }, "") return s @@ -32105,6 +32167,114 @@ func (m *ClusterAutoscaler) Unmarshal(dAtA []byte) error { } } m.MaxDrainParallelism = &v + case 19: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InitialNodeGroupBackoffDuration", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.InitialNodeGroupBackoffDuration == nil { + m.InitialNodeGroupBackoffDuration = &v11.Duration{} + } + if err := m.InitialNodeGroupBackoffDuration.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 20: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxNodeGroupBackoffDuration", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.MaxNodeGroupBackoffDuration == nil { + m.MaxNodeGroupBackoffDuration = &v11.Duration{} + } + if err := m.MaxNodeGroupBackoffDuration.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 21: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeGroupBackoffResetTimeout", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.NodeGroupBackoffResetTimeout == nil { + m.NodeGroupBackoffResetTimeout = &v11.Duration{} + } + if err := m.NodeGroupBackoffResetTimeout.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/core/v1beta1/generated.proto b/pkg/apis/core/v1beta1/generated.proto index 658e7768912..a71d51c072f 100644 --- a/pkg/apis/core/v1beta1/generated.proto +++ b/pkg/apis/core/v1beta1/generated.proto @@ -560,6 +560,18 @@ message ClusterAutoscaler { // Default: 1 // +optional optional int32 maxDrainParallelism = 18; + + // InitialNodeGroupBackoffDuration is the duration of first backoff after a new node failed to start (default: 5m). + // +optional + optional .k8s.io.apimachinery.pkg.apis.meta.v1.Duration initialNodeGroupBackoffDuration = 19; + + // MaxNodeGroupBackoffDuration is the maximum backoff duration for a NodeGroup after new nodes failed to start (default: 30m). + // +optional + optional .k8s.io.apimachinery.pkg.apis.meta.v1.Duration maxNodeGroupBackoffDuration = 20; + + // NodeGroupBackoffResetTimeout is the time after last failed scale-up when the backoff duration is reset (default: 3h). + // +optional + optional .k8s.io.apimachinery.pkg.apis.meta.v1.Duration nodeGroupBackoffResetTimeout = 21; } // ClusterAutoscalerOptions contains the cluster autoscaler configurations for a worker pool. diff --git a/pkg/apis/core/v1beta1/types_shoot.go b/pkg/apis/core/v1beta1/types_shoot.go index a58e7c1cd91..aa8dd751b6c 100644 --- a/pkg/apis/core/v1beta1/types_shoot.go +++ b/pkg/apis/core/v1beta1/types_shoot.go @@ -734,6 +734,15 @@ type ClusterAutoscaler struct { // Default: 1 // +optional MaxDrainParallelism *int32 `json:"maxDrainParallelism,omitempty" protobuf:"varint,18,opt,name=maxDrainParallelism"` + // InitialNodeGroupBackoffDuration is the duration of first backoff after a new node failed to start (default: 5m). + // +optional + InitialNodeGroupBackoffDuration *metav1.Duration `json:"initialNodeGroupBackoffDuration,omitempty" protobuf:"bytes,19,opt,name=initialNodeGroupBackoffDuration"` + // MaxNodeGroupBackoffDuration is the maximum backoff duration for a NodeGroup after new nodes failed to start (default: 30m). + // +optional + MaxNodeGroupBackoffDuration *metav1.Duration `json:"maxNodeGroupBackoffDuration,omitempty" protobuf:"bytes,20,opt,name=maxNodeGroupBackoffDuration"` + // NodeGroupBackoffResetTimeout is the time after last failed scale-up when the backoff duration is reset (default: 3h). + // +optional + NodeGroupBackoffResetTimeout *metav1.Duration `json:"nodeGroupBackoffResetTimeout,omitempty" protobuf:"bytes,21,opt,name=nodeGroupBackoffResetTimeout"` } // ExpanderMode is type used for Expander values diff --git a/pkg/apis/core/v1beta1/zz_generated.conversion.go b/pkg/apis/core/v1beta1/zz_generated.conversion.go index 1db27e77ee2..c95a329cc51 100644 --- a/pkg/apis/core/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/core/v1beta1/zz_generated.conversion.go @@ -2855,6 +2855,9 @@ func autoConvert_v1beta1_ClusterAutoscaler_To_core_ClusterAutoscaler(in *Cluster out.StatusTaints = *(*[]string)(unsafe.Pointer(&in.StatusTaints)) out.MaxScaleDownParallelism = (*int32)(unsafe.Pointer(in.MaxScaleDownParallelism)) out.MaxDrainParallelism = (*int32)(unsafe.Pointer(in.MaxDrainParallelism)) + out.InitialNodeGroupBackoffDuration = (*metav1.Duration)(unsafe.Pointer(in.InitialNodeGroupBackoffDuration)) + out.MaxNodeGroupBackoffDuration = (*metav1.Duration)(unsafe.Pointer(in.MaxNodeGroupBackoffDuration)) + out.NodeGroupBackoffResetTimeout = (*metav1.Duration)(unsafe.Pointer(in.NodeGroupBackoffResetTimeout)) return nil } @@ -2872,6 +2875,9 @@ func autoConvert_core_ClusterAutoscaler_To_v1beta1_ClusterAutoscaler(in *core.Cl out.ScanInterval = (*metav1.Duration)(unsafe.Pointer(in.ScanInterval)) out.Expander = (*ExpanderMode)(unsafe.Pointer(in.Expander)) out.MaxNodeProvisionTime = (*metav1.Duration)(unsafe.Pointer(in.MaxNodeProvisionTime)) + out.InitialNodeGroupBackoffDuration = (*metav1.Duration)(unsafe.Pointer(in.InitialNodeGroupBackoffDuration)) + out.MaxNodeGroupBackoffDuration = (*metav1.Duration)(unsafe.Pointer(in.MaxNodeGroupBackoffDuration)) + out.NodeGroupBackoffResetTimeout = (*metav1.Duration)(unsafe.Pointer(in.NodeGroupBackoffResetTimeout)) out.MaxGracefulTerminationSeconds = (*int32)(unsafe.Pointer(in.MaxGracefulTerminationSeconds)) out.StartupTaints = *(*[]string)(unsafe.Pointer(&in.StartupTaints)) out.StatusTaints = *(*[]string)(unsafe.Pointer(&in.StatusTaints)) diff --git a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go index 5cfe903594a..f7c3b8e4a74 100644 --- a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go @@ -1023,6 +1023,21 @@ func (in *ClusterAutoscaler) DeepCopyInto(out *ClusterAutoscaler) { *out = new(int32) **out = **in } + if in.InitialNodeGroupBackoffDuration != nil { + in, out := &in.InitialNodeGroupBackoffDuration, &out.InitialNodeGroupBackoffDuration + *out = new(metav1.Duration) + **out = **in + } + if in.MaxNodeGroupBackoffDuration != nil { + in, out := &in.MaxNodeGroupBackoffDuration, &out.MaxNodeGroupBackoffDuration + *out = new(metav1.Duration) + **out = **in + } + if in.NodeGroupBackoffResetTimeout != nil { + in, out := &in.NodeGroupBackoffResetTimeout, &out.NodeGroupBackoffResetTimeout + *out = new(metav1.Duration) + **out = **in + } return } diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index 93f6b55d465..51673d7ca8c 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -1367,6 +1367,10 @@ func ValidateClusterAutoscaler(autoScaler core.ClusterAutoscaler, version string allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(*autoScaler.Verbosity), fldPath.Child("verbosity"))...) } + allErrs = append(allErrs, ValidatePositiveDuration(autoScaler.InitialNodeGroupBackoffDuration, fldPath.Child("initialNodeGroupBackoffDuration"))...) + allErrs = append(allErrs, ValidatePositiveDuration(autoScaler.MaxNodeGroupBackoffDuration, fldPath.Child("maxNodeGroupBackoffDuration"))...) + allErrs = append(allErrs, ValidatePositiveDuration(autoScaler.NodeGroupBackoffResetTimeout, fldPath.Child("nodeGroupBackoffResetTimeout"))...) + return allErrs } diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index c4cec75e980..c3a3ed5f07e 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -3742,6 +3742,36 @@ var _ = Describe("Shoot Validation Tests", func() { Entry("invalid with negative verbosity", core.ClusterAutoscaler{ Verbosity: &negativeInteger, }, version_1_32, ConsistOf(field.Invalid(field.NewPath("verbosity"), int64(negativeInteger), "must be greater than or equal to 0").WithOrigin("minimum"))), + Entry("valid with initialNodeGroupBackoffDuration", core.ClusterAutoscaler{ + InitialNodeGroupBackoffDuration: &metav1.Duration{Duration: time.Minute}, + }, version_1_32, BeEmpty()), + Entry("invalid with negative initialNodeGroupBackoffDuration", core.ClusterAutoscaler{ + InitialNodeGroupBackoffDuration: &negativeDuration, + }, version_1_32, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("initialNodeGroupBackoffDuration"), + "Detail": Equal("must be non-negative"), + })))), + Entry("valid with maxNodeGroupBackoffDuration", core.ClusterAutoscaler{ + MaxNodeGroupBackoffDuration: &metav1.Duration{Duration: time.Minute}, + }, version_1_32, BeEmpty()), + Entry("invalid with negative maxNodeGroupBackoffDuration", core.ClusterAutoscaler{ + MaxNodeGroupBackoffDuration: &negativeDuration, + }, version_1_32, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("maxNodeGroupBackoffDuration"), + "Detail": Equal("must be non-negative"), + })))), + Entry("valid with nodeGroupBackoffResetTimeout", core.ClusterAutoscaler{ + NodeGroupBackoffResetTimeout: &metav1.Duration{Duration: time.Minute}, + }, version_1_32, BeEmpty()), + Entry("invalid with negative nodeGroupBackoffResetTimeout", core.ClusterAutoscaler{ + NodeGroupBackoffResetTimeout: &negativeDuration, + }, version_1_32, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("nodeGroupBackoffResetTimeout"), + "Detail": Equal("must be non-negative"), + })))), ) Describe("taint validation", func() { diff --git a/pkg/apis/core/zz_generated.deepcopy.go b/pkg/apis/core/zz_generated.deepcopy.go index 7648fb2f518..43c931263cb 100644 --- a/pkg/apis/core/zz_generated.deepcopy.go +++ b/pkg/apis/core/zz_generated.deepcopy.go @@ -973,6 +973,21 @@ func (in *ClusterAutoscaler) DeepCopyInto(out *ClusterAutoscaler) { *out = new(metav1.Duration) **out = **in } + if in.InitialNodeGroupBackoffDuration != nil { + in, out := &in.InitialNodeGroupBackoffDuration, &out.InitialNodeGroupBackoffDuration + *out = new(metav1.Duration) + **out = **in + } + if in.MaxNodeGroupBackoffDuration != nil { + in, out := &in.MaxNodeGroupBackoffDuration, &out.MaxNodeGroupBackoffDuration + *out = new(metav1.Duration) + **out = **in + } + if in.NodeGroupBackoffResetTimeout != nil { + in, out := &in.NodeGroupBackoffResetTimeout, &out.NodeGroupBackoffResetTimeout + *out = new(metav1.Duration) + **out = **in + } if in.MaxGracefulTerminationSeconds != nil { in, out := &in.MaxGracefulTerminationSeconds, &out.MaxGracefulTerminationSeconds *out = new(int32) diff --git a/pkg/apiserver/openapi/openapi_generated.go b/pkg/apiserver/openapi/openapi_generated.go index f3848af0412..9d11d1fa681 100644 --- a/pkg/apiserver/openapi/openapi_generated.go +++ b/pkg/apiserver/openapi/openapi_generated.go @@ -2496,6 +2496,24 @@ func schema_pkg_apis_core_v1beta1_ClusterAutoscaler(ref common.ReferenceCallback Format: "int32", }, }, + "initialNodeGroupBackoffDuration": { + SchemaProps: spec.SchemaProps{ + Description: "InitialNodeGroupBackoffDuration is the duration of first backoff after a new node failed to start (default: 5m).", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + "maxNodeGroupBackoffDuration": { + SchemaProps: spec.SchemaProps{ + Description: "MaxNodeGroupBackoffDuration is the maximum backoff duration for a NodeGroup after new nodes failed to start (default: 30m).", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + "nodeGroupBackoffResetTimeout": { + SchemaProps: spec.SchemaProps{ + Description: "NodeGroupBackoffResetTimeout is the time after last failed scale-up when the backoff duration is reset (default: 3h).", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, }, }, }, diff --git a/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler.go b/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler.go index e97c277cfba..0c32d0859bb 100644 --- a/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler.go +++ b/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler.go @@ -479,6 +479,9 @@ func (c *clusterAutoscaler) computeCommand(workersHavePriorityConfigured bool) [ fmt.Sprintf("--expander=%s", expanderMode), fmt.Sprintf("--max-graceful-termination-sec=%d", *c.config.MaxGracefulTerminationSeconds), fmt.Sprintf("--max-node-provision-time=%s", c.config.MaxNodeProvisionTime.Duration), + fmt.Sprintf("--initial-node-group-backoff-duration=%s", c.config.InitialNodeGroupBackoffDuration.Duration), + fmt.Sprintf("--max-node-group-backoff-duration=%s", c.config.MaxNodeGroupBackoffDuration.Duration), + fmt.Sprintf("--node-group-backoff-reset-timeout=%s", c.config.NodeGroupBackoffResetTimeout.Duration), fmt.Sprintf("--scale-down-utilization-threshold=%f", *c.config.ScaleDownUtilizationThreshold), fmt.Sprintf("--scale-down-unneeded-time=%s", c.config.ScaleDownUnneededTime.Duration), fmt.Sprintf("--scale-down-delay-after-add=%s", c.config.ScaleDownDelayAfterAdd.Duration), diff --git a/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler_test.go b/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler_test.go index f6efefc5402..77632dab40b 100644 --- a/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler_test.go +++ b/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler_test.go @@ -122,38 +122,45 @@ var _ = Describe("ClusterAutoscaler", func() { configExpander = gardencorev1beta1.ClusterAutoscalerExpanderRandom configMaxGracefulTerminationSeconds int32 = 60 * 60 * 24 configMaxNodeProvisionTime = &metav1.Duration{Duration: time.Second} - configScaleDownDelayAfterAdd = &metav1.Duration{Duration: time.Second} - configScaleDownDelayAfterDelete = &metav1.Duration{Duration: time.Second} - configScaleDownDelayAfterFailure = &metav1.Duration{Duration: time.Second} - configScaleDownUnneededTime = &metav1.Duration{Duration: time.Second} - configScaleDownUtilizationThreshold = ptr.To(float64(1.2345)) - configScanInterval = &metav1.Duration{Duration: time.Second} - configIgnoreDaemonsetsUtilization = true - configVerbosity int32 = 4 - configMaxEmptyBulkDelete = ptr.To[int32](20) - configMaxScaleDownParallelism = ptr.To[int32](20) - configMaxDrainParallelism = ptr.To[int32](2) - configNewPodScaleUpDelay = &metav1.Duration{Duration: time.Second} - configTaints = []string{"taint-1", "taint-2"} - configFull = &gardencorev1beta1.ClusterAutoscaler{ - Expander: &configExpander, - MaxGracefulTerminationSeconds: &configMaxGracefulTerminationSeconds, - MaxNodeProvisionTime: configMaxNodeProvisionTime, - ScaleDownDelayAfterAdd: configScaleDownDelayAfterAdd, - ScaleDownDelayAfterDelete: configScaleDownDelayAfterDelete, - ScaleDownDelayAfterFailure: configScaleDownDelayAfterFailure, - ScaleDownUnneededTime: configScaleDownUnneededTime, - ScaleDownUtilizationThreshold: configScaleDownUtilizationThreshold, - ScanInterval: configScanInterval, - StartupTaints: configTaints, - StatusTaints: configTaints, - IgnoreTaints: configTaints, - IgnoreDaemonsetsUtilization: &configIgnoreDaemonsetsUtilization, - Verbosity: &configVerbosity, - MaxEmptyBulkDelete: configMaxEmptyBulkDelete, - MaxScaleDownParallelism: configMaxScaleDownParallelism, - MaxDrainParallelism: configMaxDrainParallelism, - NewPodScaleUpDelay: configNewPodScaleUpDelay, + // add the 3 new config options here + configInitialNodeGroupBackoffDuration = &metav1.Duration{Duration: 10 * time.Minute} + configMaxNodeGroupBackoffDuration = &metav1.Duration{Duration: 20 * time.Minute} + configNodeGroupBackoffResetTimeout = &metav1.Duration{Duration: time.Hour} + configScaleDownDelayAfterAdd = &metav1.Duration{Duration: time.Second} + configScaleDownDelayAfterDelete = &metav1.Duration{Duration: time.Second} + configScaleDownDelayAfterFailure = &metav1.Duration{Duration: time.Second} + configScaleDownUnneededTime = &metav1.Duration{Duration: time.Second} + configScaleDownUtilizationThreshold = ptr.To(float64(1.2345)) + configScanInterval = &metav1.Duration{Duration: time.Second} + configIgnoreDaemonsetsUtilization = true + configVerbosity int32 = 4 + configMaxEmptyBulkDelete = ptr.To[int32](20) + configMaxScaleDownParallelism = ptr.To[int32](20) + configMaxDrainParallelism = ptr.To[int32](2) + configNewPodScaleUpDelay = &metav1.Duration{Duration: time.Second} + configTaints = []string{"taint-1", "taint-2"} + configFull = &gardencorev1beta1.ClusterAutoscaler{ + Expander: &configExpander, + MaxGracefulTerminationSeconds: &configMaxGracefulTerminationSeconds, + MaxNodeProvisionTime: configMaxNodeProvisionTime, + InitialNodeGroupBackoffDuration: configInitialNodeGroupBackoffDuration, + MaxNodeGroupBackoffDuration: configMaxNodeGroupBackoffDuration, + NodeGroupBackoffResetTimeout: configNodeGroupBackoffResetTimeout, + ScaleDownDelayAfterAdd: configScaleDownDelayAfterAdd, + ScaleDownDelayAfterDelete: configScaleDownDelayAfterDelete, + ScaleDownDelayAfterFailure: configScaleDownDelayAfterFailure, + ScaleDownUnneededTime: configScaleDownUnneededTime, + ScaleDownUtilizationThreshold: configScaleDownUtilizationThreshold, + ScanInterval: configScanInterval, + StartupTaints: configTaints, + StatusTaints: configTaints, + IgnoreTaints: configTaints, + IgnoreDaemonsetsUtilization: &configIgnoreDaemonsetsUtilization, + Verbosity: &configVerbosity, + MaxEmptyBulkDelete: configMaxEmptyBulkDelete, + MaxScaleDownParallelism: configMaxScaleDownParallelism, + MaxDrainParallelism: configMaxDrainParallelism, + NewPodScaleUpDelay: configNewPodScaleUpDelay, } genericTokenKubeconfigSecretName = "generic-token-kubeconfig" @@ -297,6 +304,9 @@ var _ = Describe("ClusterAutoscaler", func() { "--expander=least-waste", "--max-graceful-termination-sec=600", "--max-node-provision-time=20m0s", + "--initial-node-group-backoff-duration=5m0s", + "--max-node-group-backoff-duration=30m0s", + "--node-group-backoff-reset-timeout=3h0m0s", "--scale-down-utilization-threshold=0.500000", "--scale-down-unneeded-time=30m0s", "--scale-down-delay-after-add=1h0m0s", @@ -315,6 +325,9 @@ var _ = Describe("ClusterAutoscaler", func() { "--expander="+expander, fmt.Sprintf("--max-graceful-termination-sec=%d", configMaxGracefulTerminationSeconds), fmt.Sprintf("--max-node-provision-time=%s", configMaxNodeProvisionTime.Duration), + fmt.Sprintf("--initial-node-group-backoff-duration=%s", configInitialNodeGroupBackoffDuration.Duration), + fmt.Sprintf("--max-node-group-backoff-duration=%s", configMaxNodeGroupBackoffDuration.Duration), + fmt.Sprintf("--node-group-backoff-reset-timeout=%s", configNodeGroupBackoffResetTimeout.Duration), fmt.Sprintf("--scale-down-utilization-threshold=%f", *configScaleDownUtilizationThreshold), fmt.Sprintf("--scale-down-unneeded-time=%s", configScaleDownUnneededTime.Duration), fmt.Sprintf("--scale-down-delay-after-add=%s", configScaleDownDelayAfterAdd.Duration), From ae45c36f885e302e509bd12a05b2a0e895708bd5 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 12 Nov 2025 14:09:02 +0100 Subject: [PATCH 091/176] Update quay.io/coreos/etcd Docker tag to v3.6.6 (#13423) --- .../charts/runtime/templates/apiserver/deployment.yaml | 2 +- example/gardener-local/etcd/statefulset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/gardener/controlplane/charts/runtime/templates/apiserver/deployment.yaml b/charts/gardener/controlplane/charts/runtime/templates/apiserver/deployment.yaml index 69c9c1dc74d..50f3a679ae8 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/apiserver/deployment.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/apiserver/deployment.yaml @@ -381,7 +381,7 @@ spec: {{- end }} {{- if .Values.global.apiserver.etcd.useSidecar }} - name: etcd - image: quay.io/coreos/etcd:v3.6.5 + image: quay.io/coreos/etcd:v3.6.6 imagePullPolicy: IfNotPresent command: - /usr/local/bin/etcd diff --git a/example/gardener-local/etcd/statefulset.yaml b/example/gardener-local/etcd/statefulset.yaml index 27fe835e368..70e5d8a042f 100644 --- a/example/gardener-local/etcd/statefulset.yaml +++ b/example/gardener-local/etcd/statefulset.yaml @@ -14,7 +14,7 @@ spec: app: etcd spec: containers: - - image: quay.io/coreos/etcd:v3.6.5 + - image: quay.io/coreos/etcd:v3.6.6 name: etcd command: - etcd From 4dbaf0c38a8f9408f91ab0e009b67a36956e0ce0 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 12 Nov 2025 14:09:09 +0100 Subject: [PATCH 092/176] Update dependency kubernetes/kubernetes to v1.34.2 (#13429) --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 02dd2c23ab7..01d8d73cd2c 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -69,7 +69,7 @@ HELM_VERSION ?= v3.19.1 # renovate: datasource=github-releases depName=kubernetes-sigs/kind KIND_VERSION ?= v0.30.0 # renovate: datasource=github-releases depName=kubernetes/kubernetes -KUBECTL_VERSION ?= v1.34.1 +KUBECTL_VERSION ?= v1.34.2 # renovate: datasource=github-releases depName=kubernetes-sigs/kustomize KUSTOMIZE_VERSION ?= v5.3.0 # renovate: datasource=github-releases depName=prometheus/prometheus From a9be1f43cc766d663260c5376658b4c4208aa1f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20G=C3=B6tz?= <47362717+oliver-goetz@users.noreply.github.com> Date: Wed, 12 Nov 2025 15:56:04 +0100 Subject: [PATCH 093/176] Improve autoscaling of `istio-ingressgateway` when `IstioTLSTermination` feature gate is enabled (#13370) * Remove `memory` resource from Istio HPA when IstioTLSTermination is enabled * Add VPA for `memory` with `updateMode` `Initial` to Istio when IstioTLSTermination is enabled * Increase the maximum number of `istio-ingressgateway` pods in case IstioTLSTermination feature gate is enabled * Address PR review feedback --- .../{autoscale.yaml => autoscale-hpa.yaml} | 2 ++ .../templates/autoscale-vpa.yaml | 23 ++++++++++++ pkg/component/networking/istio/istio_test.go | 17 ++++++++- ...ngress_autoscaler_tls_termination_hpa.yaml | 36 +++++++++++++++++++ ...ngress_autoscaler_tls_termination_vpa.yaml | 22 ++++++++++++ pkg/component/shared/istio.go | 10 ++++++ pkg/component/shared/istio_test.go | 17 +++++++++ 7 files changed, 126 insertions(+), 1 deletion(-) rename pkg/component/networking/istio/charts/istio/istio-ingress/templates/{autoscale.yaml => autoscale-hpa.yaml} (94%) create mode 100644 pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale-vpa.yaml create mode 100644 pkg/component/networking/istio/test_charts/ingress_autoscaler_tls_termination_hpa.yaml create mode 100644 pkg/component/networking/istio/test_charts/ingress_autoscaler_tls_termination_vpa.yaml diff --git a/pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale.yaml b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale-hpa.yaml similarity index 94% rename from pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale.yaml rename to pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale-hpa.yaml index d5c49428fb1..9ebad8001ee 100644 --- a/pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale.yaml +++ b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale-hpa.yaml @@ -20,12 +20,14 @@ spec: target: averageUtilization: 80 type: Utilization +{{- if eq .Values.terminateAPIServerTLS false }} - type: Resource resource: name: memory target: averageUtilization: 80 type: Utilization +{{- end }} behavior: scaleDown: stabilizationWindowSeconds: 1800 diff --git a/pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale-vpa.yaml b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale-vpa.yaml new file mode 100644 index 00000000000..26bc66d5d0a --- /dev/null +++ b/pkg/component/networking/istio/charts/istio/istio-ingress/templates/autoscale-vpa.yaml @@ -0,0 +1,23 @@ +{{- if eq .Values.terminateAPIServerTLS true }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: istio-ingressgateway + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/version: {{ .Values.ingressVersion }} +{{ .Values.labels | toYaml | indent 4 }} +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: istio-ingressgateway + updatePolicy: + updateMode: Initial + resourcePolicy: + containerPolicies: + - containerName: istio-proxy + controlledValues: RequestsOnly + controlledResources: + - memory +{{- end }} diff --git a/pkg/component/networking/istio/istio_test.go b/pkg/component/networking/istio/istio_test.go index a877a7f25b7..debb10ab87b 100644 --- a/pkg/component/networking/istio/istio_test.go +++ b/pkg/component/networking/istio/istio_test.go @@ -141,6 +141,18 @@ var _ = Describe("istiod", func() { return str } + istioIngressAutoscalerTLSTerminationHPA = func(min *int, max *int) string { + data, _ := os.ReadFile("./test_charts/ingress_autoscaler_tls_termination_hpa.yaml") + str := strings.ReplaceAll(string(data), "", strconv.Itoa(ptr.Deref(min, 2))) + str = strings.ReplaceAll(str, "", strconv.Itoa(ptr.Deref(max, 9))) + return str + } + + istioIngressAutoscalerTLSTerminationVPA = func() string { + data, _ := os.ReadFile("./test_charts/ingress_autoscaler_tls_termination_vpa.yaml") + return string(data) + } + istioIngressEnvoyVPNFilter = func(i int) string { data, _ := os.ReadFile("./test_charts/ingress_vpn_envoy_filter.yaml") return strings.Split(string(data), "---\n")[i] @@ -392,7 +404,6 @@ var _ = Describe("istiod", func() { expectedIstioManifests := []string{ istioIngressNamespace(), - istioIngressAutoscaler(minReplicas, maxReplicas), istioIngressRole(), istioIngressRoleBinding(), istioIngressService(), @@ -439,7 +450,11 @@ var _ = Describe("istiod", func() { if expectAPIServerTLSTermination { expectedIstioManifests = append(expectedIstioManifests, istioAPIServerTLSTerminationEnvoyFilter()) + expectedIstioManifests = append(expectedIstioManifests, istioIngressAutoscalerTLSTerminationHPA(minReplicas, maxReplicas)) + expectedIstioManifests = append(expectedIstioManifests, istioIngressAutoscalerTLSTerminationVPA()) expectedIstioManifests = append(expectedIstioManifests, istioStripTrailingDotEnvoyFilter()) + } else { + expectedIstioManifests = append(expectedIstioManifests, istioIngressAutoscaler(minReplicas, maxReplicas)) } if igw[0].TerminateLoadBalancerProxyProtocol { diff --git a/pkg/component/networking/istio/test_charts/ingress_autoscaler_tls_termination_hpa.yaml b/pkg/component/networking/istio/test_charts/ingress_autoscaler_tls_termination_hpa.yaml new file mode 100644 index 00000000000..13f4fbe2aa8 --- /dev/null +++ b/pkg/component/networking/istio/test_charts/ingress_autoscaler_tls_termination_hpa.yaml @@ -0,0 +1,36 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: istio-ingressgateway + namespace: test-ingress + labels: + app.kubernetes.io/version: 1.27.1 + app: istio-ingressgateway + foo: bar +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: istio-ingressgateway + minReplicas: + maxReplicas: + metrics: + - type: Resource + resource: + name: cpu + target: + averageUtilization: 80 + type: Utilization + behavior: + scaleDown: + stabilizationWindowSeconds: 1800 + policies: + - type: Pods + value: 1 + periodSeconds: 1800 + scaleUp: + stabilizationWindowSeconds: 60 + policies: + - type: Pods + value: 1 + periodSeconds: 60 diff --git a/pkg/component/networking/istio/test_charts/ingress_autoscaler_tls_termination_vpa.yaml b/pkg/component/networking/istio/test_charts/ingress_autoscaler_tls_termination_vpa.yaml new file mode 100644 index 00000000000..27246f1ef35 --- /dev/null +++ b/pkg/component/networking/istio/test_charts/ingress_autoscaler_tls_termination_vpa.yaml @@ -0,0 +1,22 @@ +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: istio-ingressgateway + namespace: test-ingress + labels: + app.kubernetes.io/version: 1.27.1 + app: istio-ingressgateway + foo: bar +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: istio-ingressgateway + updatePolicy: + updateMode: Initial + resourcePolicy: + containerPolicies: + - containerName: istio-proxy + controlledValues: RequestsOnly + controlledResources: + - memory diff --git a/pkg/component/shared/istio.go b/pkg/component/shared/istio.go index b3dd8b39c95..24156b94c52 100644 --- a/pkg/component/shared/istio.go +++ b/pkg/component/shared/istio.go @@ -25,6 +25,7 @@ import ( "github.com/gardener/gardener/pkg/component/networking/istio" "github.com/gardener/gardener/pkg/component/networking/nginxingress" vpnseedserver "github.com/gardener/gardener/pkg/component/networking/vpn/seedserver" + "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/utils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -81,6 +82,10 @@ func NewIstio( // replicas, which apparently works in all known Gardener scenarios. Reducing it to less // per zone gives some room for autoscaling while it is assumed to never reach the maximum. maxReplicas = ptr.To(len(zones) * 6) + if features.DefaultFeatureGate.Enabled(features.IstioTLSTermination) { + // When IstioTLSTermination server is enabled, more resources might be required on seeds. + maxReplicas = ptr.To(len(zones) * 8) + } } policyLabels := commonIstioIngressNetworkPolicyLabels(vpnEnabled) @@ -178,6 +183,11 @@ func AddIstioIngressGateway( } else { zones = []string{*zone} + if features.DefaultFeatureGate.Enabled(features.IstioTLSTermination) { + // When IstioTLSTermination server is enabled, more resources might be required on seeds. + maxReplicas = ptr.To(len(zones) * 12) + } + enforceSpreadAcrossHosts, err = ShouldEnforceSpreadAcrossHosts(ctx, cl, []string{*zone}) if err != nil { return err diff --git a/pkg/component/shared/istio_test.go b/pkg/component/shared/istio_test.go index 2ac77db6b8c..89b837731f7 100644 --- a/pkg/component/shared/istio_test.go +++ b/pkg/component/shared/istio_test.go @@ -24,6 +24,8 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component/networking/istio" . "github.com/gardener/gardener/pkg/component/shared" + "github.com/gardener/gardener/pkg/features" + gardenletfeatures "github.com/gardener/gardener/pkg/gardenlet/features" imagevectorutils "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/gardener/gardener/pkg/utils/test" ) @@ -92,6 +94,9 @@ func checkIstio(istioDeploy istio.Interface, testValues istioTestValues) { if zoneSize := len(testValues.zones); zoneSize > 1 { minReplicas = ptr.To(zoneSize * 2) maxReplicas = ptr.To(zoneSize * 6) + if features.DefaultFeatureGate.Enabled(features.IstioTLSTermination) { + maxReplicas = ptr.To(zoneSize * 8) + } } networkPolicyLabels := map[string]string{ @@ -207,6 +212,8 @@ var _ = Describe("Istio", func() { ) BeforeEach(func() { + gardenletfeatures.RegisterFeatureGates() + zones = nil istioDeploy = nil }) @@ -249,6 +256,16 @@ var _ = Describe("Istio", func() { }) }) + Context("with IstioTLSTermination enabled", func() { + BeforeEach(func() { + DeferCleanup(test.WithFeatureGate(features.DefaultFeatureGate, features.IstioTLSTermination, true)) + }) + + It("should successfully create a new Istio deployer", func() { + checkIstio(istioDeploy, testValues) + }) + }) + Context("with proxy protocol termination", func() { BeforeEach(func() { proxyProtocolLB = true From 7c0127f653d4f63417513bcbaa2f88f1713b8ef6 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Wed, 12 Nov 2025 15:56:11 +0100 Subject: [PATCH 094/176] [GEP-28] `gardenadm init`: Deploy/restore `DNSRecord` (#13385) * Avoid overwriting extension state on consecutive runs * Handle provider client in local `DNSRecord` controller * `gardenadm init`: Restore bootstrap `DNSRecord` * Adapt `IsShootNamespace` for self-hosted shoots In self-hosted shoots, `kube-system` is used as the control plane namespace. Instead of performing a basic name prefix check, we read the namespace and check the role label. Without this, the `DNSRecord` controller won't pass the `Cluster` object to the actuator inside the self-hosted shoot. * Add e2e test coverage * Adapt naming in `GetProviderClient` * Introduce `UseExistingSecret` in `DNSRecord` deployer Previously, the `DNSRecord` secret was not deployed if `SecretData` was nil. However, the `SecretData` is always nil for provider-local in the usual e2e tests, because the local provider secrets don't contain actual credentials. Now that the local `DNSRecord` controller tries to read the referenced secret, this becomes a problem. From https://prow.gardener.cloud/view/gs/gardener-prow/pr-logs/pull/gardener_gardener/13385/pull-gardener-e2e-kind/1986757627064881152: ``` Shoot Tests Create and Delete Hibernated Shoot Workerless Shoot [It] Wait for Shoot to be deleted [Shoot, default, hibernated, workerless] /home/prow/go/src/github.com/gardener/gardener/test/e2e/gardener/shoot/create_delete_hibernated.go:45 ... 2025-11-07T11:52:59.440Z INFO Waiting for deletion {"shoot": {"name":"e2e-hib-wl","namespace":"garden-local"}, "lastOperation": "&LastOperation{Description:Flow \"Shoot cluster deletion\" encountered task errors: [task \" ``` Instead of implicitly defining whether the `DNSRecord` deployer should manage the secret, this commit introduces an explicit value. This ensure that the secret data doesn't influence the code's decision to manage the secret or not. This basically reverts/reworks https://github.com/gardener/gardener/pull/10645/commits/aaf61afa63e3a742362e3351e13a63242f2b9744. With this, we don't need special handling in the local `DNSRecord` controller. * Adapt error/description of `PreferredAddress` * Use all control plane addresses in `DNSRecord` * Only deploy `DNSRecord` secret on `Destroy` if `SecretName` is set In multiple places, we instantiate the `DNSRecord` component with neither `SecretName` nor `SecretData`. The changed logic in this PR causes `DNSRecord.Destroy()` to try deploying a secret without a name. For example, this causes the `Garden` deletion to fail with: ``` lastOperation: description: "1 error occurred:\n\t* task \"Destroying DNSRecords for virtual garden cluster and ingress controller\" failed: 3 errors occurred:\n\t* Secret \"\" is invalid: metadata.name: Required value: name or generateName is required\n\t* Secret \"\" is invalid: metadata.name: Required value: name or generateName is required\n\t* Secret \"\" is invalid: metadata.name: Required value: name or generateName is required\n\n\n\n Operation will be retried." ``` For simplicity, the `DNSRecord` component should keep support for this way of destroying existing `DNSRecord` objects. Hence, we skip deploying the secret, if `SecretName` is empty. This is basically the same behavior regarding `Destroy` as before the previous commit in this file in this PR. * Refine deletion order in `make garden-down` The previous behavior immediately deleted the DNS and backup secrets together with the `Garden`: ``` project.core.gardener.cloud/garden annotated Error from server (NotFound): projects.core.gardener.cloud "local" not found garden.operator.gardener.cloud/local annotated secret "garden-dns-local" deleted from garden namespace secret "virtual-garden-etcd-main-backup-local" deleted from garden namespace garden.operator.gardener.cloud "local" deleted ``` With the changed provider-local `DNSRecord` controller in this PR, the DNS secret becomes required during `DNSRecord` deletion. Hence, we only delete the `Garden`, wait for it to be gone, and only then delete the other resources. --- dev-setup/garden.sh | 6 +- .../pkg/controller/dnsrecord/reconciler.go | 4 +- .../pkg/controller/extension/reconciler.go | 4 +- extensions/pkg/predicate/preconditions.go | 5 +- .../pkg/predicate/preconditions_test.go | 21 +-- .../pkg/predicate/predicate_suite_test.go | 17 ++ .../extensions/dnsrecord/dnsrecord.go | 38 ++-- .../extensions/dnsrecord/dnsrecord_test.go | 24 ++- .../kubernetes/apiserverexposure/service.go | 6 +- .../apiserverexposure/service_test.go | 31 ++-- pkg/extensions/customresources.go | 8 + pkg/extensions/customresources_test.go | 27 +++ pkg/gardenadm/botanist/dnsrecord.go | 61 ++++++- pkg/gardenadm/botanist/dnsrecord_test.go | 172 ++++++++++++++++++ pkg/gardenadm/botanist/machines.go | 14 +- pkg/gardenadm/botanist/machines_test.go | 33 ++-- pkg/gardenadm/botanist/ssh.go | 4 +- pkg/gardenadm/cmd/init/init.go | 8 + .../garden/garden/reconciler_reconcile.go | 1 + .../controller/dnsrecord/actuator.go | 51 ++++-- .../controller/dnsrecord/actuator_test.go | 17 +- .../controller/dnsrecord/add.go | 2 +- .../controller/extension/shoot/actuator.go | 5 +- .../controller/networkpolicy/add.go | 3 +- .../controller/networkpolicy/add_test.go | 7 +- pkg/provider-local/local/client.go | 16 +- pkg/utils/gardener/shoot.go | 12 +- pkg/utils/gardener/shoot_test.go | 56 ++++++ test/e2e/gardenadm/managedinfra/gardenadm.go | 5 + 29 files changed, 536 insertions(+), 122 deletions(-) create mode 100644 extensions/pkg/predicate/predicate_suite_test.go create mode 100644 pkg/gardenadm/botanist/dnsrecord_test.go diff --git a/dev-setup/garden.sh b/dev-setup/garden.sh index 87f1b99c947..ec1d24a0b25 100755 --- a/dev-setup/garden.sh +++ b/dev-setup/garden.sh @@ -30,8 +30,7 @@ case "$COMMAND" in down) kubectl --kubeconfig "$VIRTUAL_GARDEN_KUBECONFIG" annotate projects local garden confirmation.gardener.cloud/deletion=true || true kubectl annotate garden local confirmation.gardener.cloud/deletion=true || true - - kubectl delete -k "$(dirname "$0")/garden/overlays/$SCENARIO" --ignore-not-found + kubectl delete garden local --wait=false --ignore-not-found echo "Waiting for the garden to be deleted..." kubectl wait --for=delete garden local --timeout=300s @@ -39,6 +38,9 @@ case "$COMMAND" in kubectl wait --for=condition=RequiredRuntime="False" extensions.operator.gardener.cloud provider-local --timeout=120s kubectl wait --for=condition=RequiredVirtual="False" extensions.operator.gardener.cloud provider-local --timeout=30s kubectl wait --for=condition=Installed="False" extensions.operator.gardener.cloud provider-local --timeout=30s + + # cleanup the remaining resources required for successful deletion of the garden + kubectl delete -k "$(dirname "$0")/garden/overlays/$SCENARIO" --ignore-not-found ;; *) diff --git a/extensions/pkg/controller/dnsrecord/reconciler.go b/extensions/pkg/controller/dnsrecord/reconciler.go index 7040b51e668..12274886d7a 100644 --- a/extensions/pkg/controller/dnsrecord/reconciler.go +++ b/extensions/pkg/controller/dnsrecord/reconciler.go @@ -63,7 +63,9 @@ func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( } var cluster *extensions.Cluster - if gardenerutils.IsShootNamespace(dns.Namespace) { + if isShootNamespace, err := gardenerutils.IsShootNamespace(ctx, r.client, dns.Namespace); err != nil { + return reconcile.Result{}, fmt.Errorf("error checking if DNSRecord is in a shoot namespace: %w", err) + } else if isShootNamespace { var err error cluster, err = extensionscontroller.GetCluster(ctx, r.client, dns.Namespace) if err != nil { diff --git a/extensions/pkg/controller/extension/reconciler.go b/extensions/pkg/controller/extension/reconciler.go index 0bd05491cce..60e2f04d6db 100644 --- a/extensions/pkg/controller/extension/reconciler.go +++ b/extensions/pkg/controller/extension/reconciler.go @@ -72,7 +72,9 @@ func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( } var cluster *extensions.Cluster - if gardenerutils.IsShootNamespace(ex.Namespace) { + if isShootNamespace, err := gardenerutils.IsShootNamespace(ctx, r.client, ex.Namespace); err != nil { + return reconcile.Result{}, fmt.Errorf("error checking if Extension is in a shoot namespace: %w", err) + } else if isShootNamespace { var err error cluster, err = extensionscontroller.GetCluster(ctx, r.client, ex.Namespace) if err != nil { diff --git a/extensions/pkg/predicate/preconditions.go b/extensions/pkg/predicate/preconditions.go index 3b709a3d3a6..82a60d38fa3 100644 --- a/extensions/pkg/predicate/preconditions.go +++ b/extensions/pkg/predicate/preconditions.go @@ -45,7 +45,10 @@ func (p *shootNotFailedPredicate) Create(e event.CreateEvent) bool { return false } - if !gardenerutils.IsShootNamespace(e.Object.GetNamespace()) { + if isShootNamespace, err := gardenerutils.IsShootNamespace(p.ctx, p.reader, e.Object.GetNamespace()); err != nil { + logger.Error(err, "Error checking if extension object is in a shoot namespace") + return true + } else if !isShootNamespace { return true } diff --git a/extensions/pkg/predicate/preconditions_test.go b/extensions/pkg/predicate/preconditions_test.go index 3cf59d5fc9d..7036cbf30de 100644 --- a/extensions/pkg/predicate/preconditions_test.go +++ b/extensions/pkg/predicate/preconditions_test.go @@ -82,6 +82,14 @@ var _ = Describe("Preconditions", func() { fakeClient = fakeclient.NewClientBuilder(). WithScheme(kubernetes.SeedScheme). + WithObjects(&corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Labels: map[string]string{ + v1beta1constants.GardenRole: v1beta1constants.GardenRoleShoot, + }, + }, + }). Build() // Create fake manager @@ -140,19 +148,6 @@ var _ = Describe("Preconditions", func() { }) It("should return false if it is a shoot namespace, but cluster is not existing", func() { - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Labels: map[string]string{ - v1beta1constants.GardenRole: v1beta1constants.GardenRoleShoot, - }, - }, - } - Expect(fakeClient.Create(ctx, ns)).To(Succeed()) - DeferCleanup(func() { - Expect(fakeClient.Delete(ctx, ns)).To(Succeed()) - }) - Expect(run()).To(BeFalse()) }) } diff --git a/extensions/pkg/predicate/predicate_suite_test.go b/extensions/pkg/predicate/predicate_suite_test.go new file mode 100644 index 00000000000..b9596847474 --- /dev/null +++ b/extensions/pkg/predicate/predicate_suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package predicate_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPredicate(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Extensions Predicate Suite") +} diff --git a/pkg/component/extensions/dnsrecord/dnsrecord.go b/pkg/component/extensions/dnsrecord/dnsrecord.go index 1c09df94cf7..ea415835040 100644 --- a/pkg/component/extensions/dnsrecord/dnsrecord.go +++ b/pkg/component/extensions/dnsrecord/dnsrecord.go @@ -66,8 +66,11 @@ type Values struct { Type string // Class holds the extension class used to control the responsibility for multiple provider extensions. Class *extensionsv1alpha1.ExtensionClass - // SecretData is the secret data of the DNSRecord (containing provider credentials, etc.). If not provided, the - // secret in the Namespace with name SecretName will be referenced in the DNSRecord object. + // UseExistingSecret indicates whether to use an existing secret referenced by SecretName instead of creating/updating + // it with the given SecretData. + UseExistingSecret bool + // SecretData is the secret data of the DNSRecord (containing provider credentials, etc.), used for creating/updating + // the secret referenced by SecretName. Ignored if UseExistingSecret is true. SecretData map[string][]byte // Zone is the DNS hosted zone of the DNSRecord. Zone *string @@ -94,7 +97,7 @@ func New( waitSevereThreshold time.Duration, waitTimeout time.Duration, ) Interface { - deployer := &dnsRecord{ + return &dnsRecord{ log: log, client: client, values: values, @@ -108,18 +111,13 @@ func New( Namespace: values.Namespace, }, }, - } - - if values.SecretData != nil { - deployer.secret = &corev1.Secret{ + secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: values.SecretName, Namespace: values.Namespace, }, - } + }, } - - return deployer } type dnsRecord struct { @@ -158,11 +156,6 @@ func (d *dnsRecord) deploy(ctx context.Context, operation string) (extensionsv1a metav1.SetMetaDataAnnotation(&d.dnsRecord.ObjectMeta, gardenerutils.AnnotationKeyIPStack, d.values.IPStack) } - secretRef := corev1.SecretReference{Name: d.values.SecretName, Namespace: d.values.Namespace} - if d.secret != nil { - secretRef = corev1.SecretReference{Name: d.secret.Name, Namespace: d.secret.Namespace} - } - for k, v := range d.values.Labels { if d.dnsRecord.Labels == nil { d.dnsRecord.Labels = make(map[string]string) @@ -175,7 +168,10 @@ func (d *dnsRecord) deploy(ctx context.Context, operation string) (extensionsv1a Type: d.values.Type, Class: d.values.Class, }, - SecretRef: secretRef, + SecretRef: corev1.SecretReference{ + Name: d.secret.Name, + Namespace: d.secret.Namespace, + }, Zone: d.values.Zone, Name: d.values.DNSName, RecordType: d.values.RecordType, @@ -221,7 +217,7 @@ func (d *dnsRecord) deploy(ctx context.Context, operation string) (extensionsv1a } func (d *dnsRecord) deploySecret(ctx context.Context) error { - if d.secret == nil { + if d.values.UseExistingSecret { return nil } @@ -255,8 +251,12 @@ func (d *dnsRecord) Migrate(ctx context.Context) error { // Destroy deletes the DNSRecord resource. func (d *dnsRecord) Destroy(ctx context.Context) error { - if err := d.deploySecret(ctx); err != nil { - return err + // Deploy the provider secret if desired, so that the extension controller can use it to delete the DNSRecord. + // The SecretName value might be empty during destruction, i.e., for using the existing secret on the cluster. + if d.values.SecretName != "" { + if err := d.deploySecret(ctx); err != nil { + return err + } } return extensions.DeleteExtensionObject( diff --git a/pkg/component/extensions/dnsrecord/dnsrecord_test.go b/pkg/component/extensions/dnsrecord/dnsrecord_test.go index df7be4f0bdf..f2ee85b6200 100644 --- a/pkg/component/extensions/dnsrecord/dnsrecord_test.go +++ b/pkg/component/extensions/dnsrecord/dnsrecord_test.go @@ -182,7 +182,7 @@ var _ = Describe("DNSRecord", func() { }) It("should only deploy the DNSRecord resource but not the secret", func() { - values.SecretData = nil + values.UseExistingSecret = true dnsRecord = dnsrecord.New(log, c, values, dnsrecord.DefaultInterval, dnsrecord.DefaultSevereThreshold, dnsrecord.DefaultTimeout) Expect(dnsRecord.Deploy(ctx)).To(Succeed()) @@ -678,6 +678,28 @@ var _ = Describe("DNSRecord", func() { Expect(dnsRecord.Destroy(ctx)).To(Succeed()) }) + It("should skip updating the DNSRecord secret if SecretName is empty", func() { + values.SecretName = "" + + dns := &extensionsv1alpha1.DNSRecord{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "confirmation.gardener.cloud/deletion": "true", + v1beta1constants.GardenerTimestamp: now.UTC().Format(time.RFC3339Nano), + }, + }, + } + + mc := mockclient.NewMockClient(ctrl) + mc.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&extensionsv1alpha1.DNSRecord{}), gomock.Any()) + mc.EXPECT().Delete(ctx, dns) + + dnsRecord := dnsrecord.New(log, mc, values, dnsrecord.DefaultInterval, dnsrecord.DefaultSevereThreshold, dnsrecord.DefaultTimeout) + Expect(dnsRecord.Destroy(ctx)).To(Succeed()) + }) + It("should succeed if the resource does not exist", func() { Expect(dnsRecord.Destroy(ctx)).To(Succeed()) }) diff --git a/pkg/component/kubernetes/apiserverexposure/service.go b/pkg/component/kubernetes/apiserverexposure/service.go index 98d87cf47a5..77b017b9c27 100644 --- a/pkg/component/kubernetes/apiserverexposure/service.go +++ b/pkg/component/kubernetes/apiserverexposure/service.go @@ -132,11 +132,13 @@ func (s *service) Deploy(ctx context.Context) error { {MatchLabels: map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleIstioIngress}}, {MatchLabels: map[string]string{v1beta1constants.LabelNetworkPolicyAccessTargetAPIServer: v1beta1constants.LabelNetworkPolicyAllowed}}, } + networkPolicyPort := networkingv1.NetworkPolicyPort{Port: ptr.To(intstr.FromInt32(kubeapiserverconstants.Port)), Protocol: ptr.To(corev1.ProtocolTCP)} // For shoot namespaces the kube-apiserver service needs extra labels and annotations to create required network policies // which allow a connection from istio-ingress components to kube-apiserver. - networkPolicyPort := networkingv1.NetworkPolicyPort{Port: ptr.To(intstr.FromInt32(kubeapiserverconstants.Port)), Protocol: ptr.To(corev1.ProtocolTCP)} - if gardenerutils.IsShootNamespace(obj.Namespace) { + if isShootNamespace, err := gardenerutils.IsShootNamespace(ctx, s.client, obj.Namespace); err != nil { + return fmt.Errorf("error checking if service is in a shoot namespace: %w", err) + } else if isShootNamespace { utilruntime.Must(gardenerutils.InjectNetworkPolicyAnnotationsForScrapeTargets(obj, networkPolicyPort)) metav1.SetMetaDataAnnotation(&obj.ObjectMeta, resourcesv1alpha1.NetworkingPodLabelSelectorNamespaceAlias, v1beta1constants.LabelNetworkPolicyShootNamespaceAlias) diff --git a/pkg/component/kubernetes/apiserverexposure/service_test.go b/pkg/component/kubernetes/apiserverexposure/service_test.go index 701fcb6a8ee..47dc1de234d 100644 --- a/pkg/component/kubernetes/apiserverexposure/service_test.go +++ b/pkg/component/kubernetes/apiserverexposure/service_test.go @@ -42,7 +42,7 @@ var _ = Describe("#Service", func() { clusterIPsFunc func([]string) ingressIPFunc func(string) namePrefix string - namespace string + namespace *corev1.Namespace expectedName string sniServiceObjKey client.ObjectKey ) @@ -58,7 +58,11 @@ var _ = Describe("#Service", func() { ingressIP = "" clusterIP = "" namePrefix = "test-" - namespace = "test-namespace" + namespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + }, + } expectedName = "test-kube-apiserver" sniServiceObjKey = client.ObjectKey{Name: "foo", Namespace: "bar"} clusterIPsFunc = func(c []string) { clusterIP = c[0] } @@ -74,7 +78,7 @@ var _ = Describe("#Service", func() { }, ObjectMeta: metav1.ObjectMeta{ Name: expectedName, - Namespace: namespace, + Namespace: namespace.Name, Labels: map[string]string{ "app": "kubernetes", "role": "apiserver", @@ -110,6 +114,7 @@ var _ = Describe("#Service", func() { }, } + Expect(c.Create(ctx, namespace)).To(Succeed()) Expect(c.Create(ctx, sniService)).To(Succeed()) }) @@ -120,7 +125,7 @@ var _ = Describe("#Service", func() { defaultDepWaiter = NewService( log, c, - namespace, + namespace.Name, values, func() client.ObjectKey { return sniServiceObjKey }, &retryfake.Ops{MaxAttempts: 1}, @@ -134,7 +139,7 @@ var _ = Describe("#Service", func() { Expect(defaultDepWaiter.Deploy(ctx)).To(Succeed()) actual := &corev1.Service{} - Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: expectedName}, actual)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace.Name, Name: expectedName}, actual)).To(Succeed()) Expect(actual).To(DeepEqual(expected)) Expect(clusterIP).To(Equal("1.1.1.1")) @@ -150,14 +155,14 @@ var _ = Describe("#Service", func() { It("deletes service", func() { Expect(defaultDepWaiter.Destroy(ctx)).To(Succeed()) - Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: expectedName}, &corev1.Service{})).To(BeNotFoundError()) + Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace.Name, Name: expectedName}, &corev1.Service{})).To(BeNotFoundError()) }) It("waits for deletion service", func() { Expect(defaultDepWaiter.Destroy(ctx)).To(Succeed()) Expect(defaultDepWaiter.WaitCleanup(ctx)).To(Succeed()) - Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: expectedName}, &corev1.Service{})).To(BeNotFoundError()) + Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace.Name, Name: expectedName}, &corev1.Service{})).To(BeNotFoundError()) }) } @@ -174,13 +179,13 @@ var _ = Describe("#Service", func() { Context("when service is designed for shoots", func() { BeforeEach(func() { - namespace = "shoot-" + expected.Namespace + metav1.SetMetaDataLabel(&namespace.ObjectMeta, "gardener.cloud/role", "shoot") + Expect(c.Update(ctx, namespace)).To(Succeed()) expected.Annotations = utils.MergeStringMaps(map[string]string{ "foo": "bar", "networking.istio.io/exportTo": "*", }, shootNetpolAnnotations()) - expected.Namespace = namespace }) assertService() @@ -200,7 +205,7 @@ var _ = Describe("#Service", func() { Expect(defaultDepWaiter.Deploy(ctx)).To(Succeed()) actual := &corev1.Service{} - Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: expectedName}, actual)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace.Name, Name: expectedName}, actual)).To(Succeed()) Expect(actual.Spec.TrafficDistribution).To(PointTo(Equal(corev1.ServiceTrafficDistributionPreferSameZone))) }) @@ -215,7 +220,7 @@ var _ = Describe("#Service", func() { Expect(defaultDepWaiter.Deploy(ctx)).To(Succeed()) actual := &corev1.Service{} - Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: expectedName}, actual)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace.Name, Name: expectedName}, actual)).To(Succeed()) Expect(actual.Spec.TrafficDistribution).To(PointTo(Equal(corev1.ServiceTrafficDistributionPreferClose))) }) @@ -230,7 +235,7 @@ var _ = Describe("#Service", func() { Expect(defaultDepWaiter.Deploy(ctx)).To(Succeed()) actual := &corev1.Service{} - Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: expectedName}, actual)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace.Name, Name: expectedName}, actual)).To(Succeed()) Expect(actual.Spec.TrafficDistribution).To(PointTo(Equal(corev1.ServiceTrafficDistributionPreferClose))) Expect(actual.Labels).To(HaveKeyWithValue("endpoint-slice-hints.resources.gardener.cloud/consider", "true")) @@ -246,7 +251,7 @@ var _ = Describe("#Service", func() { Expect(defaultDepWaiter.Deploy(ctx)).To(Succeed()) actual := &corev1.Service{} - Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: expectedName}, actual)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace.Name, Name: expectedName}, actual)).To(Succeed()) Expect(actual.Annotations).To(HaveKeyWithValue("service.kubernetes.io/topology-mode", "auto")) Expect(actual.Labels).To(HaveKeyWithValue("endpoint-slice-hints.resources.gardener.cloud/consider", "true")) diff --git a/pkg/extensions/customresources.go b/pkg/extensions/customresources.go index 4d730c725fe..90095bcf70b 100644 --- a/pkg/extensions/customresources.go +++ b/pkg/extensions/customresources.go @@ -269,6 +269,14 @@ func RestoreExtensionObjectState( extensionObj extensionsv1alpha1.Object, kind string, ) error { + if extensionObj.GetExtensionStatus().GetLastOperation() != nil { + // Extension controller has already started restoration/reconciliation of this extension object. + // Don't overwrite its status anymore to avoid conflicts and potential state loss. + // This can happen if `gardenadm init` is re-run with a `ShootState` manifest because it always sets the operation + // of the in-memory Shoot object to `Restore` which triggers restoration of all extension objects. + return nil + } + var resourceRefs []autoscalingv1.CrossVersionObjectReference if shootState.Spec.Extensions != nil { resourceName := extensionObj.GetName() diff --git a/pkg/extensions/customresources_test.go b/pkg/extensions/customresources_test.go index dccae0db9d2..308b85cb40a 100644 --- a/pkg/extensions/customresources_test.go +++ b/pkg/extensions/customresources_test.go @@ -631,6 +631,33 @@ var _ = Describe("extensions", func() { Expect(err).NotTo(HaveOccurred()) Expect(expected.Status.State).To(Equal(expectedState)) }) + + It("should not update the state if the extension controller has already picked up the object", func() { + defer test.WithVars( + &TimeNow, mockNow.Do, + )() + mockNow.EXPECT().Do().Return(now.UTC()).AnyTimes() + + Expect(c.Create(ctx, expected)).To(Succeed()) + + changedStatus := &runtime.RawExtension{ + Raw: []byte(`{"data":"changed-value"}`), + } + expected.Status.LastOperation = &gardencorev1beta1.LastOperation{ + State: gardencorev1beta1.LastOperationStateProcessing, + } + expected.Status.State = changedStatus + Expect(c.Status().Update(ctx, expected)).To(Succeed()) + + Expect(RestoreExtensionObjectState( + ctx, + c, + shootState, + expected, + extensionsv1alpha1.WorkerResource, + )).To(Succeed()) + Expect(expected.Status.State).To(Equal(changedStatus)) + }) }) }) diff --git a/pkg/gardenadm/botanist/dnsrecord.go b/pkg/gardenadm/botanist/dnsrecord.go index f06cbc17911..be538cbc2f7 100644 --- a/pkg/gardenadm/botanist/dnsrecord.go +++ b/pkg/gardenadm/botanist/dnsrecord.go @@ -6,7 +6,14 @@ package botanist import ( "context" + "fmt" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" extensionsv1alpha1helper "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1/helper" "github.com/gardener/gardener/pkg/component" ) @@ -21,9 +28,9 @@ func (b *GardenadmBotanist) DeployBootstrapDNSRecord(ctx context.Context) error return err } - machineAddr, err := PreferredAddressForMachine(machine) + machineAddr, err := PreferredAddress(machine.Status.Addresses) if err != nil { - return err + return fmt.Errorf("failed getting preferred address for machine %q: %w", machine.Name, err) } b.Shoot.Components.Extensions.ExternalDNSRecord.SetRecordType(extensionsv1alpha1helper.GetDNSRecordType(machineAddr)) @@ -31,3 +38,53 @@ func (b *GardenadmBotanist) DeployBootstrapDNSRecord(ctx context.Context) error return component.OpWait(b.Shoot.Components.Extensions.ExternalDNSRecord).Deploy(ctx) } + +// RestoreExternalDNSRecord restores the external DNSRecord pointing to the control plane nodes based on the extension +// state from the bootstrap cluster stored by `gardenadm bootstrap` in the ShootState. +func (b *GardenadmBotanist) RestoreExternalDNSRecord(ctx context.Context) error { + // The DNSRecord values are not persisted in the ShootState, so we recalculate them from the Node objects. + // This is the same logic as in DeployBootstrapDNSRecord, but we fetch the control plane Node objects using a label + // selector instead of fetching the Machine objects. + // We expect that any bootstrapped control plane node accepts traffic on its preferred address. + controlPlaneWorkerPool := v1beta1helper.ControlPlaneWorkerPoolForShoot(b.Shoot.GetInfo().Spec.Provider.Workers) + if controlPlaneWorkerPool == nil { + return fmt.Errorf("failed fetching the control plane worker pool for the shoot") + } + + nodeList := &corev1.NodeList{} + if err := b.SeedClientSet.Client().List(ctx, nodeList, client.MatchingLabels{v1beta1constants.LabelWorkerPool: controlPlaneWorkerPool.Name}); err != nil { + return fmt.Errorf("failed to list machines: %w", err) + } + if len(nodeList.Items) == 0 { + return fmt.Errorf("no control plane nodes founds") + } + + // Collect preferred addresses of all control plane nodes and ensure they are of the same type. + var ( + values []string + recordType extensionsv1alpha1.DNSRecordType + ) + + for _, node := range nodeList.Items { + nodeAddr, err := PreferredAddress(node.Status.Addresses) + if err != nil { + return fmt.Errorf("failed getting preferred address for node %q: %w", node.Name, err) + } + values = append(values, nodeAddr) + + currentRecordType := extensionsv1alpha1helper.GetDNSRecordType(nodeAddr) + if recordType == "" { + recordType = currentRecordType + } else if recordType != currentRecordType { + return fmt.Errorf("inconsistent address types for control plane nodes: found both %q and %q: %v", recordType, currentRecordType, values) + } + } + + b.Shoot.Components.Extensions.ExternalDNSRecord.SetRecordType(recordType) + b.Shoot.Components.Extensions.ExternalDNSRecord.SetValues(values) + + if err := b.Shoot.Components.Extensions.ExternalDNSRecord.Restore(ctx, b.Shoot.GetShootState()); err != nil { + return err + } + return b.Shoot.Components.Extensions.ExternalDNSRecord.Wait(ctx) +} diff --git a/pkg/gardenadm/botanist/dnsrecord_test.go b/pkg/gardenadm/botanist/dnsrecord_test.go new file mode 100644 index 00000000000..ccdc65a678e --- /dev/null +++ b/pkg/gardenadm/botanist/dnsrecord_test.go @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package botanist + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" + fakekubernetes "github.com/gardener/gardener/pkg/client/kubernetes/fake" + mockdnsrecord "github.com/gardener/gardener/pkg/component/extensions/dnsrecord/mock" + "github.com/gardener/gardener/pkg/gardenlet/operation" + botanistpkg "github.com/gardener/gardener/pkg/gardenlet/operation/botanist" + "github.com/gardener/gardener/pkg/gardenlet/operation/shoot" +) + +var _ = Describe("DNSRecord", func() { + const controlPlaneWorkerPoolName = "control-plane" + + var ( + b *GardenadmBotanist + + fakeClient client.Client + + externalDNSRecord *mockdnsrecord.MockInterface + ) + + BeforeEach(func() { + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + fakeClientSet := fakekubernetes.NewClientSetBuilder().WithClient(fakeClient).Build() + + ctrl := gomock.NewController(GinkgoT()) + externalDNSRecord = mockdnsrecord.NewMockInterface(ctrl) + + b = &GardenadmBotanist{ + Botanist: &botanistpkg.Botanist{Operation: &operation.Operation{ + SeedClientSet: fakeClientSet, + ShootClientSet: fakeClientSet, + Shoot: &shoot.Shoot{ + ControlPlaneNamespace: "kube-system", + Components: &shoot.Components{ + Extensions: &shoot.Extensions{ + ExternalDNSRecord: externalDNSRecord, + }, + }, + }, + }, + }, + } + + b.Shoot.SetInfo(&gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Provider: gardencorev1beta1.Provider{ + Workers: []gardencorev1beta1.Worker{{ + Name: controlPlaneWorkerPoolName, + ControlPlane: &gardencorev1beta1.WorkerControlPlane{}, + }}, + }, + }, + }) + b.Shoot.SetShootState(&gardencorev1beta1.ShootState{}) + }) + + Describe("#RestoreExternalDNSRecord", func() { + It("should fail if there is no control plane worker pool", func(ctx SpecContext) { + shoot := b.Shoot.GetInfo() + shoot.Spec.Provider.Workers = []gardencorev1beta1.Worker{{ + Name: "worker", + }} + b.Shoot.SetInfo(shoot) + + Expect(b.RestoreExternalDNSRecord(ctx)).To(MatchError("failed fetching the control plane worker pool for the shoot")) + }) + + It("should fail if there are no control plane nodes", func(ctx SpecContext) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + Labels: map[string]string{ + "worker.gardener.cloud/pool": "worker", + }, + }, + } + Expect(fakeClient.Create(ctx, node)).To(Succeed()) + + Expect(b.RestoreExternalDNSRecord(ctx)).To(MatchError("no control plane nodes founds")) + }) + + It("should fail if the control plane node doesn't have any addresses", func(ctx SpecContext) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: controlPlaneWorkerPoolName + "-1", + Labels: map[string]string{ + "worker.gardener.cloud/pool": controlPlaneWorkerPoolName, + }, + }, + } + Expect(fakeClient.Create(ctx, node)).To(Succeed()) + + Expect(b.RestoreExternalDNSRecord(ctx)).To(MatchError(ContainSubstring("no addresses found in status"))) + }) + + It("should fail if the control plane nodes have different address types", func(ctx SpecContext) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: controlPlaneWorkerPoolName + "-1", + Labels: map[string]string{ + "worker.gardener.cloud/pool": controlPlaneWorkerPoolName, + }, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{{ + Type: corev1.NodeInternalIP, + Address: "10.0.0.1", + }}, + }, + } + + node2 := node.DeepCopy() + node2.Name = controlPlaneWorkerPoolName + "-2" + node2.Status.Addresses = []corev1.NodeAddress{{ + Type: corev1.NodeHostName, + Address: node2.Name, + }} + + Expect(fakeClient.Create(ctx, node)).To(Succeed()) + Expect(fakeClient.Create(ctx, node2)).To(Succeed()) + + Expect(b.RestoreExternalDNSRecord(ctx)).To(MatchError(ContainSubstring("inconsistent address types"))) + }) + + It("should set the correct values and restore the DNSRecord", func(ctx SpecContext) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: controlPlaneWorkerPoolName + "-1", + Labels: map[string]string{ + "worker.gardener.cloud/pool": controlPlaneWorkerPoolName, + }, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{{ + Type: corev1.NodeInternalIP, + Address: "10.0.0.1", + }}, + }, + } + + node2 := node.DeepCopy() + node2.Name = controlPlaneWorkerPoolName + "-2" + node2.Status.Addresses[0].Address = "10.0.0.2" + + Expect(fakeClient.Create(ctx, node)).To(Succeed()) + Expect(fakeClient.Create(ctx, node2)).To(Succeed()) + + externalDNSRecord.EXPECT().SetRecordType(extensionsv1alpha1.DNSRecordTypeA) + externalDNSRecord.EXPECT().SetValues([]string{node.Status.Addresses[0].Address, node2.Status.Addresses[0].Address}) + externalDNSRecord.EXPECT().Restore(gomock.Any(), gomock.Any()).Return(nil) + externalDNSRecord.EXPECT().Wait(gomock.Any()).Return(nil) + + Expect(b.RestoreExternalDNSRecord(ctx)).To(Succeed()) + }) + }) +}) diff --git a/pkg/gardenadm/botanist/machines.go b/pkg/gardenadm/botanist/machines.go index e103dc75851..ba41ac113f1 100644 --- a/pkg/gardenadm/botanist/machines.go +++ b/pkg/gardenadm/botanist/machines.go @@ -36,7 +36,7 @@ func (b *GardenadmBotanist) GetMachineByIndex(index int) (*machinev1alpha1.Machi return &b.controlPlaneMachines[index], nil } -// addressTypePreference when retrieving the SSH Address of a machine. Higher value means higher priority. +// addressTypePreference when retrieving the SSH Address of a node/machine. Higher value means higher priority. // Unknown address types have the lowest priority (0). var addressTypePreference = map[corev1.NodeAddressType]int{ // internal names have priority, as we jump via a bastion host @@ -48,14 +48,14 @@ var addressTypePreference = map[corev1.NodeAddressType]int{ corev1.NodeHostName: 1, } -// PreferredAddressForMachine returns the preferred address of the given machine based on addressTypePreference. -// If the machine has no addresses, an error is returned. -func PreferredAddressForMachine(machine *machinev1alpha1.Machine) (string, error) { - if len(machine.Status.Addresses) == 0 { - return "", fmt.Errorf("no addresses found in status of machine %s", machine.Name) +// PreferredAddress returns the preferred address of the given node/machine addresses based on addressTypePreference. +// If the node/machine has no addresses, an error is returned. +func PreferredAddress(addresses []corev1.NodeAddress) (string, error) { + if len(addresses) == 0 { + return "", fmt.Errorf("no addresses found in status") } - address := slices.MaxFunc(machine.Status.Addresses, func(a, b corev1.NodeAddress) int { + address := slices.MaxFunc(addresses, func(a, b corev1.NodeAddress) int { return addressTypePreference[a.Type] - addressTypePreference[b.Type] }) diff --git a/pkg/gardenadm/botanist/machines_test.go b/pkg/gardenadm/botanist/machines_test.go index 43f8d6d0cd5..35d435b35fa 100644 --- a/pkg/gardenadm/botanist/machines_test.go +++ b/pkg/gardenadm/botanist/machines_test.go @@ -5,63 +5,58 @@ package botanist_test import ( - machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/gardener/gardener/pkg/gardenadm/botanist" ) var _ = Describe("Machines", func() { - Describe("#PreferredAddressForMachine", func() { - var machine *machinev1alpha1.Machine + Describe("#PreferredAddress", func() { + var addresses []corev1.NodeAddress BeforeEach(func() { - machine = &machinev1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{Name: "test-machine"}, - Status: machinev1alpha1.MachineStatus{}, - } + addresses = []corev1.NodeAddress{} }) It("should return error if no addresses are present", func() { - Expect(PreferredAddressForMachine(machine)).Error().To(MatchError(ContainSubstring("no addresses found"))) + Expect(PreferredAddress(addresses)).Error().To(MatchError(ContainSubstring("no addresses found"))) }) It("should return the only address present", func() { - machine.Status.Addresses = []corev1.NodeAddress{{Type: corev1.NodeExternalIP, Address: "1.2.3.4"}} - Expect(PreferredAddressForMachine(machine)).To(Equal("1.2.3.4")) + addresses = []corev1.NodeAddress{{Type: corev1.NodeExternalIP, Address: "1.2.3.4"}} + Expect(PreferredAddress(addresses)).To(Equal("1.2.3.4")) }) It("should return the address with the highest preference", func() { - machine.Status.Addresses = []corev1.NodeAddress{ + addresses = []corev1.NodeAddress{ {Type: corev1.NodeExternalIP, Address: "1.2.3.4"}, {Type: corev1.NodeHostName, Address: "host.local"}, {Type: corev1.NodeInternalIP, Address: "10.0.0.2"}, } - Expect(PreferredAddressForMachine(machine)).To(Equal("10.0.0.2")) + Expect(PreferredAddress(addresses)).To(Equal("10.0.0.2")) }) It("should prefer InternalDNS over ExternalIP", func() { - machine.Status.Addresses = []corev1.NodeAddress{ + addresses = []corev1.NodeAddress{ {Type: corev1.NodeExternalIP, Address: "1.2.3.4"}, {Type: corev1.NodeInternalDNS, Address: "internal.dns"}, } - Expect(PreferredAddressForMachine(machine)).To(Equal("internal.dns")) + Expect(PreferredAddress(addresses)).To(Equal("internal.dns")) }) It("should return unknown type if only unknown is present", func() { - machine.Status.Addresses = []corev1.NodeAddress{{Type: "UnknownType", Address: "unknown.addr"}} - Expect(PreferredAddressForMachine(machine)).To(Equal("unknown.addr")) + addresses = []corev1.NodeAddress{{Type: "UnknownType", Address: "unknown.addr"}} + Expect(PreferredAddress(addresses)).To(Equal("unknown.addr")) }) It("should prefer known type over unknown type", func() { - machine.Status.Addresses = []corev1.NodeAddress{ + addresses = []corev1.NodeAddress{ {Type: "UnknownType", Address: "unknown.addr"}, {Type: corev1.NodeExternalDNS, Address: "external.dns"}, } - Expect(PreferredAddressForMachine(machine)).To(Equal("external.dns")) + Expect(PreferredAddress(addresses)).To(Equal("external.dns")) }) }) }) diff --git a/pkg/gardenadm/botanist/ssh.go b/pkg/gardenadm/botanist/ssh.go index 7d4bf6e7a49..2ca8588de57 100644 --- a/pkg/gardenadm/botanist/ssh.go +++ b/pkg/gardenadm/botanist/ssh.go @@ -33,9 +33,9 @@ func (b *GardenadmBotanist) ConnectToControlPlaneMachine(ctx context.Context) er return err } - machineAddr, err := PreferredAddressForMachine(machine) + machineAddr, err := PreferredAddress(machine.Status.Addresses) if err != nil { - return err + return fmt.Errorf("failed getting preferred address for machine %q: %w", machine.Name, err) } sshAddr := net.JoinHostPort(machineAddr, "22") diff --git a/pkg/gardenadm/cmd/init/init.go b/pkg/gardenadm/cmd/init/init.go index f429d8df853..35602aa2482 100644 --- a/pkg/gardenadm/cmd/init/init.go +++ b/pkg/gardenadm/cmd/init/init.go @@ -249,6 +249,14 @@ func run(ctx context.Context, opts *Options) error { waitUntilExtensionControllersInPodNetworkReady, ) + // Later on, we will expose the control plane via a LoadBalancer and point the DNSRecord to it. + // TODO(timebertt): adapt this step to consider the external LoadBalancer if it is already available + _ = g.Add(flow.Task{ + Name: "Restoring external DNSRecord", + Fn: b.RestoreExternalDNSRecord, + SkipIf: !b.Shoot.HasManagedInfrastructure(), + Dependencies: flow.NewTaskIDs(syncPointBootstrapped), + }) reconcileBackupBucket = g.Add(flow.Task{ Name: "Deploying BackupBucket for ETCD data", Fn: b.ReconcileBackupBucket, diff --git a/pkg/operator/controller/garden/garden/reconciler_reconcile.go b/pkg/operator/controller/garden/garden/reconciler_reconcile.go index f907db9b1cf..8e68ca12efa 100644 --- a/pkg/operator/controller/garden/garden/reconciler_reconcile.go +++ b/pkg/operator/controller/garden/garden/reconciler_reconcile.go @@ -1205,6 +1205,7 @@ func (r *Reconciler) reconcileDNSRecords(ctx context.Context, log logr.Logger, g Type: provider.Type, Class: ptr.To(extensionsv1alpha1.ExtensionClassGarden), SecretName: provider.SecretRef.Name, + UseExistingSecret: true, ReconcileOnlyOnChangeOrError: true, }, dnsrecord.DefaultInterval, diff --git a/pkg/provider-local/controller/dnsrecord/actuator.go b/pkg/provider-local/controller/dnsrecord/actuator.go index 905d40f3207..44e6a2187de 100644 --- a/pkg/provider-local/controller/dnsrecord/actuator.go +++ b/pkg/provider-local/controller/dnsrecord/actuator.go @@ -22,11 +22,16 @@ import ( extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" "github.com/gardener/gardener/pkg/component/networking/coredns" + "github.com/gardener/gardener/pkg/provider-local/local" ) // Actuator implements the DNSRecord actuator for the local DNS provider. type Actuator struct { - Client client.Client + // RuntimeClient uses provider-local's in-cluster config, e.g., for the seed/bootstrap cluster it runs in. + // It's used to interact with extension objects. By default, it's also used as the provider client to interact with + // DNS resources (the provider-local-coredns ConfigMap), unless a kubeconfig is specified in the secret referenced in + // the DNSRecord. + RuntimeClient client.Client } // shouldHandleDNSRecord returns true for DNSRecords that are implemented in provider-local by writing to the custom @@ -38,30 +43,37 @@ func shouldHandleDNSRecord(dnsRecord *extensionsv1alpha1.DNSRecord) bool { } // Reconcile ensures that the DNS record is correctly represented in the CoreDNS config map. -func (a *Actuator) Reconcile(ctx context.Context, _ logr.Logger, dnsRecord *extensionsv1alpha1.DNSRecord, cluster *extensionscontroller.Cluster) error { - if !shouldHandleDNSRecord(dnsRecord) { - return nil - } - - config, err := a.configForDNSRecord(ctx, dnsRecord, cluster) - if err != nil { - return err - } +func (a *Actuator) Reconcile(ctx context.Context, log logr.Logger, dnsRecord *extensionsv1alpha1.DNSRecord, cluster *extensionscontroller.Cluster) error { + return a.reconcile(ctx, log, dnsRecord, func(configMap *corev1.ConfigMap) error { + config, err := a.configForDNSRecord(ctx, dnsRecord, cluster) + if err != nil { + return err + } - return a.patchCoreDNSConfigMap(ctx, func(configMap *corev1.ConfigMap) { configMap.Data[keyForDNSRecord(dnsRecord)] = config + return nil }) } // Delete removes the DNS record from the CoreDNS config map. -func (a *Actuator) Delete(ctx context.Context, _ logr.Logger, dnsRecord *extensionsv1alpha1.DNSRecord, _ *extensionscontroller.Cluster) error { +func (a *Actuator) Delete(ctx context.Context, log logr.Logger, dnsRecord *extensionsv1alpha1.DNSRecord, _ *extensionscontroller.Cluster) error { + return a.reconcile(ctx, log, dnsRecord, func(configMap *corev1.ConfigMap) error { + delete(configMap.Data, keyForDNSRecord(dnsRecord)) + return nil + }) +} + +func (a *Actuator) reconcile(ctx context.Context, log logr.Logger, dnsRecord *extensionsv1alpha1.DNSRecord, mutate func(configMap *corev1.ConfigMap) error) error { if !shouldHandleDNSRecord(dnsRecord) { return nil } - return a.patchCoreDNSConfigMap(ctx, func(configMap *corev1.ConfigMap) { - delete(configMap.Data, keyForDNSRecord(dnsRecord)) - }) + providerClient, err := local.GetProviderClient(ctx, log, a.RuntimeClient, dnsRecord.Spec.SecretRef) + if err != nil { + return fmt.Errorf("could not create client for DNS resources: %w", err) + } + + return patchCoreDNSConfigMap(ctx, providerClient, mutate) } // ForceDelete is the same as Delete for the local DNS provider. @@ -85,11 +97,10 @@ func (a *Actuator) Restore(ctx context.Context, log logr.Logger, dnsRecord *exte return a.Reconcile(ctx, log, dnsRecord, cluster) } -func (a *Actuator) patchCoreDNSConfigMap(ctx context.Context, mutate func(configMap *corev1.ConfigMap)) error { +func patchCoreDNSConfigMap(ctx context.Context, providerClient client.Client, mutate func(configMap *corev1.ConfigMap) error) error { configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: coredns.CustomConfigMapName, Namespace: "gardener-extension-provider-local-coredns"}} - _, err := controllerutil.CreateOrPatch(ctx, a.Client, configMap, func() error { - mutate(configMap) - return nil + _, err := controllerutil.CreateOrPatch(ctx, providerClient, configMap, func() error { + return mutate(configMap) }) return err } @@ -117,7 +128,7 @@ func (a *Actuator) configForDNSRecord(ctx context.Context, dnsRecord *extensions } namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: dnsRecord.Namespace}} - if err := a.Client.Get(ctx, client.ObjectKeyFromObject(namespace), namespace); err != nil { + if err := a.RuntimeClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace); err != nil { return "", err } diff --git a/pkg/provider-local/controller/dnsrecord/actuator_test.go b/pkg/provider-local/controller/dnsrecord/actuator_test.go index fb6edac999f..8bc2d41474a 100644 --- a/pkg/provider-local/controller/dnsrecord/actuator_test.go +++ b/pkg/provider-local/controller/dnsrecord/actuator_test.go @@ -47,12 +47,12 @@ var _ = Describe("Actuator", func() { configMapWithRule *corev1.ConfigMap ) - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { log = logger.MustNewZapLogger(logger.DebugLevel, logger.FormatJSON, zap.WriteTo(GinkgoWriter)) fakeClient = fakeclient.NewClientBuilder().WithScheme(scheme.Scheme).Build() actuator = &Actuator{ - Client: fakeClient, + RuntimeClient: fakeClient, } namespace = "foo" @@ -82,6 +82,15 @@ var _ = Describe("Actuator", func() { }, }, } + + dnsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dns-secret", + Namespace: namespace, + }, + } + Expect(fakeClient.Create(ctx, dnsSecret)).To(Succeed()) + apiDNSRecord = &extensionsv1alpha1.DNSRecord{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -90,6 +99,10 @@ var _ = Describe("Actuator", func() { Name: "api.something.local.gardener.cloud", Values: []string{"1.2.3.4", "5.6.7.8"}, TTL: ptr.To[int64](123), + SecretRef: corev1.SecretReference{ + Name: dnsSecret.Name, + Namespace: dnsSecret.Namespace, + }, }, } otherDNSRecord = &extensionsv1alpha1.DNSRecord{ diff --git a/pkg/provider-local/controller/dnsrecord/add.go b/pkg/provider-local/controller/dnsrecord/add.go index 8356a4da4d7..f67f1990885 100644 --- a/pkg/provider-local/controller/dnsrecord/add.go +++ b/pkg/provider-local/controller/dnsrecord/add.go @@ -35,7 +35,7 @@ type AddOptions struct { func AddToManagerWithOptions(ctx context.Context, mgr manager.Manager, opts AddOptions) error { return dnsrecord.Add(mgr, dnsrecord.AddArgs{ Actuator: &Actuator{ - Client: mgr.GetClient(), + RuntimeClient: mgr.GetClient(), }, ControllerOptions: opts.Controller, Predicates: dnsrecord.DefaultPredicates(ctx, mgr, opts.IgnoreOperationAnnotation), diff --git a/pkg/provider-local/controller/extension/shoot/actuator.go b/pkg/provider-local/controller/extension/shoot/actuator.go index 3e2e7308e94..cf8c9d1d8d5 100644 --- a/pkg/provider-local/controller/extension/shoot/actuator.go +++ b/pkg/provider-local/controller/extension/shoot/actuator.go @@ -6,6 +6,7 @@ package shoot import ( "context" + "fmt" "time" "github.com/go-logr/logr" @@ -82,7 +83,9 @@ func (a *actuator) Reconcile(ctx context.Context, _ logr.Logger, ex *extensionsv return err } - if gardenerutils.IsShootNamespace(ex.Namespace) { + if isShootNamespace, err := gardenerutils.IsShootNamespace(ctx, a.client, ex.Namespace); err != nil { + return fmt.Errorf("error checking if Extension is in a shoot namespace: %w", err) + } else if isShootNamespace { cluster, err := extensionscontroller.GetCluster(ctx, a.client, ex.Namespace) if err != nil { return err diff --git a/pkg/provider-local/controller/networkpolicy/add.go b/pkg/provider-local/controller/networkpolicy/add.go index 895fa9df9ce..6601ef61d06 100644 --- a/pkg/provider-local/controller/networkpolicy/add.go +++ b/pkg/provider-local/controller/networkpolicy/add.go @@ -16,7 +16,6 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/provider-local/local" - gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) // ControllerName is the name of the controller. @@ -46,7 +45,7 @@ func (r *Reconciler) AddToManager(mgr manager.Manager) error { // IsShootNamespace returns a predicate that returns true if the namespace is a shoot namespace. func IsShootNamespace() predicate.Predicate { return predicate.NewPredicateFuncs(func(obj client.Object) bool { - return gardenerutils.IsShootNamespace(obj.GetName()) + return obj.GetLabels()[v1beta1constants.GardenRole] == v1beta1constants.GardenRoleShoot }) } diff --git a/pkg/provider-local/controller/networkpolicy/add_test.go b/pkg/provider-local/controller/networkpolicy/add_test.go index f142c590f3b..4f82bebf4ad 100644 --- a/pkg/provider-local/controller/networkpolicy/add_test.go +++ b/pkg/provider-local/controller/networkpolicy/add_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -20,7 +21,7 @@ var _ = Describe("Add", func() { p predicate.Predicate ) - Describe("#IsShootNamespacePredicate", func() { + Describe("#IsShootNamespace", func() { BeforeEach(func() { p = IsShootNamespace() namespace = &corev1.Namespace{} @@ -28,6 +29,7 @@ var _ = Describe("Add", func() { It("should return true because the namespace is a shoot namespace", func() { namespace.Name = "shoot-garden-local" + metav1.SetMetaDataLabel(&namespace.ObjectMeta, "gardener.cloud/role", "shoot") Expect(p.Create(event.CreateEvent{Object: namespace})).To(BeTrue()) Expect(p.Update(event.UpdateEvent{ObjectNew: namespace})).To(BeTrue()) @@ -37,6 +39,7 @@ var _ = Describe("Add", func() { It("should return true because the namespace is a shoot namespace", func() { namespace.Name = "shoot--local-local" + metav1.SetMetaDataLabel(&namespace.ObjectMeta, "gardener.cloud/role", "shoot") Expect(p.Create(event.CreateEvent{Object: namespace})).To(BeTrue()) Expect(p.Update(event.UpdateEvent{ObjectNew: namespace})).To(BeTrue()) @@ -44,7 +47,7 @@ var _ = Describe("Add", func() { Expect(p.Generic(event.GenericEvent{Object: namespace})).To(BeTrue()) }) - It("should return true because the namespace is not a shoot namespace", func() { + It("should return false because the namespace is not a shoot namespace", func() { namespace.Name = "foo" Expect(p.Create(event.CreateEvent{Object: namespace})).To(BeFalse()) diff --git a/pkg/provider-local/local/client.go b/pkg/provider-local/local/client.go index b782254fe84..267b850549c 100644 --- a/pkg/provider-local/local/client.go +++ b/pkg/provider-local/local/client.go @@ -17,28 +17,28 @@ import ( ) // GetProviderClient returns a Kubernetes client for the cluster in which provider-local should manage infrastructure -// resources, e.g., Services, NetworkPolicies, machine Pods, etc. If the cloudprovider secret contains a kubeconfig, +// resources, e.g., Services, NetworkPolicies, machine Pods, etc. If the provider secret contains a kubeconfig, // a client for that kubeconfig is created. Otherwise, the given client for the runtime cluster is returned. // See https://github.com/gardener/gardener/blob/master/docs/extensions/provider-local.md#credentials. func GetProviderClient(ctx context.Context, log logr.Logger, runtimeClient client.Client, secretRef corev1.SecretReference) (client.Client, error) { - cloudProviderSecret, err := kubernetesutils.GetSecretByReference(ctx, runtimeClient, &secretRef) + providerSecret, err := kubernetesutils.GetSecretByReference(ctx, runtimeClient, &secretRef) if err != nil { - return nil, fmt.Errorf("could not retrieve cloudprovider secret: %w", err) + return nil, fmt.Errorf("could not retrieve provider secret: %w", err) } - if len(cloudProviderSecret.Data[kubernetes.KubeConfig]) == 0 { - log.Info("Using in-cluster config for provider client as no kubeconfig is specified in the cloudprovider secret") + if len(providerSecret.Data[kubernetes.KubeConfig]) == 0 { + log.Info("Using in-cluster config for provider client as no kubeconfig is specified in the provider secret") return runtimeClient, nil } - clientSet, err := kubernetes.NewClientFromBytes(cloudProviderSecret.Data[kubernetes.KubeConfig], + clientSet, err := kubernetes.NewClientFromBytes(providerSecret.Data[kubernetes.KubeConfig], kubernetes.WithClientOptions(client.Options{Scheme: kubernetes.SeedScheme}), kubernetes.WithDisabledCachedClient(), ) if err != nil { - return nil, fmt.Errorf("could not create client from cloudprovider secret: %w", err) + return nil, fmt.Errorf("could not create client from provider secret: %w", err) } - log.Info("Using kubeconfig from cloudprovider secret for provider client") + log.Info("Using kubeconfig from provider secret for provider client") return clientSet.Client(), nil } diff --git a/pkg/utils/gardener/shoot.go b/pkg/utils/gardener/shoot.go index 9ef5d5e9e6b..faab955388f 100644 --- a/pkg/utils/gardener/shoot.go +++ b/pkg/utils/gardener/shoot.go @@ -784,9 +784,15 @@ func ComputeTechnicalID(projectName string, shoot *gardencorev1beta1.Shoot) stri return fmt.Sprintf("%s-%s--%s", v1beta1constants.TechnicalIDPrefix, projectName, shoot.Name) } -// IsShootNamespace returns true if the given namespace is a shoot namespace, i.e. it starts with the technical id prefix. -func IsShootNamespace(namespace string) bool { - return strings.HasPrefix(namespace, v1beta1constants.TechnicalIDPrefix) +// IsShootNamespace returns true if the given namespace is a shoot control plane namespace, i.e., if it has the +// garden.cloud/role=shoot label. +func IsShootNamespace(ctx context.Context, reader client.Reader, namespaceName string) (bool, error) { + namespace := &corev1.Namespace{} + if err := reader.Get(ctx, client.ObjectKey{Name: namespaceName}, namespace); err != nil { + return false, err + } + + return namespace.Labels[v1beta1constants.GardenRole] == v1beta1constants.GardenRoleShoot, nil } // GetShootConditionTypes returns all known shoot condition types. diff --git a/pkg/utils/gardener/shoot_test.go b/pkg/utils/gardener/shoot_test.go index 929dfbf3b82..0a78a1a8760 100644 --- a/pkg/utils/gardener/shoot_test.go +++ b/pkg/utils/gardener/shoot_test.go @@ -1502,6 +1502,62 @@ var _ = Describe("Shoot", func() { }) }) + Describe("#IsShootNamespace", func() { + var ( + client client.Client + + namespace *corev1.Namespace + ) + + JustBeforeEach(func() { + namespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shoot--foo--bar", + Labels: map[string]string{ + v1beta1constants.GardenRole: v1beta1constants.GardenRoleShoot, + }, + }, + } + + client = fakeclient.NewClientBuilder(). + WithScheme(kubernetesscheme.Scheme). + WithObjects(namespace). + Build() + }) + + It("returns true if the namespace has the shoot role label", func(ctx SpecContext) { + Expect(IsShootNamespace(ctx, client, namespace.Name)).To(BeTrue()) + }) + + It("returns false if the namespace does not have the shoot role label", func(ctx SpecContext) { + namespace.Labels["gardener.cloud/role"] = "some-other-role" + Expect(client.Update(ctx, namespace)).To(Succeed()) + + Expect(IsShootNamespace(ctx, client, namespace.Name)).To(BeFalse()) + }) + + Context("kube-system (used in self-hosted shoots)", func() { + BeforeEach(func() { + namespace.Name = "kube-system" + }) + + It("returns true if the namespace has the shoot role label", func(ctx SpecContext) { + Expect(IsShootNamespace(ctx, client, namespace.Name)).To(BeTrue()) + }) + + It("returns false if the namespace does not have the shoot role label", func(ctx SpecContext) { + namespace.Labels["gardener.cloud/role"] = "kube-system" + Expect(client.Update(ctx, namespace)).To(Succeed()) + + Expect(IsShootNamespace(ctx, client, namespace.Name)).To(BeFalse()) + }) + }) + + It("returns error if the namespace does not exist", func(ctx SpecContext) { + Expect(IsShootNamespace(ctx, client, "missing")).Error().To(BeNotFoundError()) + }) + }) + Describe("#GetShootConditionTypes", func() { It("should return all shoot condition types", func() { Expect(GetShootConditionTypes(false)).To(HaveExactElements( diff --git a/test/e2e/gardenadm/managedinfra/gardenadm.go b/test/e2e/gardenadm/managedinfra/gardenadm.go index b7670dcd22e..f9d5ff95b63 100644 --- a/test/e2e/gardenadm/managedinfra/gardenadm.go +++ b/test/e2e/gardenadm/managedinfra/gardenadm.go @@ -265,6 +265,11 @@ var _ = Describe("gardenadm managed infrastructure scenario tests", Label("garde Eventually(ctx, Get(service)).Should(Succeed()) }, SpecTimeout(time.Minute)) + It("should deploy/restore the DNSRecord in the shoot", func(ctx SpecContext) { + dnsRecord := &extensionsv1alpha1.DNSRecord{ObjectMeta: metav1.ObjectMeta{Name: shootName + "-external", Namespace: "kube-system"}} + Eventually(ctx, shootKomega.Object(dnsRecord)).Should(BeHealthy(health.CheckExtensionObject)) + }, SpecTimeout(time.Minute)) + It("should finish successfully", func(ctx SpecContext) { Wait(ctx, session) Eventually(ctx, session.Out).Should(gbytes.Say("work in progress")) From 7d289e24c4ac9bb4b0c2112cba5e750f8a7704ff Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Wed, 12 Nov 2025 15:56:19 +0100 Subject: [PATCH 095/176] [GEP-28] HA: Augment `gardenadm join` to request short-lived client certificate via bootstrap token (similar to `gardenadm connect`) (#13387) * [node-agent] Add role label to nodes NAME STATUS ROLES AGE VERSION machine-0 Ready 3h38m v1.33.0 vs. NAME STATUS ROLES AGE VERSION machine-0 Ready control-plane 3h39m v1.33.0 Also applies to hosted shoots :) * `gardenadm join`: Add `--control-plane` option * Prefactor: Extract function for creating a bootstrap garden client set * Prefactor: Make short-lived cert-based kubeconfig retrieval reusable * `gardenadm join --control-plane` - for now, this only requests a short-lived client certificate based on the bootstrap token (similar to what happens in `gardenadm connect`) - in the future, we augment the flow to perform steps required for joining a control plane node * Auto-approve `gardenadm` clients in `gardener-resource-manager` * Streamline options for `token create` and `join` - remove `--gardener-node-agent-secret-name` option from `gardenadm join` - no longer specify `--worker-pool-name` in `gardenadm token create --print-join-command` --> by default, the first worker pool in the Shoot manifest will be chosen - the user can still override the worker pool name when running `gardenadm join` by specifying `--worker-pool-name` - with the bootstrap token, the `join` command creates a short-lived client that automatically determines the gardener-node-agent secret name * Deploy RBAC resources for temporary `gardenadm` client * Scale up unmanaged machines to `4` - prep for local development of HA/joining control plane nodes - `machine-{0,1,2}` could be the control plane nodes - `machine-3` could be the worker node * `gardenadm init`: Adapt `join` instructions * Address PR review feedback --- dev-setup/gardenadm/machines/machine.yaml | 2 +- docs/cli-reference/gardenadm/gardenadm.md | 2 +- .../cli-reference/gardenadm/gardenadm_join.md | 27 +- .../gardenadm/gardenadm_token_create.md | 15 +- pkg/component/shoot/system/system.go | 78 ++++ pkg/component/shoot/system/system_test.go | 76 ++++ pkg/gardenadm/botanist/controlplane.go | 16 +- pkg/gardenadm/botanist/controlplane_test.go | 29 +- pkg/gardenadm/cmd/connect/connect.go | 99 +----- pkg/gardenadm/cmd/init/init.go | 29 +- pkg/gardenadm/cmd/join/join.go | 137 ++++++- pkg/gardenadm/cmd/join/join_test.go | 202 +++++++++++ pkg/gardenadm/cmd/join/options.go | 16 +- pkg/gardenadm/cmd/join/options_test.go | 10 +- pkg/gardenadm/cmd/temporary_client.go | 108 ++++++ pkg/gardenadm/cmd/temporary_client_test.go | 334 ++++++++++++++++++ pkg/gardenadm/cmd/token/create/create.go | 31 +- pkg/gardenadm/cmd/token/create/create_test.go | 16 +- pkg/gardenadm/cmd/token/create/options.go | 8 - .../cmd/token/create/options_test.go | 13 - .../operation/botanist/shootsystem.go | 1 + .../operatingsystemconfig/reconciler.go | 17 + .../controller/csrapprover/reconciler.go | 36 +- skaffold-gardenadm.yaml | 2 +- .../operatingsystemconfig_test.go | 24 ++ 25 files changed, 1070 insertions(+), 258 deletions(-) create mode 100644 pkg/gardenadm/cmd/join/join_test.go create mode 100644 pkg/gardenadm/cmd/temporary_client.go create mode 100644 pkg/gardenadm/cmd/temporary_client_test.go diff --git a/dev-setup/gardenadm/machines/machine.yaml b/dev-setup/gardenadm/machines/machine.yaml index 5c29819b444..1c1bf69c1fb 100644 --- a/dev-setup/gardenadm/machines/machine.yaml +++ b/dev-setup/gardenadm/machines/machine.yaml @@ -5,7 +5,7 @@ metadata: labels: app: machine spec: - replicas: 2 + replicas: 4 serviceName: "machine" selector: matchLabels: diff --git a/docs/cli-reference/gardenadm/gardenadm.md b/docs/cli-reference/gardenadm/gardenadm.md index 793db701a9e..a22697d7c63 100644 --- a/docs/cli-reference/gardenadm/gardenadm.md +++ b/docs/cli-reference/gardenadm/gardenadm.md @@ -16,7 +16,7 @@ gardenadm bootstraps and manages self-hosted shoot clusters in the Gardener proj * [gardenadm connect](gardenadm_connect.md) - Deploy a gardenlet for further cluster management * [gardenadm discover](gardenadm_discover.md) - Conveniently download Gardener configuration resources from an existing garden cluster * [gardenadm init](gardenadm_init.md) - Bootstrap the first control plane node -* [gardenadm join](gardenadm_join.md) - Bootstrap worker nodes and join them to the cluster +* [gardenadm join](gardenadm_join.md) - Bootstrap control plane or worker nodes and join them to the cluster * [gardenadm token](gardenadm_token.md) - Manage bootstrap and discovery tokens for gardenadm join * [gardenadm version](gardenadm_version.md) - Print the client version information diff --git a/docs/cli-reference/gardenadm/gardenadm_join.md b/docs/cli-reference/gardenadm/gardenadm_join.md index 760bb6fb6a3..e3eb665d769 100644 --- a/docs/cli-reference/gardenadm/gardenadm_join.md +++ b/docs/cli-reference/gardenadm/gardenadm_join.md @@ -1,15 +1,13 @@ ## gardenadm join -Bootstrap worker nodes and join them to the cluster +Bootstrap control plane or worker nodes and join them to the cluster ### Synopsis -Bootstrap worker nodes and join them to the cluster. +Bootstrap control plane or worker nodes and join them to the cluster. This command helps to initialize and configure a node to join an existing self-hosted shoot cluster. -It ensures that the necessary configurations are applied and the node is properly registered as a worker or control plane node. - -Note that further control plane nodes cannot be joined currently. +It ensures that the necessary configurations are applied and the node is properly registered as a control plane or worker node. ``` gardenadm join [flags] @@ -18,17 +16,24 @@ gardenadm join [flags] ### Examples ``` -# Bootstrap a worker node and join it to the cluster -gardenadm join --bootstrap-token --ca-certificate --gardener-node-agent-secret-name +# Bootstrap a control plane node and join it to the cluster +gardenadm join --bootstrap-token --ca-certificate --control-plane + +# Bootstrap a worker node and join it to the cluster (by default, it is assigned to the first worker pool in the Shoot manifest) +gardenadm join --bootstrap-token --ca-certificate + +# Bootstrap a worker node in a specific worker pool and join it to the cluster +gardenadm join --bootstrap-token --ca-certificate --worker-pool-name ``` ### Options ``` - --bootstrap-token string Bootstrap token for joining the cluster (create it with 'gardenadm token' on a control plane node) - --ca-certificate bytesBase64 Base64-encoded certificate authority bundle of the control plane - --gardener-node-agent-secret-name string Name of the Secret from which gardener-node-agent should download its operating system configuration - -h, --help help for join + --bootstrap-token string Bootstrap token for joining the cluster (create it with 'gardenadm token' on a control plane node) + --ca-certificate bytesBase64 Base64-encoded certificate authority bundle of the control plane + --control-plane Create a new control plane instance on this node + -h, --help help for join + -w, --worker-pool-name string Name of the worker pool to assign the joining node. ``` ### Options inherited from parent commands diff --git a/docs/cli-reference/gardenadm/gardenadm_token_create.md b/docs/cli-reference/gardenadm/gardenadm_token_create.md index 11f6e785a31..8812143ab11 100644 --- a/docs/cli-reference/gardenadm/gardenadm_token_create.md +++ b/docs/cli-reference/gardenadm/gardenadm_token_create.md @@ -32,14 +32,13 @@ gardenadm token create ### Options ``` - -d, --description string Description for the bootstrap token used for 'gardenadm join' - -h, --help help for create - -c, --print-connect-command Instead of only printing the token, print the full machine-readable 'gardenadm connect' command that can be ran on a machine of a cluster that should be connected to Gardener - -j, --print-join-command Instead of only printing the token, print the full machine-readable 'gardenadm join' command that can be copied and ran on a machine that should join the cluster - --shoot-name string Name of the Shoot which should be connected to Gardener via 'gardenadm connect' with this bootstrap token - --shoot-namespace string Namespace of the Shoot which should be connected to Gardener via 'gardenadm connect' with this bootstrap token - --validity duration Validity duration of the bootstrap token. Minimum is 10m, maximum is 24h. (default 1h0m0s) - -w, --worker-pool-name string Name of the worker pool to use for the join command. (default "worker") + -d, --description string Description for the bootstrap token used for 'gardenadm join' + -h, --help help for create + -c, --print-connect-command Instead of only printing the token, print the full machine-readable 'gardenadm connect' command that can be ran on a machine of a cluster that should be connected to Gardener + -j, --print-join-command Instead of only printing the token, print the full machine-readable 'gardenadm join' command that can be copied and ran on a machine that should join the cluster + --shoot-name string Name of the Shoot which should be connected to Gardener via 'gardenadm connect' with this bootstrap token + --shoot-namespace string Namespace of the Shoot which should be connected to Gardener via 'gardenadm connect' with this bootstrap token + --validity duration Validity duration of the bootstrap token. Minimum is 10m, maximum is 24h. (default 1h0m0s) ``` ### Options inherited from parent commands diff --git a/pkg/component/shoot/system/system.go b/pkg/component/shoot/system/system.go index 9a724c5c222..c54eff9675f 100644 --- a/pkg/component/shoot/system/system.go +++ b/pkg/component/shoot/system/system.go @@ -23,6 +23,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component" kubeapiserverconstants "github.com/gardener/gardener/pkg/component/kubernetes/apiserver/constants" @@ -56,6 +57,8 @@ type Values struct { ExternalClusterDomain *string // IsWorkerless specifies whether the cluster has worker nodes. IsWorkerless bool + // IsSelfHosted specifies whether the cluster is self-hosted. + IsSelfHosted bool // KubernetesVersion is the version of the cluster. KubernetesVersion *semver.Version // EncryptedResources is the list of resources which are encrypted by the kube-apiserver. @@ -324,6 +327,10 @@ func (s *shootSystem) computeResourcesData() (map[string][]byte, error) { } } + if err := registry.Add(s.selfHostedShootResources()...); err != nil { + return nil, err + } + return registry.SerializedObjects() } @@ -471,3 +478,74 @@ func addNetworkToMap(name string, cidrs []net.IPNet, data map[string]string) { data[name] = networks } } + +func (s *shootSystem) selfHostedShootResources() []client.Object { + if !s.values.IsSelfHosted { + return nil + } + + var ( + rbacName = "gardener.cloud:gardenadm" + + clusterRole = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: rbacName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{extensionsv1alpha1.SchemeGroupVersion.Group}, + Resources: []string{"clusters"}, + Verbs: []string{"get"}, + ResourceNames: []string{metav1.NamespaceSystem}, + }, + }, + } + clusterRoleBinding = &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: rbacName, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: clusterRole.Name, + }, + Subjects: []rbacv1.Subject{{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: v1beta1constants.ShootsGroup, + }}, + } + + role = &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: rbacName, + Namespace: metav1.NamespaceSystem, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{corev1.GroupName}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch"}, + }, + }, + } + roleBinding = &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: rbacName, + Namespace: metav1.NamespaceSystem, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: v1beta1constants.ShootsGroup, + }}, + } + ) + + return []client.Object{clusterRole, clusterRoleBinding, role, roleBinding} +} diff --git a/pkg/component/shoot/system/system_test.go b/pkg/component/shoot/system/system_test.go index dcb56f5d0a4..f2f1bdf8439 100644 --- a/pkg/component/shoot/system/system_test.go +++ b/pkg/component/shoot/system/system_test.go @@ -564,6 +564,82 @@ var _ = Describe("ShootSystem", func() { Expect(managedResource).To(contain(clusterRole)) }) }) + + When("shoot is self-hosted", func() { + BeforeEach(func() { + values.IsSelfHosted = true + }) + + It("should successfully deploy all gardenadm RBAC resources", func() { + expectedClusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:gardenadm", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"extensions.gardener.cloud"}, + Resources: []string{"clusters"}, + Verbs: []string{"get"}, + ResourceNames: []string{"kube-system"}, + }, + }, + } + + expectedClusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:gardenadm", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "gardener.cloud:gardenadm", + }, + Subjects: []rbacv1.Subject{{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Group", + Name: "gardener.cloud:system:shoots", + }}, + } + + expectedRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:gardenadm", + Namespace: "kube-system", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch"}, + }, + }, + } + + expectedRoleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:gardenadm", + Namespace: "kube-system", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "gardener.cloud:gardenadm", + }, + Subjects: []rbacv1.Subject{{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Group", + Name: "gardener.cloud:system:shoots", + }}, + } + + Expect(managedResource).To(contain( + expectedClusterRole, + expectedClusterRoleBinding, + expectedRole, + expectedRoleBinding, + )) + }) + }) }) }) diff --git a/pkg/gardenadm/botanist/controlplane.go b/pkg/gardenadm/botanist/controlplane.go index 97e58c0b639..04b279b1709 100644 --- a/pkg/gardenadm/botanist/controlplane.go +++ b/pkg/gardenadm/botanist/controlplane.go @@ -23,7 +23,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -349,21 +348,8 @@ func (b *GardenadmBotanist) CreateClientSet(ctx context.Context) (kubernetes.Int return clientSet, nil } -// NewWithConfig in alias for kubernetes.NewWithConfig. -// Exposed for unit testing. -var NewWithConfig = kubernetes.NewWithConfig - // DiscoverKubernetesVersion discovers the Kubernetes version of the control plane. -func (b *GardenadmBotanist) DiscoverKubernetesVersion(controlPlaneAddress string, caBundle []byte, token string) (*semver.Version, error) { - clientSet, err := NewWithConfig(kubernetes.WithRESTConfig(&rest.Config{ - Host: controlPlaneAddress, - TLSClientConfig: rest.TLSClientConfig{CAData: caBundle}, - BearerToken: token, - })) - if err != nil { - return nil, fmt.Errorf("failed creating a new client: %w", err) - } - +func (b *GardenadmBotanist) DiscoverKubernetesVersion(clientSet kubernetes.Interface) (*semver.Version, error) { version, err := semver.NewVersion(clientSet.Version()) if err != nil { return nil, fmt.Errorf("failed parsing semver version %q: %w", clientSet.Version(), err) diff --git a/pkg/gardenadm/botanist/controlplane_test.go b/pkg/gardenadm/botanist/controlplane_test.go index 524157852cd..4ff96ac75dd 100644 --- a/pkg/gardenadm/botanist/controlplane_test.go +++ b/pkg/gardenadm/botanist/controlplane_test.go @@ -6,7 +6,6 @@ package botanist import ( "context" - "fmt" "time" "github.com/Masterminds/semver/v3" @@ -35,38 +34,14 @@ var _ = Describe("ControlPlane", func() { }) Describe("#DiscoverKubernetesVersion", func() { - var ( - controlPlaneAddress = "control-plane-address" - token = "token" - caBundle = []byte("ca-bundle") - ) - It("should succeed discovering the version", func() { - DeferCleanup(test.WithVar(&NewWithConfig, func(_ ...kubernetes.ConfigFunc) (kubernetes.Interface, error) { - return fakekubernetes.NewClientSetBuilder().WithVersion("1.33.0").Build(), nil - })) - - version, err := b.DiscoverKubernetesVersion(controlPlaneAddress, caBundle, token) + version, err := b.DiscoverKubernetesVersion(fakekubernetes.NewClientSetBuilder().WithVersion("1.33.0").Build()) Expect(err).NotTo(HaveOccurred()) Expect(version).To(Equal(semver.MustParse("1.33.0"))) }) - It("should fail creating the client set from the kubeconfig", func() { - DeferCleanup(test.WithVar(&NewWithConfig, func(_ ...kubernetes.ConfigFunc) (kubernetes.Interface, error) { - return nil, fmt.Errorf("fake err") - })) - - version, err := b.DiscoverKubernetesVersion(controlPlaneAddress, caBundle, token) - Expect(err).To(MatchError(ContainSubstring("fake err"))) - Expect(version).To(BeNil()) - }) - It("should fail parsing the kubernetes version", func() { - DeferCleanup(test.WithVar(&NewWithConfig, func(_ ...kubernetes.ConfigFunc) (kubernetes.Interface, error) { - return fakekubernetes.NewClientSetBuilder().WithVersion("cannot-parse").Build(), nil - })) - - version, err := b.DiscoverKubernetesVersion(controlPlaneAddress, caBundle, token) + version, err := b.DiscoverKubernetesVersion(fakekubernetes.NewClientSetBuilder().WithVersion("cannot-parse").Build()) Expect(err).To(MatchError(ContainSubstring("failed parsing semver version"))) Expect(version).To(BeNil()) }) diff --git a/pkg/gardenadm/cmd/connect/connect.go b/pkg/gardenadm/cmd/connect/connect.go index 33af9defb1c..904b32f87dd 100644 --- a/pkg/gardenadm/cmd/connect/connect.go +++ b/pkg/gardenadm/cmd/connect/connect.go @@ -6,20 +6,15 @@ package connect import ( "context" - "crypto/x509/pkix" "fmt" - "net" - "path/filepath" "time" - "github.com/spf13/afero" "github.com/spf13/cobra" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "k8s.io/component-base/version" "k8s.io/utils/clock" @@ -36,16 +31,12 @@ import ( "github.com/gardener/gardener/pkg/gardenadm/botanist" "github.com/gardener/gardener/pkg/gardenadm/cmd" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" - gardenletbootstraputil "github.com/gardener/gardener/pkg/gardenlet/bootstrap/util" "github.com/gardener/gardener/pkg/utils/flow" - "github.com/gardener/gardener/pkg/utils/kubernetes/certificatesigningrequest" "github.com/gardener/gardener/pkg/utils/kubernetes/health" "github.com/gardener/gardener/pkg/utils/oci" "github.com/gardener/gardener/pkg/utils/retry" ) -const bootstrapKubeconfigValidity = 10 * time.Minute - // NewCommand creates a new cobra.Command. func NewCommand(globalOpts *cmd.Options) *cobra.Command { opts := &Options{Options: globalOpts} @@ -95,21 +86,25 @@ func run(ctx context.Context, opts *Options) error { if alreadyConnected, err := isGardenletDeployed(ctx, b); err != nil { return fmt.Errorf("failed checking if gardenlet is already deployed: %w", err) } else if !alreadyConnected || opts.Force { - bootstrapClientSet, err := kubernetes.NewWithConfig(kubernetes.WithRESTConfig(&rest.Config{ - Host: opts.ControlPlaneAddress, - TLSClientConfig: rest.TLSClientConfig{CAData: opts.CertificateAuthority}, - BearerToken: opts.BootstrapToken, - }), kubernetes.WithClientOptions(client.Options{Scheme: kubernetes.GardenScheme}), kubernetes.WithDisabledCachedClient()) + bootstrapClientSet, err := cmd.NewClientSetFromBootstrapToken(opts.ControlPlaneAddress, opts.CertificateAuthority, opts.BootstrapToken, kubernetes.GardenScheme) if err != nil { - return fmt.Errorf("failed creating garden client set: %w", err) + return fmt.Errorf("failed creating a new bootstrap garden client set: %w", err) } + var ( g = flow.NewGraph("connect") retrieveShortLivedKubeconfig = g.Add(flow.Task{ Name: "Retrieving short-lived kubeconfig for garden cluster to prepare Gardener resources", Fn: func(ctx context.Context) error { - return initializeTemporaryGardenClient(ctx, b, bootstrapClientSet) + gardenClientSet, err := cmd.InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + if err != nil { + return fmt.Errorf("failed retrieving short-lived kubeconfig for garden cluster: %w", err) + } + + b.Logger.Info("Successfully retrieved short-lived bootstrap kubeconfig for garden cluster") + b.GardenClient = gardenClientSet.Client() + return nil }, }) prepareResources = g.Add(flow.Task{ @@ -196,78 +191,6 @@ func isGardenletDeployed(ctx context.Context, b *botanist.GardenadmBotanist) (bo return true, nil } -func cachedBootstrapKubeconfigPath(fs afero.Afero) string { - return filepath.Join(fs.GetTempDir(""), "gardenadm-connect-bootstrap-kubeconfig") -} - -func initializeTemporaryGardenClient(ctx context.Context, b *botanist.GardenadmBotanist, bootstrapClientSet kubernetes.Interface) error { - bootstrapKubeconfig, cached, err := getCachedBootstrapKubeconfig(b) - if err != nil { - return fmt.Errorf("failed retrieving cached bootstrap kubeconfig: %w", err) - } - - if !cached { - bootstrapKubeconfig, err = requestShortLivedBootstrapKubeconfig(ctx, b, bootstrapClientSet) - if err != nil { - return fmt.Errorf("failed to request short-lived bootstrap kubeconfig via CertificateSigningRequest API: %w", err) - } - - if err := b.FS.WriteFile(cachedBootstrapKubeconfigPath(b.FS), bootstrapKubeconfig, 0600); err != nil { - return fmt.Errorf("failed writing the retrieved bootstrap kubeconfig to a temporary file: %w", err) - } - } - - return setGardenClientFromKubeconfig(b, bootstrapKubeconfig) -} - -func getCachedBootstrapKubeconfig(b *botanist.GardenadmBotanist) ([]byte, bool, error) { - fileInfo, err := b.FS.Stat(cachedBootstrapKubeconfigPath(b.FS)) - if err != nil || time.Since(fileInfo.ModTime()) > bootstrapKubeconfigValidity-2*time.Minute { - // We deliberately ignore the error here - this is just a best-effort attempt to cache the bootstrap kubeconfig. - // If the file doesn't exist, or we cannot read/find it for whatever reason, we just consider it as a cache - // miss. - // Otherwise, if the last modifications time of the file is older than the validity of the bootstrap kubeconfig, - // we consider it as expired and thus a cache miss. - return nil, false, nil //nolint:nilerr - } - - data, err := b.FS.ReadFile(cachedBootstrapKubeconfigPath(b.FS)) - if err != nil { - return nil, false, fmt.Errorf("failed reading the cached bootstrap kubeconfig: %w", err) - } - - return data, true, nil -} - -func requestShortLivedBootstrapKubeconfig(ctx context.Context, b *botanist.GardenadmBotanist, bootstrapClientSet kubernetes.Interface) ([]byte, error) { - certificateSubject := &pkix.Name{ - Organization: []string{v1beta1constants.ShootsGroup}, - CommonName: v1beta1constants.GardenadmUserNamePrefix + b.Shoot.GetInfo().Namespace + ":" + b.Shoot.GetInfo().Name, - } - - certData, privateKeyData, _, err := certificatesigningrequest.RequestCertificate(ctx, b.Logger, bootstrapClientSet.Kubernetes(), certificateSubject, []string{}, []net.IP{}, &metav1.Duration{Duration: bootstrapKubeconfigValidity}, "gardenadm-connect-csr-") - if err != nil { - return nil, fmt.Errorf("unable to bootstrap the kubeconfig for the Garden cluster: %w", err) - } - - return gardenletbootstraputil.CreateKubeconfigWithClientCertificate(bootstrapClientSet.RESTConfig(), privateKeyData, certData) -} - -func setGardenClientFromKubeconfig(b *botanist.GardenadmBotanist, kubeconfig []byte) error { - gardenClientSet, err := kubernetes.NewClientFromBytes( - kubeconfig, - kubernetes.WithClientOptions(client.Options{Scheme: kubernetes.GardenScheme}), - kubernetes.WithDisabledCachedClient(), - ) - if err != nil { - return fmt.Errorf("failed creating garden client set from short-lived kubeconfig: %w", err) - } - - b.Logger.Info("Successfully retrieved short-lived bootstrap kubeconfig for garden cluster") - b.GardenClient = gardenClientSet.Client() - return nil -} - func prepareGardenerResources(ctx context.Context, b *botanist.GardenadmBotanist) error { if err := b.GardenClient.Get(ctx, client.ObjectKeyFromObject(b.Resources.CloudProfile), &gardencorev1beta1.CloudProfile{}); err != nil { return fmt.Errorf("failed checking for existence of CloudProfile %s (this is not created by 'gardenadm connect' and must exist in the garden cluster): %w", b.Resources.CloudProfile.Name, err) diff --git a/pkg/gardenadm/cmd/init/init.go b/pkg/gardenadm/cmd/init/init.go index 35602aa2482..8701e683ec8 100644 --- a/pkg/gardenadm/cmd/init/init.go +++ b/pkg/gardenadm/cmd/init/init.go @@ -336,27 +336,20 @@ To start using your cluster, you need to run the following as a regular user: sudo cp -i %s $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config kubectl get nodes -`, botanist.PathKubeconfig) - for _, worker := range b.Shoot.GetInfo().Spec.Provider.Workers { - if worker.ControlPlane == nil { - fmt.Fprintf(opts.Out, ` -You can now join any number of worker machines to pool %[1]q (or any other -worker pool). Run this on a control plane node: +You can now join any number of control-plane or worker nodes. A bootstrap token +is required to authenticate a new machine when joining the cluster. To create +such a token, run this on a control-plane node: - gardenadm token create --print-join-command --worker-pool-name %[1]s + gardenadm token create --print-join-command -Copy the output and run it as root on each node you would like to join the -cluster. -`, worker.Name) - break - } - } +Copy the output and run it as root on the machine you would like to join the +cluster. Append '--control-plane' to the printed command if the machine should +be joined as a control-plane node. - fmt.Fprintf(opts.Out, ` -Note that the mentioned kubeconfig file will be disabled once you deploy the -gardenlet and connect this cluster to an existing Gardener installation. Run -this while targeting the garden cluster to which you want to connect this +Note that the above mentioned kubeconfig file will be disabled once you deploy +the gardenlet and connect this cluster to an existing Gardener installation. +Run this while targeting the garden cluster to which you want to connect this self-hosted shoot cluster: gardenadm token create --print-connect-command --shoot-namespace=%s --shoot-name=%s @@ -366,7 +359,7 @@ gardenlet for connectivity to Gardener. Please use the shoots/adminkubeconfig subresource to retrieve a kubeconfig, see https://gardener.cloud/docs/gardener/shoot/shoot_access/. -`, b.Shoot.GetInfo().Namespace, b.Shoot.GetInfo().Name) +`, botanist.PathKubeconfig, b.Shoot.GetInfo().Namespace, b.Shoot.GetInfo().Name) return nil } diff --git a/pkg/gardenadm/cmd/join/join.go b/pkg/gardenadm/cmd/join/join.go index 369304b3e8b..988bf614440 100644 --- a/pkg/gardenadm/cmd/join/join.go +++ b/pkg/gardenadm/cmd/join/join.go @@ -9,7 +9,15 @@ import ( "fmt" "github.com/spf13/cobra" - + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/client/kubernetes" + gardenerextensions "github.com/gardener/gardener/pkg/extensions" "github.com/gardener/gardener/pkg/gardenadm/botanist" "github.com/gardener/gardener/pkg/gardenadm/cmd" shootpkg "github.com/gardener/gardener/pkg/gardenlet/operation/shoot" @@ -22,15 +30,19 @@ func NewCommand(globalOpts *cmd.Options) *cobra.Command { cmd := &cobra.Command{ Use: "join", - Short: "Bootstrap worker nodes and join them to the cluster", - Long: `Bootstrap worker nodes and join them to the cluster. + Short: "Bootstrap control plane or worker nodes and join them to the cluster", + Long: `Bootstrap control plane or worker nodes and join them to the cluster. This command helps to initialize and configure a node to join an existing self-hosted shoot cluster. -It ensures that the necessary configurations are applied and the node is properly registered as a worker or control plane node. +It ensures that the necessary configurations are applied and the node is properly registered as a control plane or worker node.`, + Example: `# Bootstrap a control plane node and join it to the cluster +gardenadm join --bootstrap-token --ca-certificate --control-plane + +# Bootstrap a worker node and join it to the cluster (by default, it is assigned to the first worker pool in the Shoot manifest) +gardenadm join --bootstrap-token --ca-certificate -Note that further control plane nodes cannot be joined currently.`, - Example: `# Bootstrap a worker node and join it to the cluster -gardenadm join --bootstrap-token --ca-certificate --gardener-node-agent-secret-name `, +# Bootstrap a worker node in a specific worker pool and join it to the cluster +gardenadm join --bootstrap-token --ca-certificate --worker-pool-name `, Args: cobra.ExactArgs(1), @@ -62,11 +74,16 @@ func run(ctx context.Context, opts *Options) error { return fmt.Errorf("failed creating gardenadm botanist: %w", err) } - version, err := b.DiscoverKubernetesVersion(opts.ControlPlaneAddress, opts.CertificateAuthority, opts.BootstrapToken) + bootstrapClientSet, err := cmd.NewClientSetFromBootstrapToken(opts.ControlPlaneAddress, opts.CertificateAuthority, opts.BootstrapToken, kubernetes.SeedScheme) + if err != nil { + return fmt.Errorf("failed creating a new bootstrap client set: %w", err) + } + version, err := b.DiscoverKubernetesVersion(bootstrapClientSet) if err != nil { return fmt.Errorf("failed discovering Kubernetes version of cluster: %w", err) } b.Shoot = &shootpkg.Shoot{KubernetesVersion: version} + b.Shoot.SetInfo(nil) alreadyJoined, err := b.IsGardenerNodeAgentInitialized(ctx) if err != nil { @@ -75,13 +92,41 @@ func run(ctx context.Context, opts *Options) error { if !alreadyJoined { var ( - g = flow.NewGraph("join") + g = flow.NewGraph("join") + gardenerNodeAgentSecretName string + + retrieveShortLivedKubeconfig = g.Add(flow.Task{ + Name: "Retrieving short-lived shoot cluster kubeconfig to prepare control plane scale-up", + Fn: func(ctx context.Context) error { + shootClientSet, err := cmd.InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + if err != nil { + return fmt.Errorf("failed retrieving short-lived kubeconfig: %w", err) + } + + b.Logger.Info("Successfully retrieved short-lived bootstrap kubeconfig") + b.ShootClientSet = shootClientSet + return nil + }, + }) + determineGardenerNodeAgentSecretName = g.Add(flow.Task{ + Name: "Determining gardener-node-agent Secret containing the configuration for this node", + Fn: func(ctx context.Context) error { + var err error + gardenerNodeAgentSecretName, err = GetGardenerNodeAgentSecretName(ctx, opts, b) + return err + }, + Dependencies: flow.NewTaskIDs(retrieveShortLivedKubeconfig), + }) + syncPointReadyForGardenerNodeInit = flow.NewTaskIDs( + determineGardenerNodeAgentSecretName, + ) generateGardenerNodeInitConfig = g.Add(flow.Task{ Name: "Preparing gardener-node-init configuration", Fn: func(ctx context.Context) error { - return b.PrepareGardenerNodeInitConfiguration(ctx, opts.GardenerNodeAgentSecretName, opts.ControlPlaneAddress, opts.CertificateAuthority, opts.BootstrapToken) + return b.PrepareGardenerNodeInitConfiguration(ctx, gardenerNodeAgentSecretName, opts.ControlPlaneAddress, opts.CertificateAuthority, opts.BootstrapToken) }, + Dependencies: flow.NewTaskIDs(syncPointReadyForGardenerNodeInit), }) _ = g.Add(flow.Task{ Name: "Applying OperatingSystemConfig using gardener-node-agent's reconciliation logic", @@ -97,9 +142,17 @@ func run(ctx context.Context, opts *Options) error { } } - fmt.Fprintf(opts.Out, ` + if opts.ControlPlane { + fmt.Fprintf(opts.Out, ` +Your node has successfully been instructed to join the cluster as a control-plane instance! +`) + } else { + fmt.Fprintf(opts.Out, ` Your node has successfully been instructed to join the cluster as a worker! +`) + } + fmt.Fprintf(opts.Out, ` The bootstrap token will be deleted automatically by kube-controller-manager after it has expired. If you want to delete it right away, run the following command on any control plane node: @@ -114,3 +167,65 @@ logs of kubelet by running 'journalctl -u kubelet'. return nil } + +// GetGardenerNodeAgentSecretName retrieves the Secret for gardener-node-agent which contains the operating system +// configuration for this node. +func GetGardenerNodeAgentSecretName(ctx context.Context, opts *Options, b *botanist.GardenadmBotanist) (string, error) { + workerPoolName, err := getWorkerPoolName(ctx, opts, b) + if err != nil { + return "", fmt.Errorf("failed to determine worker pool name in Shoot manifest: %w", err) + } + + secretList := &corev1.SecretList{} + if err := b.ShootClientSet.Client().List(ctx, secretList, client.InNamespace(metav1.NamespaceSystem), client.MatchingLabels{ + v1beta1constants.GardenRole: v1beta1constants.GardenRoleOperatingSystemConfig, + v1beta1constants.LabelWorkerPool: workerPoolName, + }); err != nil { + return "", fmt.Errorf("failed listing gardener-node-agent secrets: %w", err) + } + + if len(secretList.Items) == 0 { + return "", fmt.Errorf("no gardener-node-agent secrets found for worker pool %q", workerPoolName) + } + + gardenerNodeAgentSecret := secretList.Items[0] + if len(secretList.Items) > 1 { + opts.Log.V(1).Info("Multiple gardener-node-agent secrets found, using the first one", "secretName", gardenerNodeAgentSecret.Name) + } + + return gardenerNodeAgentSecret.Name, nil +} + +func getWorkerPoolName(ctx context.Context, opts *Options, b *botanist.GardenadmBotanist) (string, error) { + if opts.WorkerPoolName != "" { + return opts.WorkerPoolName, nil + } + + cluster, err := gardenerextensions.GetCluster(ctx, b.ShootClientSet.Client(), metav1.NamespaceSystem) + if err != nil { + return "", fmt.Errorf("failed reading extensions.gardener.cloud/v1alpha1.Cluster object: %w", err) + } + + if opts.ControlPlane { + return getControlPlaneWorkerPoolName(cluster.Shoot.Spec.Provider.Workers) + } + return getFirstWorkerPoolName(cluster.Shoot.Spec.Provider.Workers) +} + +func getControlPlaneWorkerPoolName(workers []gardencorev1beta1.Worker) (string, error) { + if pool := helper.ControlPlaneWorkerPoolForShoot(workers); pool != nil { + return pool.Name, nil + } + + return "", fmt.Errorf("no control plane worker pool found in Shoot manifest") +} + +func getFirstWorkerPoolName(workers []gardencorev1beta1.Worker) (string, error) { + for _, worker := range workers { + if worker.ControlPlane == nil { + return worker.Name, nil + } + } + + return "", fmt.Errorf("no non-control-plane pool found in Shoot manifest") +} diff --git a/pkg/gardenadm/cmd/join/join_test.go b/pkg/gardenadm/cmd/join/join_test.go new file mode 100644 index 00000000000..870a962837c --- /dev/null +++ b/pkg/gardenadm/cmd/join/join_test.go @@ -0,0 +1,202 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package join_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" + fakekubernetes "github.com/gardener/gardener/pkg/client/kubernetes/fake" + "github.com/gardener/gardener/pkg/gardenadm/botanist" + . "github.com/gardener/gardener/pkg/gardenadm/cmd/join" + operationpkg "github.com/gardener/gardener/pkg/gardenlet/operation" + botanistpkg "github.com/gardener/gardener/pkg/gardenlet/operation/botanist" +) + +var _ = Describe("Join", func() { + Describe("#GetGardenerNodeAgentSecretName", func() { + var ( + ctx = context.Background() + + fakeClient client.Client + b *botanist.GardenadmBotanist + options *Options + + shoot *gardencorev1beta1.Shoot + cluster *extensionsv1alpha1.Cluster + ) + + BeforeEach(func() { + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + b = &botanist.GardenadmBotanist{ + Botanist: &botanistpkg.Botanist{ + Operation: &operationpkg.Operation{ + ShootClientSet: fakekubernetes.NewClientSetBuilder().WithClient(fakeClient).Build(), + }, + }, + } + options = &Options{} + + shoot = &gardencorev1beta1.Shoot{} + + shootRaw, err := runtime.Encode(&json.Serializer{}, shoot) + Expect(err).NotTo(HaveOccurred()) + + cluster = &extensionsv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "kube-system"}, + Spec: extensionsv1alpha1.ClusterSpec{ + Shoot: runtime.RawExtension{Raw: shootRaw}, + }, + } + }) + + When("Cluster object does not exist", func() { + It("should fail when worker pool name is not set", func() { + secretName, err := GetGardenerNodeAgentSecretName(ctx, options, b) + Expect(err).To(MatchError(ContainSubstring(`clusters.extensions.gardener.cloud "kube-system" not found`))) + Expect(secretName).To(BeEmpty()) + }) + + When("worker pool name is set", func() { + BeforeEach(func() { + options.WorkerPoolName = "some-pool-name" + }) + + It("should fail because there are no gardener-node-agent secrets", func() { + secretName, err := GetGardenerNodeAgentSecretName(ctx, options, b) + Expect(err).To(MatchError(ContainSubstring("no gardener-node-agent secrets found"))) + Expect(secretName).To(BeEmpty()) + }) + + It("should succeed when there are gardener-node-agent secrets", func() { + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-node-agent-test-pool", + Namespace: "kube-system", + Labels: map[string]string{ + "gardener.cloud/role": "operating-system-config", + "worker.gardener.cloud/pool": options.WorkerPoolName, + }, + }} + + Expect(fakeClient.Create(ctx, secret)).To(Succeed()) + + secretName, err := GetGardenerNodeAgentSecretName(ctx, options, b) + Expect(err).NotTo(HaveOccurred()) + Expect(secretName).To(Equal(secret.Name)) + }) + }) + }) + + When("Cluster object exists", func() { + BeforeEach(func() { + Expect(fakeClient.Create(ctx, cluster)).To(Succeed()) + }) + + When("control plane node should be joined", func() { + BeforeEach(func() { + options.ControlPlane = true + }) + + It("should fail when there is no control plane pool in the Shoot spec", func() { + secretName, err := GetGardenerNodeAgentSecretName(ctx, options, b) + Expect(err).To(MatchError(ContainSubstring("no control plane worker pool found in Shoot manifest"))) + Expect(secretName).To(BeEmpty()) + }) + + When("control plane pool is in Shoot spec", func() { + BeforeEach(func() { + shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, gardencorev1beta1.Worker{Name: "cp", ControlPlane: &gardencorev1beta1.WorkerControlPlane{}}) + + var err error + cluster.Spec.Shoot.Raw, err = runtime.Encode(&json.Serializer{}, shoot) + Expect(err).NotTo(HaveOccurred()) + + Expect(fakeClient.Update(ctx, cluster)).To(Succeed()) + }) + + It("should fail because there are no gardener-node-agent secrets", func() { + secretName, err := GetGardenerNodeAgentSecretName(ctx, options, b) + Expect(err).To(MatchError(ContainSubstring("no gardener-node-agent secrets found"))) + Expect(secretName).To(BeEmpty()) + }) + + It("should succeed when there are gardener-node-agent secrets", func() { + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-node-agent-cp", + Namespace: "kube-system", + Labels: map[string]string{ + "gardener.cloud/role": "operating-system-config", + "worker.gardener.cloud/pool": "cp", + }, + }} + + Expect(fakeClient.Create(ctx, secret)).To(Succeed()) + + secretName, err := GetGardenerNodeAgentSecretName(ctx, options, b) + Expect(err).NotTo(HaveOccurred()) + Expect(secretName).To(Equal(secret.Name)) + }) + }) + }) + + When("worker node should be joined", func() { + It("should fail when there is no non-control-plane pool in Shoot manifest", func() { + secretName, err := GetGardenerNodeAgentSecretName(ctx, options, b) + Expect(err).To(MatchError(ContainSubstring("no non-control-plane pool found in Shoot manifest"))) + Expect(secretName).To(BeEmpty()) + }) + + When("there are non-control-plane pools in Shoot manifest", func() { + BeforeEach(func() { + shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, + gardencorev1beta1.Worker{Name: "worker1"}, + gardencorev1beta1.Worker{Name: "worker2"}, + ) + + var err error + cluster.Spec.Shoot.Raw, err = runtime.Encode(&json.Serializer{}, shoot) + Expect(err).NotTo(HaveOccurred()) + + Expect(fakeClient.Update(ctx, cluster)).To(Succeed()) + }) + + It("should fail because there are no gardener-node-agent secrets", func() { + secretName, err := GetGardenerNodeAgentSecretName(ctx, options, b) + Expect(err).To(MatchError(ContainSubstring("no gardener-node-agent secrets found"))) + Expect(secretName).To(BeEmpty()) + }) + + It("should succeed when there are gardener-node-agent secrets", func() { + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-node-agent-worker1", + Namespace: "kube-system", + Labels: map[string]string{ + "gardener.cloud/role": "operating-system-config", + "worker.gardener.cloud/pool": "worker1", + }, + }} + + Expect(fakeClient.Create(ctx, secret)).To(Succeed()) + + secretName, err := GetGardenerNodeAgentSecretName(ctx, options, b) + Expect(err).NotTo(HaveOccurred()) + Expect(secretName).To(Equal(secret.Name)) + }) + }) + }) + }) + }) +}) diff --git a/pkg/gardenadm/cmd/join/options.go b/pkg/gardenadm/cmd/join/options.go index 0974ffe7875..3bf1b4f8d1f 100644 --- a/pkg/gardenadm/cmd/join/options.go +++ b/pkg/gardenadm/cmd/join/options.go @@ -23,9 +23,11 @@ type Options struct { BootstrapToken string // CertificateAuthority is the CA bundle of the control plane. CertificateAuthority []byte - // GardenerNodeAgentSecretName is the name of the secret from which gardener-node-agent should download its - // operating system configuration. - GardenerNodeAgentSecretName string + // WorkerPoolName is the name of the worker pool to use for the join command. If not provided, the node is assigned + // to the first worker pool in the Shoot manifest. + WorkerPoolName string + // ControlPlane indicates whether the node should be joined as a control plane node. + ControlPlane bool } // ParseArgs parses the arguments to the options. @@ -42,8 +44,9 @@ func (o *Options) Validate() error { if len(o.BootstrapToken) == 0 { return fmt.Errorf("must provide a bootstrap token") } - if len(o.GardenerNodeAgentSecretName) == 0 { - return fmt.Errorf("must provide a secret name for gardener-node-agent") + + if o.ControlPlane && o.WorkerPoolName != "" { + return fmt.Errorf("cannot provide a worker pool name when joining a control plane node") } return nil @@ -55,5 +58,6 @@ func (o *Options) Complete() error { return nil } func (o *Options) addFlags(fs *pflag.FlagSet) { fs.BytesBase64Var(&o.CertificateAuthority, "ca-certificate", nil, "Base64-encoded certificate authority bundle of the control plane") fs.StringVar(&o.BootstrapToken, "bootstrap-token", "", "Bootstrap token for joining the cluster (create it with 'gardenadm token' on a control plane node)") - fs.StringVar(&o.GardenerNodeAgentSecretName, "gardener-node-agent-secret-name", "", "Name of the Secret from which gardener-node-agent should download its operating system configuration") + fs.StringVarP(&o.WorkerPoolName, "worker-pool-name", "w", "", "Name of the worker pool to assign the joining node.") + fs.BoolVar(&o.ControlPlane, "control-plane", false, "Create a new control plane instance on this node") } diff --git a/pkg/gardenadm/cmd/join/options_test.go b/pkg/gardenadm/cmd/join/options_test.go index bdbf0a5a7b7..a69d77bcec1 100644 --- a/pkg/gardenadm/cmd/join/options_test.go +++ b/pkg/gardenadm/cmd/join/options_test.go @@ -35,7 +35,8 @@ var _ = Describe("Options", func() { Describe("#Validate", func() { It("should succeed when proper values were provided", func() { options.BootstrapToken = "some-token" - options.GardenerNodeAgentSecretName = "some-secret-name" + options.WorkerPoolName = "some-pool-name" + Expect(options.Validate()).To(Succeed()) }) @@ -43,9 +44,12 @@ var _ = Describe("Options", func() { Expect(options.Validate()).To(MatchError(ContainSubstring("must provide a bootstrap token"))) }) - It("should fail when no node-agent secret name is provided", func() { + It("should fail when worker pool is provided when control plane instance should be joined", func() { options.BootstrapToken = "some-token" - Expect(options.Validate()).To(MatchError(ContainSubstring("must provide a secret name for gardener-node-agent"))) + options.WorkerPoolName = "some-pool-name" + options.ControlPlane = true + + Expect(options.Validate()).To(MatchError(ContainSubstring("cannot provide a worker pool name when joining a control plane node"))) }) }) diff --git a/pkg/gardenadm/cmd/temporary_client.go b/pkg/gardenadm/cmd/temporary_client.go new file mode 100644 index 00000000000..0814f0d1ac4 --- /dev/null +++ b/pkg/gardenadm/cmd/temporary_client.go @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "context" + "crypto/x509/pkix" + "fmt" + "net" + "path/filepath" + "time" + + "github.com/spf13/afero" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/gardenadm/botanist" + gardenletbootstraputil "github.com/gardener/gardener/pkg/gardenlet/bootstrap/util" + "github.com/gardener/gardener/pkg/utils/kubernetes/certificatesigningrequest" +) + +// NewClientSetFromBootstrapToken returns a Kubernetes client set based on the provided bootstrap token. +func NewClientSetFromBootstrapToken(controlPlaneAddress string, certificateAuthority []byte, bootstrapToken string, scheme *runtime.Scheme) (kubernetes.Interface, error) { + return kubernetes.NewWithConfig(kubernetes.WithRESTConfig(&rest.Config{ + Host: controlPlaneAddress, + TLSClientConfig: rest.TLSClientConfig{CAData: certificateAuthority}, + BearerToken: bootstrapToken, + }), kubernetes.WithClientOptions(client.Options{Scheme: scheme}), kubernetes.WithDisabledCachedClient()) +} + +// NewClientFromBytes is an alias for kubernetes.NewClientFromBytes. +// Exposed for testing. +var NewClientFromBytes = kubernetes.NewClientFromBytes + +// InitializeTemporaryClientSet acquires a short-lived client certificate-based kubeconfig. +func InitializeTemporaryClientSet(ctx context.Context, b *botanist.GardenadmBotanist, bootstrapClientSet kubernetes.Interface) (kubernetes.Interface, error) { + bootstrapKubeconfig, cached, err := getCachedBootstrapKubeconfig(b) + if err != nil { + return nil, fmt.Errorf("failed retrieving cached bootstrap kubeconfig: %w", err) + } + + if !cached { + bootstrapKubeconfig, err = requestShortLivedBootstrapKubeconfig(ctx, b, bootstrapClientSet) + if err != nil { + return nil, fmt.Errorf("failed to request short-lived bootstrap kubeconfig via CertificateSigningRequest API: %w", err) + } + + if err := b.FS.WriteFile(cachedBootstrapKubeconfigPath(b.FS), bootstrapKubeconfig, 0600); err != nil { + return nil, fmt.Errorf("failed writing the retrieved bootstrap kubeconfig to a temporary file: %w", err) + } + } + + return NewClientFromBytes( + bootstrapKubeconfig, + kubernetes.WithClientOptions(client.Options{Scheme: bootstrapClientSet.Client().Scheme()}), + kubernetes.WithDisabledCachedClient(), + ) +} + +func cachedBootstrapKubeconfigPath(fs afero.Afero) string { + return filepath.Join(fs.GetTempDir(""), "gardenadm-bootstrap-kubeconfig") +} + +const bootstrapKubeconfigValidity = 10 * time.Minute + +func getCachedBootstrapKubeconfig(b *botanist.GardenadmBotanist) ([]byte, bool, error) { + fileInfo, err := b.FS.Stat(cachedBootstrapKubeconfigPath(b.FS)) + if err != nil || time.Since(fileInfo.ModTime()) > bootstrapKubeconfigValidity-2*time.Minute { + // We deliberately ignore the error here - this is just a best-effort attempt to cache the bootstrap kubeconfig. + // If the file doesn't exist, or we cannot read/find it for whatever reason, we just consider it as a cache + // miss. + // Otherwise, if the last modifications time of the file is older than the validity of the bootstrap kubeconfig, + // we consider it as expired and thus a cache miss. + return nil, false, nil //nolint:nilerr + } + + data, err := b.FS.ReadFile(cachedBootstrapKubeconfigPath(b.FS)) + if err != nil { + return nil, false, fmt.Errorf("failed reading the cached bootstrap kubeconfig: %w", err) + } + + return data, true, nil +} + +func requestShortLivedBootstrapKubeconfig(ctx context.Context, b *botanist.GardenadmBotanist, bootstrapClientSet kubernetes.Interface) ([]byte, error) { + commonNameSuffix := b.HostName + if b.Shoot != nil && b.Shoot.GetInfo() != nil { + commonNameSuffix = b.Shoot.GetInfo().Namespace + ":" + b.Shoot.GetInfo().Name + } + + certificateSubject := &pkix.Name{ + Organization: []string{v1beta1constants.ShootsGroup}, + CommonName: v1beta1constants.GardenadmUserNamePrefix + commonNameSuffix, + } + + certData, privateKeyData, _, err := certificatesigningrequest.RequestCertificate(ctx, b.Logger, bootstrapClientSet.Kubernetes(), certificateSubject, []string{}, []net.IP{}, &metav1.Duration{Duration: bootstrapKubeconfigValidity}, "gardenadm-csr-") + if err != nil { + return nil, fmt.Errorf("unable to bootstrap the kubeconfig: %w", err) + } + + return gardenletbootstraputil.CreateKubeconfigWithClientCertificate(bootstrapClientSet.RESTConfig(), privateKeyData, certData) +} diff --git a/pkg/gardenadm/cmd/temporary_client_test.go b/pkg/gardenadm/cmd/temporary_client_test.go new file mode 100644 index 00000000000..ec263db2a19 --- /dev/null +++ b/pkg/gardenadm/cmd/temporary_client_test.go @@ -0,0 +1,334 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package cmd_test + +import ( + "context" + "crypto/x509/pkix" + "fmt" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/afero" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + "k8s.io/client-go/testing" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/client/kubernetes" + fakekubernetes "github.com/gardener/gardener/pkg/client/kubernetes/fake" + "github.com/gardener/gardener/pkg/gardenadm/botanist" + . "github.com/gardener/gardener/pkg/gardenadm/cmd" + operationpkg "github.com/gardener/gardener/pkg/gardenlet/operation" + botanistpkg "github.com/gardener/gardener/pkg/gardenlet/operation/botanist" + shootpkg "github.com/gardener/gardener/pkg/gardenlet/operation/shoot" + "github.com/gardener/gardener/pkg/utils/test" +) + +var _ = Describe("Temporary Client", func() { + Describe("#InitializeTemporaryClientSet", func() { + var ( + ctx = context.Background() + b *botanist.GardenadmBotanist + bootstrapClientSet kubernetes.Interface + fakeKubernetesClientset *kubernetesfake.Clientset + fakeFS afero.Afero + shoot *gardencorev1beta1.Shoot + + cachedPath string + restConfig = &rest.Config{Host: "https://test-api-server"} + + csr *certificatesv1.CertificateSigningRequest + kubeconfig string + ) + + BeforeEach(func() { + fakeFS = afero.Afero{Fs: afero.NewMemMapFs()} + + shoot = &gardencorev1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shoot", + Namespace: "test-namespace", + }, + } + + b = &botanist.GardenadmBotanist{ + FS: fakeFS, + Botanist: &botanistpkg.Botanist{ + Operation: &operationpkg.Operation{ + Logger: log.Log.WithName("test"), + Shoot: &shootpkg.Shoot{}, + }, + }, + } + b.Shoot.SetInfo(shoot) + + fakeKubernetesClientset = kubernetesfake.NewClientset() + bootstrapClientSet = fakekubernetes.NewClientSetBuilder(). + WithKubernetes(fakeKubernetesClientset). + WithClient(fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build()). + WithRESTConfig(restConfig). + Build() + + cachedPath = filepath.Join(fakeFS.GetTempDir(""), "gardenadm-bootstrap-kubeconfig") + csr = &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardenadm-csr-test", + UID: "test-uid", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: []byte("fake-csr-request"), + }, + Status: certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }}, + Certificate: []byte("some-cert"), + }, + } + kubeconfig = ` +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority-data: some-ca + server: https://test-api-server + name: test-cluster +contexts: +- context: + cluster: test-cluster + user: test-user + name: test-context +current-context: test-context +users: +- name: test-user + user: + client-certificate-data: some-cert + client-key-data: some-key +` + + DeferCleanup(test.WithVar(&NewClientFromBytes, func([]byte, ...kubernetes.ConfigFunc) (kubernetes.Interface, error) { + return fakekubernetes.NewClientSetBuilder().Build(), nil + })) + }) + + Context("when no cached kubeconfig exists", func() { + It("should successfully create a new client and cache the kubeconfig", func() { + fakeKubernetesClientset.PrependReactor("create", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + return true, csr, nil + }) + fakeKubernetesClientset.PrependReactor("get", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + return true, csr, nil + }) + + client, err := InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + Expect(err).NotTo(HaveOccurred()) + Expect(client).NotTo(BeNil()) + + // Verify kubeconfig was cached + exists, err := fakeFS.Exists(cachedPath) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + + It("should fail when CSR creation fails", func() { + fakeKubernetesClientset.PrependReactor("create", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("CSR creation failed") + }) + + _, err := InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to request short-lived bootstrap kubeconfig")) + }) + + It("should fail when CSR is denied", func() { + fakeKubernetesClientset.PrependReactor("create", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + csr := &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardenadm-csr-test", + UID: "test-uid", + }, + } + return true, csr, nil + }) + + fakeKubernetesClientset.PrependReactor("get", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + csr.Status.Conditions[0] = certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateDenied, + Status: corev1.ConditionTrue, + Reason: "TestDenial", + Message: "CSR denied for testing", + } + return true, csr, nil + }) + + _, err := InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to request short-lived bootstrap kubeconfig")) + }) + }) + + Context("when cached kubeconfig exists and is valid", func() { + BeforeEach(func() { + Expect(fakeFS.WriteFile(cachedPath, []byte(kubeconfig), 0600)).NotTo(HaveOccurred()) + }) + + It("should use cached kubeconfig and not make CSR request", func() { + // Track if CSR was called (it shouldn't be) + csrCalled := false + fakeKubernetesClientset.PrependReactor("create", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + csrCalled = true + return false, nil, nil + }) + + client, err := InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + Expect(err).NotTo(HaveOccurred()) + Expect(client).NotTo(BeNil()) + Expect(csrCalled).To(BeFalse(), "CSR should not be called when valid cache exists") + }) + }) + + Context("when cached kubeconfig exists but is expired", func() { + BeforeEach(func() { + expiredTime := time.Now().Add(-15 * time.Minute) + Expect(fakeFS.WriteFile(cachedPath, []byte(kubeconfig), 0600)).NotTo(HaveOccurred()) + Expect(fakeFS.Chtimes(cachedPath, expiredTime, expiredTime)).NotTo(HaveOccurred()) + }) + + It("should ignore expired cache and request new certificate", func() { + // Track if CSR was called (it should be) + csrCalled := false + fakeKubernetesClientset.PrependReactor("create", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + csrCalled = true + csr := &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardenadm-csr-test", + UID: "test-uid", + }, + } + return true, csr, nil + }) + + fakeKubernetesClientset.PrependReactor("get", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + return true, csr, nil + }) + + client, err := InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + Expect(err).NotTo(HaveOccurred()) + Expect(client).NotTo(BeNil()) + Expect(csrCalled).To(BeTrue(), "CSR should be called when cache is expired") + + // Verify new kubeconfig was cached (overwriting the old one) + exists, err := fakeFS.Exists(cachedPath) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + }) + + Context("when cached kubeconfig is corrupted", func() { + BeforeEach(func() { + Expect(fakeFS.WriteFile(cachedPath, []byte("corrupted kubeconfig content"), 0600)).NotTo(HaveOccurred()) + }) + + It("should fail to create client with corrupted kubeconfig", func() { + fakeErr := fmt.Errorf("corrupt") + + DeferCleanup(test.WithVar(&NewClientFromBytes, func([]byte, ...kubernetes.ConfigFunc) (kubernetes.Interface, error) { + return nil, fakeErr + })) + + _, err := InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + Expect(err).To(MatchError(fakeErr)) + }) + }) + + Context("when filesystem operations fail", func() { + It("should fail when unable to write kubeconfig to cache", func() { + // Make filesystem read-only to simulate write failure + b.FS = afero.Afero{Fs: afero.NewReadOnlyFs(afero.NewMemMapFs())} + + fakeKubernetesClientset.PrependReactor("create", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + return true, csr, nil + }) + fakeKubernetesClientset.PrependReactor("get", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + return true, csr, nil + }) + + _, err := InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed writing the retrieved bootstrap kubeconfig to a temporary file")) + }) + }) + + Context("when shoot info is nil", func() { + BeforeEach(func() { + b.Shoot.SetInfo(nil) + b.HostName = "test-hostname" + }) + + It("should use hostname as common name suffix when shoot info is not set", func() { + fakeKubernetesClientset.PrependReactor("create", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + return true, csr, nil + }) + fakeKubernetesClientset.PrependReactor("get", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + return true, csr, nil + }) + + client, err := InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + Expect(err).NotTo(HaveOccurred()) + Expect(client).NotTo(BeNil()) + + // Verify kubeconfig was cached + exists, err := fakeFS.Exists(cachedPath) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + }) + + Context("certificate subject verification", func() { + It("should use correct certificate subject when requesting CSR", func() { + var capturedSubject *pkix.Name + fakeKubernetesClientset.PrependReactor("create", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + // In a real scenario, we would capture the subject from the CSR request + // For testing purposes, we verify the expected subject is used + expectedSubject := &pkix.Name{ + Organization: []string{"gardener.cloud:system:shoots"}, + CommonName: "gardener.cloud:gardenadm:shoot:" + shoot.Namespace + ":" + shoot.Name, + } + capturedSubject = expectedSubject + + csr := &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardenadm-csr-test", + UID: "test-uid", + }, + } + return true, csr, nil + }) + + fakeKubernetesClientset.PrependReactor("get", "certificatesigningrequests", func(testing.Action) (bool, runtime.Object, error) { + return true, csr, nil + }) + + _, err := InitializeTemporaryClientSet(ctx, b, bootstrapClientSet) + Expect(err).NotTo(HaveOccurred()) + + // Verify the correct subject was used + Expect(capturedSubject).NotTo(BeNil()) + Expect(capturedSubject.Organization).To(Equal([]string{"gardener.cloud:system:shoots"})) + Expect(capturedSubject.CommonName).To(Equal("gardener.cloud:gardenadm:shoot:test-namespace:test-shoot")) + }) + }) + }) +}) diff --git a/pkg/gardenadm/cmd/token/create/create.go b/pkg/gardenadm/cmd/token/create/create.go index d895d193199..3357c78ca25 100644 --- a/pkg/gardenadm/cmd/token/create/create.go +++ b/pkg/gardenadm/cmd/token/create/create.go @@ -14,7 +14,6 @@ import ( bootstraptokenutil "k8s.io/cluster-bootstrap/token/util" "sigs.k8s.io/controller-runtime/pkg/client" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/gardenadm/cmd" tokenutils "github.com/gardener/gardener/pkg/gardenadm/cmd/token/utils" @@ -89,11 +88,7 @@ func run(ctx context.Context, opts *Options) error { switch { case opts.PrintJoinCommand: - output, err := printJoinCommand(ctx, opts, clientSet, secret) - if err != nil { - return fmt.Errorf("failed computing join command: %w", err) - } - fmt.Fprint(opts.Out, output) + fmt.Fprint(opts.Out, printJoinCommand(clientSet, secret)) case opts.PrintConnectCommand: fmt.Fprint(opts.Out, printConnectCommand(clientSet, secret)) @@ -105,31 +100,13 @@ func run(ctx context.Context, opts *Options) error { return nil } -func printJoinCommand(ctx context.Context, opts *Options, clientSet kubernetes.Interface, bootstrapTokenSecret *corev1.Secret) (string, error) { - secretList := &corev1.SecretList{} - if err := clientSet.Client().List(ctx, secretList, client.InNamespace(metav1.NamespaceSystem), client.MatchingLabels{ - v1beta1constants.GardenRole: v1beta1constants.GardenRoleOperatingSystemConfig, - v1beta1constants.LabelWorkerPool: opts.WorkerPoolName, - }); err != nil { - return "", fmt.Errorf("failed listing gardener-node-agent secrets: %w", err) - } - - if len(secretList.Items) == 0 { - return "", fmt.Errorf("no gardener-node-agent secrets found for worker pool %q", opts.WorkerPoolName) - } - - gardenerNodeAgentSecret := secretList.Items[0] - if len(secretList.Items) > 1 { - opts.Log.V(1).Info("Multiple gardener-node-agent secrets found, using the first one", "secretName", gardenerNodeAgentSecret.Name) - } - - return fmt.Sprintf(`gardenadm join --bootstrap-token %s --ca-certificate "%s" --gardener-node-agent-secret-name %s %s +func printJoinCommand(clientSet kubernetes.Interface, bootstrapTokenSecret *corev1.Secret) string { + return fmt.Sprintf(`gardenadm join --bootstrap-token %s --ca-certificate "%s" %s `, bootstraptoken.FromSecretData(bootstrapTokenSecret.Data), utils.EncodeBase64(clientSet.RESTConfig().CAData), - gardenerNodeAgentSecret.Name, clientSet.RESTConfig().Host, - ), nil + ) } func printConnectCommand(clientSet kubernetes.Interface, bootstrapTokenSecret *corev1.Secret) string { diff --git a/pkg/gardenadm/cmd/token/create/create_test.go b/pkg/gardenadm/cmd/token/create/create_test.go index 5d879b6b576..a0cfdcd7411 100644 --- a/pkg/gardenadm/cmd/token/create/create_test.go +++ b/pkg/gardenadm/cmd/token/create/create_test.go @@ -95,25 +95,11 @@ var _ = Describe("Create", func() { When("the join command should be printed", func() { BeforeEach(func() { Expect(command.Flags().Set("print-join-command", "true")).To(Succeed()) - Expect(command.Flags().Set("worker-pool-name", "test-pool")).To(Succeed()) - }) - - It("should fail because there are no gardener-node-agent-secrets", func() { - Expect(command.RunE(command, []string{token})).To(MatchError(ContainSubstring("no gardener-node-agent secrets found"))) }) It("should successfully print the join command", func() { - Expect(fakeClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ - Name: "gardener-node-agent-test-pool", - Namespace: "kube-system", - Labels: map[string]string{ - "gardener.cloud/role": "operating-system-config", - "worker.gardener.cloud/pool": "test-pool", - }, - }})).To(Succeed()) - Expect(command.RunE(command, []string{token})).To(Succeed()) - Eventually(stdOut).Should(Say(`gardenadm join --bootstrap-token abcdef.1234567890abcdef --ca-certificate "Y2EtZGF0YQ==" --gardener-node-agent-secret-name gardener-node-agent-test-pool some-host + Eventually(stdOut).Should(Say(`gardenadm join --bootstrap-token abcdef.1234567890abcdef --ca-certificate "Y2EtZGF0YQ==" some-host `)) }) }) diff --git a/pkg/gardenadm/cmd/token/create/options.go b/pkg/gardenadm/cmd/token/create/options.go index 1b54a087abc..3e97b5695d8 100644 --- a/pkg/gardenadm/cmd/token/create/options.go +++ b/pkg/gardenadm/cmd/token/create/options.go @@ -27,9 +27,6 @@ type Options struct { Description string // Validity duration of the bootstrap token. Validity time.Duration - // WorkerPoolName is the name of the worker pool to use for the join command. If not provided, it is defaulted to - // 'worker'. - WorkerPoolName string // Shoot contains the namespace and the name of the Shoot which should be connected to Gardener. This is only // relevant for bootstrap tokens that are used for connecting self-hosted shoots via `gardenadm connect`. Shoot types.NamespacedName @@ -90,10 +87,6 @@ func (o *Options) Validate() error { return fmt.Errorf("maximum validity duration is %s", maxValidity) } - if o.PrintJoinCommand && len(o.WorkerPoolName) == 0 { - return fmt.Errorf("must specify a worker pool name when using --print-join-command") - } - if o.PrintConnectCommand && (len(o.Shoot.Namespace) == 0 || len(o.Shoot.Name) == 0) { return fmt.Errorf("must specify a shoot namespace and name when using --print-connect-command") } @@ -127,7 +120,6 @@ func (o *Options) addFlags(fs *pflag.FlagSet) { fs.DurationVarP(&o.Validity, "validity", "", time.Hour, "Validity duration of the bootstrap token. Minimum is 10m, maximum is 24h.") fs.BoolVarP(&o.PrintJoinCommand, "print-join-command", "j", false, "Instead of only printing the token, print the full machine-readable 'gardenadm join' command that can be copied and ran on a machine that should join the cluster") fs.BoolVarP(&o.PrintConnectCommand, "print-connect-command", "c", false, "Instead of only printing the token, print the full machine-readable 'gardenadm connect' command that can be ran on a machine of a cluster that should be connected to Gardener") - fs.StringVarP(&o.WorkerPoolName, "worker-pool-name", "w", "worker", "Name of the worker pool to use for the join command.") fs.StringVarP(&o.Shoot.Namespace, "shoot-namespace", "", "", "Namespace of the Shoot which should be connected to Gardener via 'gardenadm connect' with this bootstrap token") fs.StringVarP(&o.Shoot.Name, "shoot-name", "", "", "Name of the Shoot which should be connected to Gardener via 'gardenadm connect' with this bootstrap token") } diff --git a/pkg/gardenadm/cmd/token/create/options_test.go b/pkg/gardenadm/cmd/token/create/options_test.go index 7ae1cbacdaa..a93af694c54 100644 --- a/pkg/gardenadm/cmd/token/create/options_test.go +++ b/pkg/gardenadm/cmd/token/create/options_test.go @@ -41,7 +41,6 @@ var _ = Describe("Options", func() { options.Token.Combined = "abcdef.abcdef1234567890" options.Validity = time.Hour options.PrintJoinCommand = true - options.WorkerPoolName = "worker-pool" Expect(options.Validate()).To(Succeed()) }) @@ -66,18 +65,6 @@ var _ = Describe("Options", func() { Expect(options.Validate()).To(MatchError(ContainSubstring("maximum validity duration is 24h0m0s"))) }) - When("the print-join-command flag is set", func() { - BeforeEach(func() { - options.PrintJoinCommand = true - options.Token.Combined = "abcdef.abcdef1234567890" - }) - - It("should an error when no worker pool name is provided", func() { - options.Validity = time.Hour - Expect(options.Validate()).To(MatchError(ContainSubstring("must specify a worker pool name when using --print-join-command"))) - }) - }) - When("the print-connect-command flag is set", func() { BeforeEach(func() { options.PrintConnectCommand = true diff --git a/pkg/gardenlet/operation/botanist/shootsystem.go b/pkg/gardenlet/operation/botanist/shootsystem.go index 496918f5d42..eb100130508 100644 --- a/pkg/gardenlet/operation/botanist/shootsystem.go +++ b/pkg/gardenlet/operation/botanist/shootsystem.go @@ -28,6 +28,7 @@ func (b *Botanist) DefaultShootSystem() shootsystem.Interface { Extensions: extensions, ExternalClusterDomain: b.Shoot.ExternalClusterDomain, IsWorkerless: b.Shoot.IsWorkerless, + IsSelfHosted: b.Shoot.IsSelfHosted(), KubernetesVersion: b.Shoot.KubernetesVersion, Object: b.Shoot.GetInfo(), ProjectName: b.Garden.Project.Name, diff --git a/pkg/nodeagent/controller/operatingsystemconfig/reconciler.go b/pkg/nodeagent/controller/operatingsystemconfig/reconciler.go index 63d68b18e05..f2db6e8e45a 100644 --- a/pkg/nodeagent/controller/operatingsystemconfig/reconciler.go +++ b/pkg/nodeagent/controller/operatingsystemconfig/reconciler.go @@ -147,6 +147,23 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( return reconcile.Result{}, fmt.Errorf("failed extracting OSC from secret: %w", err) } + if node != nil { + nodeRole := "worker" + if slices.ContainsFunc(osc.Spec.Files, func(file extensionsv1alpha1.File) bool { + return file.Path == filepath.Join(kubeletcomponent.FilePathKubernetesManifests, "kube-apiserver.yaml") + }) { + nodeRole = "control-plane" + } + + log.Info("Setting node-role label on Node object", "role", nodeRole) + + patch := client.MergeFrom(node.DeepCopy()) + metav1.SetMetaDataLabel(&node.ObjectMeta, "node-role.kubernetes.io/"+nodeRole, "") + if err := r.Client.Patch(ctx, node, patch); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to add node-role label to node object: %w", err) + } + } + log.Info("Applying containerd configuration") if err := r.ReconcileContainerdConfig(ctx, log, osc); err != nil { return reconcile.Result{}, fmt.Errorf("failed reconciling containerd configuration: %w", err) diff --git a/pkg/resourcemanager/controller/csrapprover/reconciler.go b/pkg/resourcemanager/controller/csrapprover/reconciler.go index ff789560783..d6d2dfeef3b 100644 --- a/pkg/resourcemanager/controller/csrapprover/reconciler.go +++ b/pkg/resourcemanager/controller/csrapprover/reconciler.go @@ -232,14 +232,26 @@ func (r *Reconciler) mustApproveKubeletServing(ctx context.Context, csr *certifi } func (r *Reconciler) mustApproveKubeAPIServerClient(ctx context.Context, csr *certificatesv1.CertificateSigningRequest, x509cr *x509.CertificateRequest) (string, decision, error) { - // Handle CSRs for gardener-node-agent only. There are other "kube-apiserver-client" CSRs which must not be touched. - if !strings.HasPrefix(x509cr.Subject.CommonName, v1beta1constants.NodeAgentUserNamePrefix) { - return fmt.Sprintf("commonName %q is not prefixed with %q", x509cr.Subject.CommonName, v1beta1constants.NodeAgentUserNamePrefix), csrNoOpinion, nil + // Handle CSRs for gardener-node-agent and gardenadm clients only. There might other "kube-apiserver-client" CSRs + // which must not be touched. + switch { + case strings.HasPrefix(x509cr.Subject.CommonName, v1beta1constants.NodeAgentUserNamePrefix): + return r.mustApproveGardenerNodeAgentCSR(ctx, csr, x509cr) + + case strings.HasPrefix(x509cr.Subject.CommonName, v1beta1constants.GardenadmUserNamePrefix): + return mustApproveGardenadmCSR(csr, x509cr) + + default: + return fmt.Sprintf("commonName %q is not prefixed with %q or %q", x509cr.Subject.CommonName, v1beta1constants.NodeAgentUserNamePrefix, v1beta1constants.GardenadmUserNamePrefix), csrNoOpinion, nil } +} - machineName := strings.TrimPrefix(x509cr.Subject.CommonName, v1beta1constants.NodeAgentUserNamePrefix) +func (r *Reconciler) mustApproveGardenerNodeAgentCSR(ctx context.Context, csr *certificatesv1.CertificateSigningRequest, x509cr *x509.CertificateRequest) (string, decision, error) { + var ( + machineName = strings.TrimPrefix(x509cr.Subject.CommonName, v1beta1constants.NodeAgentUserNamePrefix) + nodeName = machineName + ) - nodeName := machineName if r.Config.MachineNamespace != nil { machine := &machinev1alpha1.Machine{} if err := r.SourceClient.Get(ctx, client.ObjectKey{Namespace: *r.Config.MachineNamespace, Name: machineName}, machine); err != nil { @@ -260,13 +272,27 @@ func (r *Reconciler) mustApproveKubeAPIServerClient(ctx context.Context, csr *ce return "", csrNoOpinion, fmt.Errorf("error getting node object with name %q: %w", nodeName, err) } } + case strings.HasPrefix(csr.Spec.Username, v1beta1constants.NodeAgentUserNamePrefix): if csr.Spec.Username != x509cr.Subject.CommonName { return fmt.Sprintf("username %q and commonName %q do not match", csr.Spec.Username, x509cr.Subject.CommonName), csrDenied, nil } + default: return fmt.Sprintf("username %q is not allowed to create CSRs for a gardener-node-agent", csr.Spec.Username), csrDenied, nil } return "all checks passed", csrApproved, nil } + +func mustApproveGardenadmCSR(csr *certificatesv1.CertificateSigningRequest, x509cr *x509.CertificateRequest) (string, decision, error) { + if !strings.HasPrefix(csr.Spec.Username, bootstraptokenapi.BootstrapUserPrefix) { + return fmt.Sprintf("username %q is not allowed to create CSRs for a gardenadm client", csr.Spec.Username), csrDenied, nil + } + + if len(x509cr.Subject.Organization) != 1 || !slices.Contains(x509cr.Subject.Organization, v1beta1constants.ShootsGroup) { + return "organization in CSR does not match nodes group", csrDenied, nil + } + + return "all checks passed", csrApproved, nil +} diff --git a/skaffold-gardenadm.yaml b/skaffold-gardenadm.yaml index c945f0640ed..aee74d8838d 100644 --- a/skaffold-gardenadm.yaml +++ b/skaffold-gardenadm.yaml @@ -1311,7 +1311,7 @@ deploy: - bash - -ec - | - for i in 0 1; do + for i in 0 1 2 3; do pod="machine-$i" kubectl -n "$SKAFFOLD_NAMESPACES" exec "$pod" -- sh -c 'rm -rf /gardenadm/resources && mkdir -p /gardenadm/resources' kubectl -n "$SKAFFOLD_NAMESPACES" cp dev-setup/gardenadm/resources/generated/.skaffold-image "$pod:/tmp/.skaffold-image" diff --git a/test/integration/nodeagent/operatingsystemconfig/operatingsystemconfig_test.go b/test/integration/nodeagent/operatingsystemconfig/operatingsystemconfig_test.go index 01e9ec7c686..36b05418ab7 100644 --- a/test/integration/nodeagent/operatingsystemconfig/operatingsystemconfig_test.go +++ b/test/integration/nodeagent/operatingsystemconfig/operatingsystemconfig_test.go @@ -882,6 +882,30 @@ units: {} }) }) + Context("node-role label", func() { + When("no static Kubernetes control-plane manifests are part of the OSC files", func() { + It("should add the 'worker' role label", func() { + Eventually(func(g Gomega) map[string]string { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(node), node)).To(Succeed()) + return node.Labels + }).Should(HaveKeyWithValue("node-role.kubernetes.io/worker", "")) + }) + }) + + When("static Kubernetes control-plane manifests are part of the OSC files", func() { + BeforeEach(func() { + operatingSystemConfig.Spec.Files = append(operatingSystemConfig.Spec.Files, extensionsv1alpha1.File{Path: "/etc/kubernetes/manifests/kube-apiserver.yaml"}) + }) + + It("should add the 'control-plane' role label", func() { + Eventually(func(g Gomega) map[string]string { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(node), node)).To(Succeed()) + return node.Labels + }).Should(HaveKeyWithValue("node-role.kubernetes.io/control-plane", "")) + }) + }) + }) + Context("in-place updates", func() { var ( kubeletUnit extensionsv1alpha1.Unit From 6534840905fe90d0f0f8de93657ad2ed564b4e6a Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 12 Nov 2025 15:56:27 +0100 Subject: [PATCH 096/176] Update prometheus-operator to v0.86.2 (minor) (#13388) * Update prometheus-operator to v0.86.2 * Adapt breaking API changes * Remove status field from istio ServiceMonitor test expectation Prometheus Operator omits status fields in the ServiceMonitor if zero. type ServiceMonitor struct { ... Status ConfigResourceStatus `json:"status,omitempty,omitzero"` } --------- Co-authored-by: rfranzke Co-authored-by: Victor Herrero Otal --- ...toring.coreos.com_alertmanagerconfigs.yaml | 3209 ++++++++++------- ...d-monitoring.coreos.com_alertmanagers.yaml | 691 ++-- ...crd-monitoring.coreos.com_podmonitors.yaml | 392 +- .../10-crd-monitoring.coreos.com_probes.yaml | 374 +- ...onitoring.coreos.com_prometheusagents.yaml | 1016 +++--- ...rd-monitoring.coreos.com_prometheuses.yaml | 1552 ++++---- ...monitoring.coreos.com_prometheusrules.yaml | 147 +- ...d-monitoring.coreos.com_scrapeconfigs.yaml | 3072 +++++++++------- ...monitoring.coreos.com_servicemonitors.yaml | 336 +- ...rd-monitoring.coreos.com_thanosrulers.yaml | 666 ++-- go.mod | 2 +- go.sum | 4 +- imagevector/containers.yaml | 4 +- .../test_charts/istiod_servicemonitor.yaml | 1 - .../monitoring/prometheus/seed/podmonitors.go | 4 +- .../prometheus/seed/podmonitors_test.go | 4 +- ...toring.coreos.com_alertmanagerconfigs.yaml | 3209 ++++++++++------- ...d-monitoring.coreos.com_alertmanagers.yaml | 691 ++-- ...crd-monitoring.coreos.com_podmonitors.yaml | 392 +- .../crd-monitoring.coreos.com_probes.yaml | 374 +- ...onitoring.coreos.com_prometheusagents.yaml | 1016 +++--- ...rd-monitoring.coreos.com_prometheuses.yaml | 1552 ++++---- ...monitoring.coreos.com_prometheusrules.yaml | 147 +- ...d-monitoring.coreos.com_scrapeconfigs.yaml | 3072 +++++++++------- ...monitoring.coreos.com_servicemonitors.yaml | 336 +- ...rd-monitoring.coreos.com_thanosrulers.yaml | 666 ++-- 26 files changed, 12793 insertions(+), 10136 deletions(-) diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagerconfigs.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagerconfigs.yaml index 88429306f60..cdeeec7f519 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagerconfigs.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagerconfigs.yaml @@ -43,17 +43,11 @@ spec: metadata: type: object spec: - description: |- - AlertmanagerConfigSpec is a specification of the desired behavior of the - Alertmanager configuration. - By default, the Alertmanager configuration only applies to alerts for which - the `namespace` label is equal to the namespace of the AlertmanagerConfig - resource (see the `.spec.alertmanagerConfigMatcherStrategy` field of the - Alertmanager CRD). + description: spec defines the specification of AlertmanagerConfigSpec properties: inhibitRules: description: |- - List of inhibition rules. The rules will only apply to alerts matching + inhibitRules defines the list of inhibition rules. The rules will only apply to alerts matching the resource's namespace. items: description: |- @@ -63,23 +57,24 @@ spec: properties: equal: description: |- - Labels that must have an equal value in the source and target alert for - the inhibition to take effect. + equal defines labels that must have an equal value in the source and target alert + for the inhibition to take effect. This ensures related alerts are properly grouped. items: type: string type: array sourceMatch: description: |- - Matchers for which one or more alerts have to exist for the inhibition - to take effect. The operator enforces that the alert matches the - resource's namespace. + sourceMatch defines matchers for which one or more alerts have to exist for the inhibition + to take effect. The operator enforces that the alert matches the resource's namespace. + These are the "trigger" alerts that cause other alerts to be inhibited. items: description: Matcher defines how to match on alert's labels. properties: matchType: description: |- - Match operation available with AlertManager >= v0.22.0 and - takes precedence over Regex (deprecated) if non-empty. + matchType defines the match operation available with AlertManager >= v0.22.0. + Takes precedence over Regex (deprecated) if non-empty. + Valid values: "=" (equality), "!=" (inequality), "=~" (regex match), "!~" (regex non-match). enum: - '!=' - = @@ -87,16 +82,20 @@ spec: - '!~' type: string name: - description: Label to match. + description: |- + name defines the label to match. + This specifies which alert label should be evaluated. minLength: 1 type: string regex: description: |- - Whether to match on equality (false) or regular-expression (true). + regex defines whether to match on equality (false) or regular-expression (true). Deprecated: for AlertManager >= v0.22.0, `matchType` should be used instead. type: boolean value: - description: Label value to match. + description: |- + value defines the label value to match. + This is the expected value for the specified label. type: string required: - name @@ -104,15 +103,17 @@ spec: type: array targetMatch: description: |- - Matchers that have to be fulfilled in the alerts to be muted. The - operator enforces that the alert matches the resource's namespace. + targetMatch defines matchers that have to be fulfilled in the alerts to be muted. + The operator enforces that the alert matches the resource's namespace. + When these conditions are met, matching alerts will be inhibited (silenced). items: description: Matcher defines how to match on alert's labels. properties: matchType: description: |- - Match operation available with AlertManager >= v0.22.0 and - takes precedence over Regex (deprecated) if non-empty. + matchType defines the match operation available with AlertManager >= v0.22.0. + Takes precedence over Regex (deprecated) if non-empty. + Valid values: "=" (equality), "!=" (inequality), "=~" (regex match), "!~" (regex non-match). enum: - '!=' - = @@ -120,16 +121,20 @@ spec: - '!~' type: string name: - description: Label to match. + description: |- + name defines the label to match. + This specifies which alert label should be evaluated. minLength: 1 type: string regex: description: |- - Whether to match on equality (false) or regular-expression (true). + regex defines whether to match on equality (false) or regular-expression (true). Deprecated: for AlertManager >= v0.22.0, `matchType` should be used instead. type: boolean value: - description: Label value to match. + description: |- + value defines the label value to match. + This is the expected value for the specified label. type: string required: - name @@ -138,40 +143,40 @@ spec: type: object type: array muteTimeIntervals: - description: List of MuteTimeInterval specifying when the routes should - be muted. + description: muteTimeIntervals defines the list of MuteTimeInterval + specifying when the routes should be muted. items: description: MuteTimeInterval specifies the periods in time when notifications will be muted properties: name: - description: Name of the time interval + description: name of the time interval type: string timeIntervals: - description: TimeIntervals is a list of TimeInterval + description: timeIntervals defines a list of TimeInterval items: description: TimeInterval describes intervals of time properties: daysOfMonth: - description: DaysOfMonth is a list of DayOfMonthRange + description: daysOfMonth defines a list of DayOfMonthRange items: description: DayOfMonthRange is an inclusive range of days of the month beginning at 1 properties: end: - description: End of the inclusive range + description: end of the inclusive range maximum: 31 minimum: -31 type: integer start: - description: Start of the inclusive range + description: start of the inclusive range maximum: 31 minimum: -31 type: integer type: object type: array months: - description: Months is a list of MonthRange + description: months defines a list of MonthRange items: description: |- MonthRange is an inclusive range of months of the year beginning in January @@ -180,24 +185,25 @@ spec: type: string type: array times: - description: Times is a list of TimeRange + description: times defines a list of TimeRange items: description: TimeRange defines a start and end time in 24hr format properties: endTime: - description: EndTime is the end time in 24hr format. + description: endTime defines the end time in 24hr + format. pattern: ^((([01][0-9])|(2[0-3])):[0-5][0-9])$|(^24:00$) type: string startTime: - description: StartTime is the start time in 24hr - format. + description: startTime defines the start time in + 24hr format. pattern: ^((([01][0-9])|(2[0-3])):[0-5][0-9])$|(^24:00$) type: string type: object type: array weekdays: - description: Weekdays is a list of WeekdayRange + description: weekdays defines a list of WeekdayRange items: description: |- WeekdayRange is an inclusive range of days of the week beginning on Sunday @@ -206,7 +212,7 @@ spec: type: string type: array years: - description: Years is a list of YearRange + description: years defines a list of YearRange items: description: YearRange is an inclusive range of years pattern: ^2\d{3}(?::2\d{3}|$) @@ -219,12 +225,12 @@ spec: type: object type: array receivers: - description: List of receivers. + description: receivers defines the list of receivers. items: description: Receiver defines one or more notification integrations. properties: discordConfigs: - description: List of Discord configurations. + description: discordConfigs defines the list of Slack configurations. items: description: |- DiscordConfig configures notifications via Discord. @@ -232,7 +238,7 @@ spec: properties: apiURL: description: |- - The secret's key that contains the Discord webhook URL. + apiURL defines the secret's key that contains the Discord webhook URL. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -258,25 +264,27 @@ spec: type: object x-kubernetes-map-type: atomic avatarURL: - description: The avatar url of the message sender. + description: avatarURL defines the avatar url of the message + sender. pattern: ^https?://.+$ type: string content: - description: The template of the content's body. + description: content defines the template of the content's + body. minLength: 1 type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -301,7 +309,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -310,12 +318,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -341,7 +349,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -368,7 +376,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -394,30 +402,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -440,8 +453,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -468,7 +481,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -496,12 +509,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -537,7 +550,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -545,33 +558,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -594,8 +607,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -621,12 +634,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -649,8 +662,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -676,11 +689,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -706,7 +720,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -717,7 +731,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -727,13 +741,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -770,7 +784,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -778,31 +792,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -825,8 +840,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -852,12 +867,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -880,8 +895,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -907,11 +922,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -936,7 +952,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -947,7 +963,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -957,22 +973,26 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: The template of the message's body. + description: message defines the template of the message's + body. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean title: - description: The template of the message's title. + description: title defines the template of the message's + title. type: string username: - description: The username of the message sender. + description: username defines the username of the message + sender. minLength: 1 type: string required: @@ -980,16 +1000,18 @@ spec: type: object type: array emailConfigs: - description: List of Email configurations. + description: emailConfigs defines the list of Email configurations. items: description: EmailConfig configures notifications via Email. properties: authIdentity: - description: The identity to use for authentication. + description: |- + authIdentity defines the identity to use for SMTP authentication. + This is typically used with PLAIN authentication mechanism. type: string authPassword: description: |- - The secret's key that contains the password to use for authentication. + authPassword defines the secret's key that contains the password to use for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -1016,7 +1038,8 @@ spec: x-kubernetes-map-type: atomic authSecret: description: |- - The secret's key that contains the CRAM-MD5 secret. + authSecret defines the secret's key that contains the CRAM-MD5 secret. + This is used for CRAM-MD5 authentication mechanism. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -1042,24 +1065,32 @@ spec: type: object x-kubernetes-map-type: atomic authUsername: - description: The username to use for authentication. + description: |- + authUsername defines the username to use for SMTP authentication. + This is used for SMTP AUTH when the server requires authentication. type: string from: - description: The sender address. + description: |- + from defines the sender address for email notifications. + This appears as the "From" field in the email header. type: string headers: description: |- - Further headers email header key/value pairs. Overrides any headers - previously set by the notification implementation. + headers defines additional email header key/value pairs. + These override any headers previously set by the notification implementation. items: description: KeyValue defines a (key, value) tuple. properties: key: - description: Key of the tuple. + description: |- + key defines the key of the tuple. + This is the identifier or name part of the key-value pair. minLength: 1 type: string value: - description: Value of the tuple. + description: |- + value defines the value of the tuple. + This is the data or content associated with the key. type: string required: - key @@ -1067,36 +1098,46 @@ spec: type: object type: array hello: - description: The hostname to identify to the SMTP server. + description: |- + hello defines the hostname to identify to the SMTP server. + This is used in the SMTP HELO/EHLO command during the connection handshake. type: string html: - description: The HTML body of the email notification. + description: |- + html defines the HTML body of the email notification. + This allows for rich formatting in the email content. type: string requireTLS: description: |- - The SMTP TLS requirement. + requireTLS defines the SMTP TLS requirement. Note that Go does not support unencrypted connections to remote SMTP endpoints. type: boolean sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean smarthost: - description: The SMTP host and port through which emails - are sent. E.g. example.com:25 + description: |- + smarthost defines the SMTP host and port through which emails are sent. + Format should be "hostname:port", e.g. "smtp.example.com:587". type: string text: - description: The text body of the email notification. + description: |- + text defines the plain text body of the email notification. + This provides a fallback for email clients that don't support HTML. type: string tlsConfig: - description: TLS configuration + description: |- + tlsConfig defines the TLS configuration for SMTP connections. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1119,8 +1160,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1145,12 +1186,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1173,8 +1214,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1199,11 +1240,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file - for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1228,7 +1270,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1239,7 +1281,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1249,17 +1291,20 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object to: - description: The email address to send notifications to. + description: |- + to defines the email address to send notifications to. + This is the recipient address for alert notifications. type: string type: object type: array msteamsConfigs: description: |- - List of MSTeams configurations. + msteamsConfigs defines the list of MSTeams configurations. It requires Alertmanager >= 0.26.0. items: description: |- @@ -1267,17 +1312,18 @@ spec: It requires Alertmanager >= 0.26.0. properties: httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for Teams webhook requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -1302,7 +1348,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1311,12 +1357,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1342,7 +1388,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1369,7 +1415,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -1395,30 +1441,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1441,8 +1492,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1469,7 +1520,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -1497,12 +1548,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1538,7 +1589,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1546,33 +1597,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1595,8 +1646,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -1622,12 +1673,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1650,8 +1701,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -1677,11 +1728,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -1707,7 +1759,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1718,7 +1770,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1728,13 +1780,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -1771,7 +1823,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1779,31 +1831,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1826,8 +1879,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1853,12 +1906,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1881,8 +1934,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1908,11 +1961,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -1937,7 +1991,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1948,7 +2002,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1958,27 +2012,35 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean summary: description: |- - Message summary template. + summary defines the message summary template for Teams notifications. + This provides a brief overview that appears in Teams notification previews. It requires Alertmanager >= 0.27.0. type: string text: - description: Message body template. + description: |- + text defines the message body template for Teams notifications. + This contains the detailed content of the Teams message. type: string title: - description: Message title template. + description: |- + title defines the message title template for Teams notifications. + This appears as the main heading of the Teams message card. type: string webhookUrl: - description: MSTeams webhook URL. + description: |- + webhookUrl defines the MSTeams webhook URL for sending notifications. + This is the incoming webhook URL configured in your Teams channel. properties: key: description: The key of the secret to select from. Must @@ -2007,26 +2069,27 @@ spec: type: array msteamsv2Configs: description: |- - List of MSTeamsV2 configurations. + msteamsv2Configs defines the list of MSTeamsV2 configurations. It requires Alertmanager >= 0.28.0. items: description: |- - MSTeamsV2Config configures notifications via Microsoft Teams using the new message format with adaptive cards as required by flows + MSTeamsV2Config configures notifications via Microsoft Teams using the new message format with adaptive cards as required by flows. See https://prometheus.io/docs/alerting/latest/configuration/#msteamsv2_config It requires Alertmanager >= 0.28.0. properties: httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for Teams webhook requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -2051,7 +2114,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -2060,12 +2123,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -2091,7 +2154,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -2118,7 +2181,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -2144,30 +2207,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2190,8 +2258,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -2218,7 +2286,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -2246,12 +2314,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -2287,7 +2355,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2295,33 +2363,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2344,8 +2412,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -2371,12 +2439,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2399,8 +2467,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -2426,11 +2494,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -2456,7 +2525,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2467,7 +2536,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2477,13 +2546,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -2520,7 +2589,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2528,31 +2597,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2575,8 +2645,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -2602,12 +2672,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2630,8 +2700,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -2657,11 +2727,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -2686,7 +2757,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2697,7 +2768,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2707,24 +2778,31 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean text: - description: Message body template. + description: |- + text defines the message body template for adaptive card notifications. + This contains the detailed content displayed in the Teams adaptive card format. minLength: 1 type: string title: - description: Message title template. + description: |- + title defines the message title template for adaptive card notifications. + This appears as the main heading in the Teams adaptive card. minLength: 1 type: string webhookURL: - description: MSTeams incoming webhook URL. + description: |- + webhookURL defines the MSTeams incoming webhook URL for adaptive card notifications. + This webhook must support the newer adaptive cards format required by Teams flows. properties: key: description: The key of the secret to select from. Must @@ -2750,24 +2828,25 @@ spec: type: object type: array name: - description: Name of the receiver. Must be unique across all - items from the list. + description: name defines the name of the receiver. Must be + unique across all items from the list. minLength: 1 type: string opsgenieConfigs: - description: List of OpsGenie configurations. + description: opsgenieConfigs defines the list of OpsGenie configurations. items: description: |- OpsGenieConfig configures notifications via OpsGenie. See https://prometheus.io/docs/alerting/latest/configuration/#opsgenie_config properties: actions: - description: Comma separated list of actions that will - be available for the alert. + description: |- + actions defines a comma separated list of actions that will be available for the alert. + These appear as action buttons in the OpsGenie interface. type: string apiKey: description: |- - The secret's key that contains the OpsGenie API key. + apiKey defines the secret's key that contains the OpsGenie API key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -2793,23 +2872,32 @@ spec: type: object x-kubernetes-map-type: atomic apiURL: - description: The URL to send OpsGenie API requests to. + description: |- + apiURL defines the URL to send OpsGenie API requests to. + When not specified, defaults to the standard OpsGenie API endpoint. type: string description: - description: Description of the incident. + description: |- + description defines the detailed description of the incident. + This provides additional context beyond the message field. type: string details: - description: A set of arbitrary key/value pairs that provide - further detail about the incident. + description: |- + details defines a set of arbitrary key/value pairs that provide further detail about the incident. + These appear as additional fields in the OpsGenie alert. items: description: KeyValue defines a (key, value) tuple. properties: key: - description: Key of the tuple. + description: |- + key defines the key of the tuple. + This is the identifier or name part of the key-value pair. minLength: 1 type: string value: - description: Value of the tuple. + description: |- + value defines the value of the tuple. + This is the data or content associated with the key. type: string required: - key @@ -2817,21 +2905,23 @@ spec: type: object type: array entity: - description: Optional field that can be used to specify - which domain alert is related to. + description: |- + entity defines an optional field that can be used to specify which domain alert is related to. + This helps group related alerts together in OpsGenie. type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for OpsGenie API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -2856,7 +2946,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -2865,12 +2955,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -2896,7 +2986,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -2923,7 +3013,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -2949,30 +3039,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2995,8 +3090,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -3023,7 +3118,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -3051,12 +3146,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3092,7 +3187,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3100,33 +3195,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3149,8 +3244,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -3176,12 +3271,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3204,8 +3299,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -3231,11 +3326,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -3261,7 +3357,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -3272,7 +3368,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3282,13 +3378,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -3325,7 +3421,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3333,31 +3429,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3380,8 +3477,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -3407,12 +3504,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3435,8 +3532,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -3462,11 +3559,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -3491,7 +3589,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -3502,7 +3600,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3512,97 +3610,129 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: Alert text limited to 130 characters. + description: |- + message defines the alert text limited to 130 characters. + This appears as the main alert title in OpsGenie. type: string note: - description: Additional alert note. + description: |- + note defines an additional alert note. + This provides supplementary information about the alert. type: string priority: - description: Priority level of alert. Possible values - are P1, P2, P3, P4, and P5. + description: |- + priority defines the priority level of alert. + Possible values are P1, P2, P3, P4, and P5, where P1 is highest priority. type: string responders: - description: List of responders responsible for notifications. + description: |- + responders defines the list of responders responsible for notifications. + These determine who gets notified when the alert is created. items: description: |- OpsGenieConfigResponder defines a responder to an incident. One of `id`, `name` or `username` has to be defined. properties: id: - description: ID of the responder. + description: |- + id defines the unique identifier of the responder. + This corresponds to the responder's ID within OpsGenie. type: string name: - description: Name of the responder. + description: |- + name defines the display name of the responder. + This is used when the responder is identified by name rather than ID. type: string type: - description: Type of responder. + description: |- + type defines the type of responder. + Valid values include "user", "team", "schedule", and "escalation". + This determines how OpsGenie interprets the other identifier fields. + enum: + - team + - teams + - user + - escalation + - schedule minLength: 1 type: string username: - description: Username of the responder. + description: |- + username defines the username of the responder. + This is typically used for user-type responders when identifying by username. type: string required: - type type: object type: array sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean source: - description: Backlink to the sender of the notification. + description: |- + source defines the backlink to the sender of the notification. + This helps identify where the alert originated from. type: string tags: - description: Comma separated list of tags attached to - the notifications. + description: |- + tags defines a comma separated list of tags attached to the notifications. + These help categorize and filter alerts within OpsGenie. type: string updateAlerts: description: |- - Whether to update message and description of the alert in OpsGenie if it already exists + updateAlerts defines Whether to update message and description of the alert in OpsGenie if it already exists By default, the alert is never updated in OpsGenie, the new message only appears in activity log. type: boolean type: object type: array pagerdutyConfigs: - description: List of PagerDuty configurations. + description: pagerdutyConfigs defines the List of PagerDuty + configurations. items: description: |- PagerDutyConfig configures notifications via PagerDuty. See https://prometheus.io/docs/alerting/latest/configuration/#pagerduty_config properties: class: - description: The class/type of the event. + description: class defines the class/type of the event. type: string client: - description: Client identification. + description: client defines the client identification. type: string clientURL: - description: Backlink to the sender of notification. + description: clientURL defines the backlink to the sender + of notification. type: string component: - description: The part or component of the affected system - that is broken. + description: component defines the part or component of + the affected system that is broken. type: string description: - description: Description of the incident. + description: description of the incident. type: string details: - description: Arbitrary key/value pairs that provide further - detail about the incident. + description: details defines the arbitrary key/value pairs + that provide further detail about the incident. items: description: KeyValue defines a (key, value) tuple. properties: key: - description: Key of the tuple. + description: |- + key defines the key of the tuple. + This is the identifier or name part of the key-value pair. minLength: 1 type: string value: - description: Value of the tuple. + description: |- + value defines the value of the tuple. + This is the data or content associated with the key. type: string required: - key @@ -3610,20 +3740,20 @@ spec: type: object type: array group: - description: A cluster or grouping of sources. + description: group defines a cluster or grouping of sources. type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -3648,7 +3778,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -3657,12 +3787,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -3688,7 +3818,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -3715,7 +3845,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -3741,30 +3871,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3787,8 +3922,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -3815,7 +3950,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -3843,12 +3978,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3884,7 +4019,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3892,33 +4027,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3941,8 +4076,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -3968,12 +4103,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3996,8 +4131,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -4023,11 +4158,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -4053,7 +4189,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4064,7 +4200,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4074,13 +4210,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -4117,7 +4253,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4125,31 +4261,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4172,8 +4309,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -4199,12 +4336,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4227,8 +4364,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -4254,11 +4391,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -4283,7 +4421,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4294,7 +4432,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4304,51 +4442,55 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object pagerDutyImageConfigs: - description: A list of image details to attach that provide - further detail about an incident. + description: pagerDutyImageConfigs defines a list of image + details to attach that provide further detail about + an incident. items: description: PagerDutyImageConfig attaches images to an incident properties: alt: - description: Alt is the optional alternative text + description: alt is the optional alternative text for the image. type: string href: - description: Optional URL; makes the image a clickable - link. + description: href defines the optional URL; makes + the image a clickable link. type: string src: - description: Src of the image being attached to + description: src of the image being attached to the incident type: string type: object type: array pagerDutyLinkConfigs: - description: A list of link details to attach that provide - further detail about an incident. + description: pagerDutyLinkConfigs defines a list of link + details to attach that provide further detail about + an incident. items: description: PagerDutyLinkConfig attaches text links to an incident properties: alt: - description: Text that describes the purpose of - the link, and can be used as the link's text. + description: alt defines the text that describes + the purpose of the link, and can be used as the + link's text. type: string href: - description: Href is the URL of the link to be attached + description: href defines the URL of the link to + be attached type: string type: object type: array routingKey: description: |- - The secret's key that contains the PagerDuty integration key (when using + routingKey defines the secret's key that contains the PagerDuty integration key (when using Events API v2). Either this field or `serviceKey` needs to be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -4375,11 +4517,12 @@ spec: type: object x-kubernetes-map-type: atomic sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean serviceKey: description: |- - The secret's key that contains the PagerDuty service key (when using + serviceKey defines the secret's key that contains the PagerDuty service key (when using integration type "Prometheus"). Either this field or `routingKey` needs to be defined. The secret needs to be in the same namespace as the AlertmanagerConfig @@ -4407,49 +4550,53 @@ spec: type: object x-kubernetes-map-type: atomic severity: - description: Severity of the incident. + description: severity of the incident. type: string source: - description: Unique location of the affected system. + description: source defines the unique location of the + affected system. type: string url: - description: The URL to send requests to. + description: url defines the URL to send requests to. type: string type: object type: array pushoverConfigs: - description: List of Pushover configurations. + description: pushoverConfigs defines the list of Pushover configurations. items: description: |- PushoverConfig configures notifications via Pushover. See https://prometheus.io/docs/alerting/latest/configuration/#pushover_config properties: device: - description: The name of a device to send the notification - to + description: |- + device defines the name of a specific device to send the notification to. + If not specified, the notification is sent to all user's devices. type: string expire: description: |- - How long your notification will continue to be retried for, unless the user - acknowledges the notification. + expire defines how long your notification will continue to be retried for, + unless the user acknowledges the notification. Only applies to priority 2 notifications. pattern: ^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$ type: string html: - description: Whether notification message is HTML or plain - text. + description: |- + html defines whether notification message is HTML or plain text. + When true, the message can include HTML formatting tags. type: boolean httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for Pushover API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -4474,7 +4621,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -4483,12 +4630,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -4514,7 +4661,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -4541,7 +4688,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -4567,30 +4714,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4613,8 +4765,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -4641,7 +4793,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -4669,12 +4821,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -4710,7 +4862,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4718,33 +4870,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4767,8 +4919,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -4794,12 +4946,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4822,8 +4974,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -4849,11 +5001,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -4879,7 +5032,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4890,7 +5043,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4900,13 +5053,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -4943,7 +5096,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4951,31 +5104,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4998,8 +5152,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5025,12 +5179,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5053,8 +5207,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5080,11 +5234,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -5109,7 +5264,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5120,7 +5275,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5130,37 +5285,45 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: Notification message. + description: |- + message defines the notification message content. + This is the main body text of the Pushover notification. type: string priority: - description: Priority, see https://pushover.net/api#priority + description: |- + priority defines the notification priority level. + See https://pushover.net/api#priority for valid values and behavior. type: string retry: description: |- - How often the Pushover servers will send the same notification to the user. - Must be at least 30 seconds. + retry defines how often the Pushover servers will send the same notification to the user. + Must be at least 30 seconds. Only applies to priority 2 notifications. pattern: ^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$ type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean sound: - description: The name of one of the sounds supported by - device clients to override the user's default sound - choice + description: |- + sound defines the name of one of the sounds supported by device clients. + This overrides the user's default sound choice for this notification. type: string title: - description: Notification title. + description: |- + title defines the notification title displayed in the Pushover message. + This appears as the bold header text in the notification. type: string token: description: |- - The secret's key that contains the registered application's API token, see https://pushover.net/apps. + token defines the secret's key that contains the registered application's API token. + See https://pushover.net/apps for application registration. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. Either `token` or `tokenFile` is required. @@ -5188,25 +5351,30 @@ spec: x-kubernetes-map-type: atomic tokenFile: description: |- - The token file that contains the registered application's API token, see https://pushover.net/apps. + tokenFile defines the token file that contains the registered application's API token. + See https://pushover.net/apps for application registration. Either `token` or `tokenFile` is required. It requires Alertmanager >= v0.26.0. type: string ttl: - description: The time to live definition for the alert - notification + description: |- + ttl defines the time to live for the alert notification. + This determines how long the notification remains active before expiring. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string url: - description: A supplementary URL shown alongside the message. + description: |- + url defines a supplementary URL shown alongside the message. + This creates a clickable link within the Pushover notification. type: string urlTitle: - description: A title for supplementary URL, otherwise - just the URL is shown + description: |- + urlTitle defines a title for the supplementary URL. + If not specified, the raw URL is shown instead. type: string userKey: description: |- - The secret's key that contains the recipient user's user key. + userKey defines the secret's key that contains the recipient user's user key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. Either `userKey` or `userKeyFile` is required. @@ -5234,7 +5402,7 @@ spec: x-kubernetes-map-type: atomic userKeyFile: description: |- - The user key file that contains the recipient user's user key. + userKeyFile defines the user key file that contains the recipient user's user key. Either `userKey` or `userKeyFile` is required. It requires Alertmanager >= v0.26.0. type: string @@ -5242,7 +5410,7 @@ spec: type: array rocketchatConfigs: description: |- - List of RocketChat configurations. + rocketchatConfigs defines the list of RocketChat configurations. It requires Alertmanager >= 0.28.0. items: description: |- @@ -5250,22 +5418,29 @@ spec: It requires Alertmanager >= 0.28.0. properties: actions: - description: Actions to include in the message. + description: |- + actions defines interactive actions to include in the message. + These appear as buttons that users can click to trigger responses. items: description: RocketChatActionConfig defines actions for RocketChat messages. properties: msg: - description: The message to send when the button - is clicked. + description: |- + msg defines the message to send when the button is clicked. + This allows the button to post a predefined message to the channel. minLength: 1 type: string text: - description: The button text. + description: |- + text defines the button text displayed to users. + This is the label that appears on the interactive button. minLength: 1 type: string url: - description: The URL the button links to. + description: |- + url defines the URL the button links to when clicked. + This creates a clickable button that opens the specified URL. pattern: ^https?://.+$ type: string type: object @@ -5273,57 +5448,69 @@ spec: type: array apiURL: description: |- - The API URL for RocketChat. + apiURL defines the API URL for RocketChat. Defaults to https://open.rocket.chat/ if not specified. pattern: ^https?://.+$ type: string channel: - description: The channel to send alerts to. + description: |- + channel defines the channel to send alerts to. + This can be a channel name (e.g., "#alerts") or a direct message recipient. minLength: 1 type: string color: - description: The message color. + description: |- + color defines the message color displayed in RocketChat. + This appears as a colored bar alongside the message. minLength: 1 type: string emoji: - description: If provided, the avatar will be displayed - as an emoji. + description: |- + emoji defines the emoji to be displayed as an avatar. + If provided, this emoji will be used instead of the default avatar or iconURL. minLength: 1 type: string fields: - description: Additional fields for the message. + description: |- + fields defines additional fields for the message attachment. + These appear as structured key-value pairs within the message. items: description: RocketChatFieldConfig defines additional fields for RocketChat messages. properties: short: - description: Whether this field should be a short - field. + description: |- + short defines whether this field should be a short field. + When true, the field may be displayed inline with other short fields to save space. type: boolean title: - description: The title of this field. + description: |- + title defines the title of this field. + This appears as bold text labeling the field content. minLength: 1 type: string value: - description: The value of this field, displayed - underneath the title value. + description: |- + value defines the value of this field, displayed underneath the title. + This contains the actual data or content for the field. minLength: 1 type: string type: object minItems: 1 type: array httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for RocketChat API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -5348,7 +5535,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -5357,12 +5544,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -5388,7 +5575,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -5415,7 +5602,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -5441,30 +5628,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5487,8 +5679,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5515,7 +5707,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5543,12 +5735,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5584,7 +5776,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5592,33 +5784,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5641,8 +5833,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -5668,12 +5860,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5696,8 +5888,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -5723,11 +5915,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -5753,7 +5946,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5764,7 +5957,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5774,13 +5967,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -5817,7 +6010,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5825,31 +6018,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5872,8 +6066,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5899,12 +6093,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5927,8 +6121,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5954,11 +6148,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -5983,7 +6178,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5994,7 +6189,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6004,47 +6199,65 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object iconURL: - description: Icon URL for the message. + description: |- + iconURL defines the icon URL for the message avatar. + This displays a custom image as the message sender's avatar. pattern: ^https?://.+$ type: string imageURL: - description: Image URL for the message. + description: |- + imageURL defines the image URL to display within the message. + This embeds an image directly in the message attachment. pattern: ^https?://.+$ type: string linkNames: - description: Whether to enable link names. + description: |- + linkNames defines whether to enable automatic linking of usernames and channels. + When true, @username and #channel references become clickable links. type: boolean sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean shortFields: - description: Whether to use short fields. + description: |- + shortFields defines whether to use short fields in the message layout. + When true, fields may be displayed side by side to save space. type: boolean text: - description: The message text to send, it is optional - because of attachments. + description: |- + text defines the message text to send. + This is optional because attachments can be used instead of or alongside text. minLength: 1 type: string thumbURL: - description: Thumbnail URL for the message. + description: |- + thumbURL defines the thumbnail URL for the message. + This displays a small thumbnail image alongside the message content. pattern: ^https?://.+$ type: string title: - description: The message title. + description: |- + title defines the message title displayed prominently in the message. + This appears as bold text at the top of the message attachment. minLength: 1 type: string titleLink: - description: The title link for the message. + description: |- + titleLink defines the URL that the title will link to when clicked. + This makes the message title clickable in the RocketChat interface. minLength: 1 type: string token: - description: The sender token. + description: |- + token defines the sender token for RocketChat authentication. + This is the personal access token or bot token used to authenticate API requests. properties: key: description: The key of the secret to select from. Must @@ -6068,7 +6281,9 @@ spec: type: object x-kubernetes-map-type: atomic tokenID: - description: The sender token ID. + description: |- + tokenID defines the sender token ID for RocketChat authentication. + This is the user ID associated with the token used for API requests. properties: key: description: The key of the secret to select from. Must @@ -6097,15 +6312,15 @@ spec: type: object type: array slackConfigs: - description: List of Slack configurations. + description: slackConfigs defines the list of Slack configurations. items: description: |- SlackConfig configures notifications via Slack. See https://prometheus.io/docs/alerting/latest/configuration/#slack_config properties: actions: - description: A list of Slack actions that are sent with - each notification. + description: actions defines a list of Slack actions that + are sent with each notification. items: description: |- SlackAction configures a single Slack action that is sent with each @@ -6115,37 +6330,64 @@ spec: properties: confirm: description: |- - SlackConfirmationField protect users from destructive actions or - particularly distinguished decisions by asking them to confirm their button - click one more time. - See https://api.slack.com/docs/interactive-message-field-guide#confirmation_fields - for more information. + confirm defines an optional confirmation dialog that appears before the action is executed. + When set, users must confirm their intent before the action proceeds. properties: dismissText: + description: |- + dismissText defines the label for the cancel button in the dialog. + When not specified, defaults to "Cancel". This button cancels the action. type: string okText: + description: |- + okText defines the label for the confirmation button in the dialog. + When not specified, defaults to "Okay". This button proceeds with the action. type: string text: + description: |- + text defines the main message displayed in the confirmation dialog. + This should be a clear question or statement asking the user to confirm their action. minLength: 1 type: string title: + description: |- + title defines the title text displayed at the top of the confirmation dialog. + When not specified, a default title will be used. type: string required: - text type: object name: + description: |- + name defines a unique identifier for the action within the message. + This value is sent back to your application when the action is triggered. type: string style: + description: |- + style defines the visual appearance of the action element. + Valid values include "default", "primary" (green), and "danger" (red). type: string text: + description: |- + text defines the user-visible label displayed on the action element. + For buttons, this is the button text. For select menus, this is the placeholder text. minLength: 1 type: string type: + description: |- + type defines the type of interactive component. + Common values include "button" for clickable buttons and "select" for dropdown menus. minLength: 1 type: string url: + description: |- + url defines the URL to open when the action is triggered. + Only applicable for button-type actions. When set, clicking the button opens this URL. type: string value: + description: |- + value defines the payload sent when the action is triggered. + This data is included in the callback sent to your application. type: string required: - text @@ -6154,7 +6396,7 @@ spec: type: array apiURL: description: |- - The secret's key that contains the Slack webhook URL. + apiURL defines the secret's key that contains the Slack webhook URL. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -6180,18 +6422,25 @@ spec: type: object x-kubernetes-map-type: atomic callbackId: + description: callbackId defines an identifier for the + message used in interactive components. type: string channel: - description: The channel or user to send notifications - to. + description: channel defines the channel or user to send + notifications to. type: string color: + description: |- + color defines the color of the left border of the Slack message attachment. + Can be a hex color code (e.g., "#ff0000") or a predefined color name. type: string fallback: + description: fallback defines a plain-text summary of + the attachment for clients that don't support attachments. type: string fields: - description: A list of Slack fields that are sent with - each notification. + description: fields defines a list of Slack fields that + are sent with each notification. items: description: |- SlackField configures a single Slack field that is sent with each notification. @@ -6200,11 +6449,21 @@ spec: See https://api.slack.com/docs/message-attachments#fields for more information. properties: short: + description: |- + short determines whether this field can be displayed alongside other short fields. + When true, Slack may display this field side by side with other short fields. + When false or not specified, the field takes the full width of the message. type: boolean title: + description: |- + title defines the label or header text displayed for this field. + This appears as bold text above the field value in the Slack message. minLength: 1 type: string value: + description: |- + value defines the content or data displayed for this field. + This appears below the title and can contain plain text or Slack markdown. minLength: 1 type: string required: @@ -6213,19 +6472,21 @@ spec: type: object type: array footer: + description: footer defines small text displayed at the + bottom of the message attachment. type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -6250,7 +6511,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6259,12 +6520,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -6290,7 +6551,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -6317,7 +6578,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -6343,30 +6604,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6389,8 +6655,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -6417,7 +6683,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -6445,12 +6711,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6486,7 +6752,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6494,33 +6760,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6543,8 +6809,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -6570,12 +6836,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6598,8 +6864,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -6625,11 +6891,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -6655,7 +6922,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6666,7 +6933,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6676,13 +6943,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -6719,7 +6986,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6727,31 +6994,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6774,8 +7042,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -6801,12 +7069,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6829,8 +7097,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -6856,11 +7124,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -6885,7 +7154,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6896,7 +7165,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6906,44 +7175,72 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object iconEmoji: + description: iconEmoji defines the emoji to use as the + bot's avatar (e.g., ":ghost:"). type: string iconURL: + description: iconURL defines the URL to an image to use + as the bot's avatar. type: string imageURL: + description: imageURL defines the URL to an image file + that will be displayed inside the message attachment. type: string linkNames: + description: |- + linkNames enables automatic linking of channel names and usernames in the message. + When true, @channel and @username will be converted to clickable links. type: boolean mrkdwnIn: + description: |- + mrkdwnIn defines which fields should be parsed as Slack markdown. + Valid values include "pretext", "text", and "fields". items: type: string type: array pretext: + description: pretext defines optional text that appears + above the message attachment block. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean shortFields: + description: |- + shortFields determines whether fields are displayed in a compact format. + When true, fields are shown side by side when possible. type: boolean text: + description: text defines the main text content of the + Slack message attachment. type: string thumbURL: + description: |- + thumbURL defines the URL to an image file that will be displayed as a thumbnail + on the right side of the message attachment. type: string title: + description: title defines the title text displayed in + the Slack message attachment. type: string titleLink: + description: titleLink defines the URL that the title + will link to when clicked. type: string username: + description: username defines the slack bot user name. type: string type: object type: array snsConfigs: - description: List of SNS configurations + description: snsConfigs defines the list of SNS configurations items: description: |- SNSConfig configures notifications via AWS SNS. @@ -6951,26 +7248,29 @@ spec: properties: apiURL: description: |- - The SNS API URL i.e. https://sns.us-east-2.amazonaws.com. + apiURL defines the SNS API URL, e.g. https://sns.us-east-2.amazonaws.com. If not specified, the SNS API URL from the SNS SDK will be used. type: string attributes: additionalProperties: type: string - description: SNS message attributes. + description: |- + attributes defines SNS message attributes as key-value pairs. + These provide additional metadata that can be used for message filtering and routing. type: object httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for SNS API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -6995,7 +7295,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -7004,12 +7304,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -7035,7 +7335,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -7062,7 +7362,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -7088,30 +7388,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7134,8 +7439,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -7162,7 +7467,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -7190,12 +7495,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7231,7 +7536,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7239,33 +7544,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7288,8 +7593,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -7315,12 +7620,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7343,8 +7648,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -7370,11 +7675,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -7400,7 +7706,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7411,7 +7717,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7421,13 +7727,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -7464,7 +7770,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7472,31 +7778,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7519,8 +7826,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -7546,12 +7853,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7574,8 +7881,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -7601,11 +7908,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -7630,7 +7938,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7641,7 +7949,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7651,29 +7959,33 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: The message content of the SNS notification. + description: |- + message defines the message content of the SNS notification. + This is the actual notification text that will be sent to subscribers. type: string phoneNumber: description: |- - Phone number if message is delivered via SMS in E.164 format. + phoneNumber defines the phone number if message is delivered via SMS in E.164 format. If you don't specify this value, you must specify a value for the TopicARN or TargetARN. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean sigv4: - description: Configures AWS's Signature Verification 4 - signing process to sign requests. + description: |- + sigv4 configures AWS's Signature Verification 4 signing process to sign requests. + This includes AWS credentials and region configuration for authentication. properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -7698,20 +8010,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used - to authenticate. + description: profile defines the named AWS profile + used to authenticate. type: string region: - description: Region is the AWS region. If blank, the - region from the default credentials chain used. + description: region defines the AWS region. If blank, + the region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used - to authenticate. + description: roleArn defines the named AWS profile + used to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -7735,25 +8047,31 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object subject: - description: Subject line when the message is delivered - to email endpoints. + description: |- + subject defines the subject line when the message is delivered to email endpoints. + This field is only used when sending to email subscribers of an SNS topic. type: string targetARN: description: |- - The mobile platform endpoint ARN if message is delivered via mobile notifications. - If you don't specify this value, you must specify a value for the topic_arn or PhoneNumber. + targetARN defines the mobile platform endpoint ARN if message is delivered via mobile notifications. + If you don't specify this value, you must specify a value for the TopicARN or PhoneNumber. type: string topicARN: description: |- - SNS topic ARN, i.e. arn:aws:sns:us-east-2:698519295917:My-Topic + topicARN defines the SNS topic ARN, e.g. arn:aws:sns:us-east-2:698519295917:My-Topic. If you don't specify this value, you must specify a value for the PhoneNumber or TargetARN. type: string type: object type: array telegramConfigs: - description: List of Telegram configurations. + description: telegramConfigs defines the list of Telegram configurations. items: description: |- TelegramConfig configures notifications via Telegram. @@ -7761,15 +8079,14 @@ spec: properties: apiURL: description: |- - The Telegram API URL i.e. https://api.telegram.org. - If not specified, default API URL will be used. + apiURL defines the Telegram API URL, e.g. https://api.telegram.org. + If not specified, the default Telegram API URL will be used. type: string botToken: description: |- - Telegram bot token. It is mutually exclusive with `botTokenFile`. + botToken defines the Telegram bot token. It is mutually exclusive with `botTokenFile`. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. - Either `botToken` or `botTokenFile` is required. properties: key: @@ -7795,30 +8112,35 @@ spec: x-kubernetes-map-type: atomic botTokenFile: description: |- - File to read the Telegram bot token from. It is mutually exclusive with `botToken`. + botTokenFile defines the file to read the Telegram bot token from. + It is mutually exclusive with `botToken`. Either `botToken` or `botTokenFile` is required. - It requires Alertmanager >= v0.26.0. type: string chatID: - description: The Telegram chat ID. + description: |- + chatID defines the Telegram chat ID where messages will be sent. + This can be a user ID, group ID, or channel ID (with @ prefix for public channels). format: int64 type: integer disableNotifications: - description: Disable telegram notifications + description: |- + disableNotifications controls whether Telegram notifications are sent silently. + When true, users will receive the message without notification sounds. type: boolean httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for Telegram API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -7843,7 +8165,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -7852,12 +8174,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -7883,7 +8205,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -7910,7 +8232,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -7936,30 +8258,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7982,8 +8309,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -8010,7 +8337,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -8038,12 +8365,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -8079,7 +8406,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8087,33 +8414,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8136,8 +8463,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -8163,12 +8490,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8191,8 +8518,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -8218,11 +8545,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -8248,7 +8576,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8259,7 +8587,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8269,13 +8597,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -8312,7 +8640,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8320,31 +8648,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8367,8 +8696,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -8394,12 +8723,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8422,8 +8751,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -8449,11 +8778,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -8478,7 +8808,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8489,7 +8819,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8499,36 +8829,44 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: Message template + description: |- + message defines the message template for the Telegram notification. + This is the content that will be sent to the specified chat. type: string messageThreadID: description: |- - The Telegram Group Topic ID. + messageThreadID defines the Telegram Group Topic ID for threaded messages. + This allows sending messages to specific topics within Telegram groups. It requires Alertmanager >= 0.26.0. format: int64 type: integer parseMode: - description: Parse mode for telegram message + description: |- + parseMode defines the parse mode for telegram message formatting. + Valid values are "MarkdownV2", "Markdown", and "HTML". + This determines how text formatting is interpreted in the message. enum: - MarkdownV2 - Markdown - HTML type: string sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean required: - chatID type: object type: array victoropsConfigs: - description: List of VictorOps configurations. + description: victoropsConfigs defines the list of VictorOps + configurations. items: description: |- VictorOpsConfig configures notifications via VictorOps. @@ -8536,7 +8874,7 @@ spec: properties: apiKey: description: |- - The secret's key that contains the API key to use when talking to the VictorOps API. + apiKey defines the secret's key that contains the API key to use when talking to the VictorOps API. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -8562,19 +8900,27 @@ spec: type: object x-kubernetes-map-type: atomic apiUrl: - description: The VictorOps API URL. + description: |- + apiUrl defines the VictorOps API URL. + When not specified, defaults to the standard VictorOps API endpoint. type: string customFields: - description: Additional custom fields for notification. + description: |- + customFields defines additional custom fields for notification. + These provide extra metadata that will be included with the VictorOps incident. items: description: KeyValue defines a (key, value) tuple. properties: key: - description: Key of the tuple. + description: |- + key defines the key of the tuple. + This is the identifier or name part of the key-value pair. minLength: 1 type: string value: - description: Value of the tuple. + description: |- + value defines the value of the tuple. + This is the data or content associated with the key. type: string required: - key @@ -8582,20 +8928,23 @@ spec: type: object type: array entityDisplayName: - description: Contains summary of the alerted problem. + description: |- + entityDisplayName contains a summary of the alerted problem. + This appears as the main title or identifier for the incident. type: string httpConfig: - description: The HTTP client's configuration. + description: httpConfig defines the HTTP client's configuration + for VictorOps API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -8620,7 +8969,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -8629,12 +8978,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -8660,7 +9009,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -8687,7 +9036,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -8713,30 +9062,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8759,8 +9113,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -8787,7 +9141,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -8815,12 +9169,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -8856,7 +9210,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8864,33 +9218,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8913,8 +9267,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -8940,12 +9294,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8968,8 +9322,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -8995,11 +9349,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -9025,7 +9380,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9036,7 +9391,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9046,13 +9401,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -9089,7 +9444,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9097,31 +9452,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9144,8 +9500,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9171,12 +9527,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9199,8 +9555,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9226,11 +9582,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -9255,7 +9612,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9266,7 +9623,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9276,58 +9633,61 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object messageType: - description: Describes the behavior of the alert (CRITICAL, - WARNING, INFO). + description: |- + messageType describes the behavior of the alert. + Valid values are "CRITICAL", "WARNING", and "INFO". type: string monitoringTool: - description: The monitoring tool the state message is - from. + description: |- + monitoringTool defines the monitoring tool the state message is from. + This helps identify the source system that generated the alert. type: string routingKey: - description: A key used to map the alert to a team. + description: |- + routingKey defines a key used to map the alert to a team. + This determines which VictorOps team will receive the alert notification. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean stateMessage: - description: Contains long explanation of the alerted - problem. + description: |- + stateMessage contains a long explanation of the alerted problem. + This provides detailed context about the incident. type: string type: object type: array webexConfigs: - description: List of Webex configurations. + description: webexConfigs defines the list of Webex configurations. items: description: |- WebexConfig configures notification via Cisco Webex See https://prometheus.io/docs/alerting/latest/configuration/#webex_config properties: apiURL: - description: |- - The Webex Teams API URL i.e. https://webexapis.com/v1/messages - Provide if different from the default API URL. + description: apiURL defines the Webex Teams API URL i.e. + https://webexapis.com/v1/messages pattern: ^https?://.+$ type: string httpConfig: - description: |- - The HTTP client's configuration. - You must supply the bot token via the `httpConfig.authorization` field. + description: httpConfig defines the HTTP client's configuration. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -9352,7 +9712,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -9361,12 +9721,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -9392,7 +9752,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -9419,7 +9779,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -9445,30 +9805,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9491,8 +9856,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9519,7 +9884,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -9547,12 +9912,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -9588,7 +9953,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9596,33 +9961,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9645,8 +10010,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -9672,12 +10037,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9700,8 +10065,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -9727,11 +10092,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -9757,7 +10123,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9768,7 +10134,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9778,13 +10144,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -9821,7 +10187,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9829,31 +10195,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9876,8 +10243,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9903,12 +10270,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9931,8 +10298,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9958,11 +10325,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -9987,7 +10355,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9998,7 +10366,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10008,45 +10376,47 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: Message template + description: message defines the message template type: string roomID: - description: ID of the Webex Teams room where to send - the messages. + description: roomID defines the ID of the Webex Teams + room where to send the messages. minLength: 1 type: string sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean required: - roomID type: object type: array webhookConfigs: - description: List of webhook configurations. + description: webhookConfigs defines the List of webhook configurations. items: description: |- WebhookConfig configures notifications via a generic receiver supporting the webhook payload. See https://prometheus.io/docs/alerting/latest/configuration/#webhook_config properties: httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for webhook requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -10071,7 +10441,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -10080,12 +10450,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -10111,7 +10481,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -10138,7 +10508,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -10164,30 +10534,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10210,8 +10585,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -10238,7 +10613,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -10266,12 +10641,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -10307,7 +10682,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10315,33 +10690,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10364,8 +10739,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -10391,12 +10766,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10419,8 +10794,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -10446,11 +10821,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -10476,7 +10852,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10487,7 +10863,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10497,13 +10873,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -10540,7 +10916,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10548,31 +10924,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10595,8 +10972,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -10622,12 +10999,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10650,8 +11027,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -10677,11 +11054,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -10706,7 +11084,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10717,7 +11095,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10727,37 +11105,38 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object maxAlerts: - description: Maximum number of alerts to be sent per webhook - message. When 0, all alerts are included. + description: |- + maxAlerts defines the maximum number of alerts to be sent per webhook message. + When 0, all alerts are included in the webhook payload. format: int32 minimum: 0 type: integer sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean timeout: description: |- - The maximum time to wait for a webhook request to complete, before failing the - request and allowing it to be retried. + timeout defines the maximum time to wait for a webhook request to complete, + before failing the request and allowing it to be retried. It requires Alertmanager >= v0.28.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string url: description: |- - The URL to send HTTP POST requests to. `urlSecret` takes precedence over - `url`. One of `urlSecret` and `url` should be defined. + url defines the URL to send HTTP POST requests to. + urlSecret takes precedence over url. One of urlSecret and url should be defined. type: string urlSecret: description: |- - The secret's key that contains the webhook URL to send HTTP requests to. - `urlSecret` takes precedence over `url`. One of `urlSecret` and `url` - should be defined. + urlSecret defines the secret's key that contains the webhook URL to send HTTP requests to. + urlSecret takes precedence over url. One of urlSecret and url should be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -10785,17 +11164,20 @@ spec: type: object type: array wechatConfigs: - description: List of WeChat configurations. + description: wechatConfigs defines the list of WeChat configurations. items: description: |- WeChatConfig configures notifications via WeChat. See https://prometheus.io/docs/alerting/latest/configuration/#wechat_config properties: agentID: + description: |- + agentID defines the application agent ID within WeChat Work. + This identifies which WeChat Work application will send the notifications. type: string apiSecret: description: |- - The secret's key that contains the WeChat API key. + apiSecret defines the secret's key that contains the WeChat API key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -10821,23 +11203,28 @@ spec: type: object x-kubernetes-map-type: atomic apiURL: - description: The WeChat API URL. + description: |- + apiURL defines the WeChat API URL. + When not specified, defaults to the standard WeChat Work API endpoint. type: string corpID: - description: The corp id for authentication. + description: |- + corpID defines the corp id for authentication. + This is the unique identifier for your WeChat Work organization. type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for WeChat API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -10862,7 +11249,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -10871,12 +11258,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -10902,7 +11289,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -10929,7 +11316,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -10955,30 +11342,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11001,8 +11393,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -11029,7 +11421,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -11057,12 +11449,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -11098,7 +11490,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11106,33 +11498,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11155,8 +11547,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -11182,12 +11574,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11210,8 +11602,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -11237,11 +11629,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -11267,7 +11660,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11278,7 +11671,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11288,13 +11681,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -11331,7 +11724,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11339,31 +11732,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11386,8 +11780,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -11413,12 +11807,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11441,8 +11835,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -11468,11 +11862,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -11497,7 +11892,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11508,7 +11903,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11518,25 +11913,39 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: API request data as defined by the WeChat - API. + description: |- + message defines the API request data as defined by the WeChat API. + This contains the actual notification content to be sent. type: string messageType: + description: |- + messageType defines the type of message to send. + Valid values include "text", "markdown", and other WeChat Work supported message types. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean toParty: + description: |- + toParty defines the target department(s) to receive the notification. + Can be a single department ID or multiple department IDs separated by '|'. type: string toTag: + description: |- + toTag defines the target tag(s) to receive the notification. + Can be a single tag ID or multiple tag IDs separated by '|'. type: string toUser: + description: |- + toUser defines the target user(s) to receive the notification. + Can be a single user ID or multiple user IDs separated by '|'. type: string type: object type: array @@ -11546,25 +11955,25 @@ spec: type: array route: description: |- - The Alertmanager route definition for alerts matching the resource's + route defines the Alertmanager route definition for alerts matching the resource's namespace. If present, it will be added to the generated Alertmanager configuration as a first-level route. properties: activeTimeIntervals: - description: ActiveTimeIntervals is a list of MuteTimeInterval + description: activeTimeIntervals is a list of MuteTimeInterval names when this route should be active. items: type: string type: array continue: description: |- - Boolean indicating whether an alert should continue matching subsequent + continue defines the boolean indicating whether an alert should continue matching subsequent sibling nodes. It will always be overridden to true for the first-level route by the Prometheus operator. type: boolean groupBy: description: |- - List of labels to group by. + groupBy defines the list of labels to group by. Labels must not be repeated (unique list). Special label "..." (aggregate by all possible labels), if provided, must be the only element in the list. items: @@ -11572,19 +11981,19 @@ spec: type: array groupInterval: description: |- - How long to wait before sending an updated notification. + groupInterval defines how long to wait before sending an updated notification. Must match the regular expression`^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$` Example: "5m" type: string groupWait: description: |- - How long to wait before sending the initial notification. + groupWait defines how long to wait before sending the initial notification. Must match the regular expression`^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$` Example: "30s" type: string matchers: description: |- - List of matchers that the alert's labels should match. For the first + matchers defines the list of matchers that the alert's labels should match. For the first level route, the operator removes any existing equality and regexp matcher on the `namespace` label and adds a `namespace: ` matcher. @@ -11593,8 +12002,9 @@ spec: properties: matchType: description: |- - Match operation available with AlertManager >= v0.22.0 and - takes precedence over Regex (deprecated) if non-empty. + matchType defines the match operation available with AlertManager >= v0.22.0. + Takes precedence over Regex (deprecated) if non-empty. + Valid values: "=" (equality), "!=" (inequality), "=~" (regex match), "!~" (regex non-match). enum: - '!=' - = @@ -11602,47 +12012,44 @@ spec: - '!~' type: string name: - description: Label to match. + description: |- + name defines the label to match. + This specifies which alert label should be evaluated. minLength: 1 type: string regex: description: |- - Whether to match on equality (false) or regular-expression (true). + regex defines whether to match on equality (false) or regular-expression (true). Deprecated: for AlertManager >= v0.22.0, `matchType` should be used instead. type: boolean value: - description: Label value to match. + description: |- + value defines the label value to match. + This is the expected value for the specified label. type: string required: - name type: object type: array muteTimeIntervals: - description: |- - Note: this comment applies to the field definition above but appears - below otherwise it gets included in the generated manifest. - CRD schema doesn't support self-referential types for now (see - https://github.com/kubernetes/kubernetes/issues/62872). We have to use - an alternative type to circumvent the limitation. The downside is that - the Kube API can't validate the data beyond the fact that it is a valid - JSON representation. - MuteTimeIntervals is a list of MuteTimeInterval names that will mute this route when matched, + description: muteTimeIntervals is a list of MuteTimeInterval names + that will mute this route when matched, items: type: string type: array receiver: description: |- - Name of the receiver for this route. If not empty, it should be listed in + receiver defines the name of the receiver for this route. If not empty, it should be listed in the `receivers` field. type: string repeatInterval: description: |- - How long to wait before repeating the last notification. + repeatInterval defines how long to wait before repeating the last notification. Must match the regular expression`^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$` Example: "4h" type: string routes: - description: Child routes. + description: routes defines the child routes. items: x-kubernetes-preserve-unknown-fields: true type: array diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagers.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagers.yaml index 96fad4d0d5d..e57360088a4 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagers.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_alertmanagers.yaml @@ -74,12 +74,12 @@ spec: type: object spec: description: |- - Specification of the desired behavior of the Alertmanager cluster. More info: + spec defines the specification of the desired behavior of the Alertmanager cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the 'Alertmanager' container. + additionalArgs allows setting additional arguments for the 'Alertmanager' container. It is intended for e.g. activating hidden flags which are not supported by the dedicated configuration options yet. The arguments are passed as-is to the Alertmanager container which may cause issues if they are invalid or not supported @@ -88,25 +88,25 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for name-only - arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. Can + be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name type: object type: array additionalPeers: - description: AdditionalPeers allows injecting a set of additional + description: additionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. items: type: string type: array affinity: - description: If specified, the pod's scheduling constraints. + description: affinity defines the pod's scheduling constraints. properties: nodeAffinity: description: Describes node affinity scheduling rules for the @@ -1019,13 +1019,13 @@ spec: type: object alertmanagerConfigMatcherStrategy: description: |- - AlertmanagerConfigMatcherStrategy defines how AlertmanagerConfig objects + alertmanagerConfigMatcherStrategy defines how AlertmanagerConfig objects process incoming alerts. properties: type: default: OnNamespace description: |- - AlertmanagerConfigMatcherStrategyType defines the strategy used by + type defines the strategy used by AlertmanagerConfig objects to match alerts in the routes and inhibition rules. @@ -1038,7 +1038,7 @@ spec: type: object alertmanagerConfigNamespaceSelector: description: |- - Namespaces to be selected for AlertmanagerConfig discovery. If nil, only + alertmanagerConfigNamespaceSelector defines the namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. properties: matchExpressions: @@ -1085,8 +1085,8 @@ spec: type: object x-kubernetes-map-type: atomic alertmanagerConfigSelector: - description: AlertmanagerConfigs to be selected for to merge and configure - Alertmanager with. + description: alertmanagerConfigSelector defines the selector to be + used for to merge and configure Alertmanager with. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. @@ -1133,7 +1133,7 @@ spec: x-kubernetes-map-type: atomic alertmanagerConfiguration: description: |- - alertmanagerConfiguration specifies the configuration of Alertmanager. + alertmanagerConfiguration defines the configuration of Alertmanager. If defined, it takes precedence over the `configSecret` field. @@ -1141,20 +1141,23 @@ spec: in a breaking way. properties: global: - description: Defines the global parameters of the Alertmanager + description: global defines the global parameters of the Alertmanager configuration. properties: httpConfig: - description: HTTP client configuration. + description: httpConfig defines the default HTTP configuration. properties: authorization: description: |- - Authorization header configuration for the client. - This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. + authorization configures the Authorization header credentials used by + the client. + + Cannot be set at the same time as `basicAuth`, `bearerTokenSecret` or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials for + authentication. properties: key: description: The key of the secret to select from. Must @@ -1179,7 +1182,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1188,12 +1191,14 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. - This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. + basicAuth defines the Basic Authentication credentials used by the + client. + + Cannot be set at the same time as `authorization`, `bearerTokenSecret` or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1219,7 +1224,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1246,10 +1251,14 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client - for authentication. - The secret needs to be in the same namespace as the Alertmanager - object and accessible by the Prometheus Operator. + bearerTokenSecret defines a key of a Secret containing the bearer token + used by the client for authentication. The secret needs to be in the + same namespace as the custom resource and readable by the Prometheus + Operator. + + Cannot be set at the same time as `authorization`, `basicAuth` or `oauth2`. + + Deprecated: use `authorization` instead. properties: key: description: The key of the secret to select from. Must @@ -1272,30 +1281,38 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the client - should follow HTTP 3xx redirects. + description: |- + followRedirects defines whether the client should follow HTTP 3xx + redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch a - token for the targets. + description: |- + oauth2 defines the OAuth2 settings used by the client. + + It requires Prometheus >= 2.27.0. + + Cannot be set at the same time as `authorization`, `basicAuth` or `bearerTokenSecret`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1318,8 +1335,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1345,7 +1362,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -1373,12 +1390,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1413,7 +1430,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1421,33 +1438,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used - for the token request.' + description: scopes defines the OAuth2 scopes used + for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1470,8 +1487,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1497,12 +1514,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1525,8 +1542,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1552,11 +1569,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -1581,7 +1599,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1592,7 +1610,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1602,13 +1620,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch - the token from.' + description: tokenUrl defines the URL to fetch the + token from. minLength: 1 type: string required: @@ -1645,7 +1663,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1653,25 +1671,26 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server to + use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: tlsConfig defines the TLS configuration used + by the client. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1694,8 +1713,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1720,12 +1739,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1748,8 +1767,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1774,11 +1793,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file - for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1803,7 +1823,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1814,7 +1834,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1824,23 +1844,25 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object type: object jira: - description: The default configuration for Jira. + description: jira defines the default configuration for Jira. properties: apiURL: description: |- - The default Jira API URL. + apiURL defines the default Jira API URL. It requires Alertmanager >= v0.28.0. pattern: ^(http|https)://.+$ type: string type: object opsGenieApiKey: - description: The default OpsGenie API Key. + description: opsGenieApiKey defines the default OpsGenie API + Key. properties: key: description: The key of the secret to select from. Must @@ -1864,7 +1886,8 @@ spec: type: object x-kubernetes-map-type: atomic opsGenieApiUrl: - description: The default OpsGenie API URL. + description: opsGenieApiUrl defines the default OpsGenie API + URL. properties: key: description: The key of the secret to select from. Must @@ -1888,28 +1911,30 @@ spec: type: object x-kubernetes-map-type: atomic pagerdutyUrl: - description: The default Pagerduty URL. + description: pagerdutyUrl defines the default Pagerduty URL. + pattern: ^(http|https)://.+$ type: string resolveTimeout: description: |- - ResolveTimeout is the default value used by alertmanager if the alert does + resolveTimeout defines the default value used by alertmanager if the alert does not include EndsAt, after this time passes it can declare the alert as resolved if it has not been updated. This has no impact on alerts from Prometheus, as they always include EndsAt. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string rocketChat: - description: The default configuration for Rocket Chat. + description: rocketChat defines the default configuration + for Rocket Chat. properties: apiURL: description: |- - The default Rocket Chat API URL. + apiURL defines the default Rocket Chat API URL. It requires Alertmanager >= v0.28.0. pattern: ^(http|https)://.+$ type: string token: description: |- - The default Rocket Chat token. + token defines the default Rocket Chat token. It requires Alertmanager >= v0.28.0. properties: @@ -1936,7 +1961,7 @@ spec: x-kubernetes-map-type: atomic tokenID: description: |- - The default Rocket Chat Token ID. + tokenID defines the default Rocket Chat Token ID. It requires Alertmanager >= v0.28.0. properties: @@ -1963,7 +1988,7 @@ spec: x-kubernetes-map-type: atomic type: object slackApiUrl: - description: The default Slack API URL. + description: slackApiUrl defines the default Slack API URL. properties: key: description: The key of the secret to select from. Must @@ -1987,13 +2012,14 @@ spec: type: object x-kubernetes-map-type: atomic smtp: - description: Configures global SMTP parameters. + description: smtp defines global SMTP parameters. properties: authIdentity: - description: SMTP Auth using PLAIN + description: authIdentity represents SMTP Auth using PLAIN type: string authPassword: - description: SMTP Auth using LOGIN and PLAIN. + description: authPassword represents SMTP Auth using LOGIN + and PLAIN. properties: key: description: The key of the secret to select from. Must @@ -2017,7 +2043,7 @@ spec: type: object x-kubernetes-map-type: atomic authSecret: - description: SMTP Auth using CRAM-MD5. + description: authSecret represents SMTP Auth using CRAM-MD5. properties: key: description: The key of the secret to select from. Must @@ -2041,34 +2067,35 @@ spec: type: object x-kubernetes-map-type: atomic authUsername: - description: SMTP Auth using CRAM-MD5, LOGIN and PLAIN. - If empty, Alertmanager doesn't authenticate to the SMTP - server. + description: authUsername represents SMTP Auth using CRAM-MD5, + LOGIN and PLAIN. If empty, Alertmanager doesn't authenticate + to the SMTP server. type: string from: - description: The default SMTP From header field. + description: from defines the default SMTP From header + field. type: string hello: - description: The default hostname to identify to the SMTP - server. + description: hello defines the default hostname to identify + to the SMTP server. type: string requireTLS: description: |- - The default SMTP TLS requirement. + requireTLS defines the default SMTP TLS requirement. Note that Go does not support unencrypted connections to remote SMTP endpoints. type: boolean smartHost: - description: The default SMTP smarthost used for sending - emails. + description: smartHost defines the default SMTP smarthost + used for sending emails. properties: host: - description: Defines the host's address, it can be - a DNS name or a literal IP address. + description: host defines the host's address, it can + be a DNS name or a literal IP address. minLength: 1 type: string port: - description: Defines the host's port, it can be a - literal port number or a port name. + description: port defines the host's port, it can + be a literal port number or a port name. minLength: 1 type: string required: @@ -2076,15 +2103,16 @@ spec: - port type: object tlsConfig: - description: The default TLS configuration for SMTP receivers + description: tlsConfig defines the default TLS configuration + for SMTP receivers properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2107,8 +2135,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -2133,12 +2161,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2161,8 +2189,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -2187,11 +2215,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file - for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2216,7 +2245,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2227,7 +2256,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2237,26 +2266,29 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object type: object telegram: - description: The default Telegram config + description: telegram defines the default Telegram config properties: apiURL: description: |- - The default Telegram API URL. + apiURL defines he default Telegram API URL. It requires Alertmanager >= v0.24.0. pattern: ^(http|https)://.+$ type: string type: object victorops: - description: The default configuration for VictorOps. + description: victorops defines the default configuration for + VictorOps. properties: apiKey: - description: The default VictorOps API Key. + description: apiKey defines the default VictorOps API + Key. properties: key: description: The key of the secret to select from. Must @@ -2280,30 +2312,33 @@ spec: type: object x-kubernetes-map-type: atomic apiURL: - description: The default VictorOps API URL. + description: apiURL defines the default VictorOps API + URL. pattern: ^(http|https)://.+$ type: string type: object webex: - description: The default configuration for Jira. + description: webex defines the default configuration for Jira. properties: apiURL: description: |- - The default Webex API URL. + apiURL defines the is the default Webex API URL. It requires Alertmanager >= v0.25.0. pattern: ^(http|https)://.+$ type: string type: object wechat: - description: The default WeChat Config + description: wechat defines the default WeChat Config properties: apiCorpID: - description: The default WeChat API Corporate ID. + description: apiCorpID defines the default WeChat API + Corporate ID. minLength: 1 type: string apiSecret: - description: The default WeChat API Secret. + description: apiSecret defines the default WeChat API + Secret. properties: key: description: The key of the secret to select from. Must @@ -2328,7 +2363,7 @@ spec: x-kubernetes-map-type: atomic apiURL: description: |- - The default WeChat API URL. + apiURL defines he default WeChat API URL. The default value is "https://qyapi.weixin.qq.com/cgi-bin/" pattern: ^(http|https)://.+$ type: string @@ -2336,19 +2371,20 @@ spec: type: object name: description: |- - The name of the AlertmanagerConfig resource which is used to generate the Alertmanager configuration. + name defines the name of the AlertmanagerConfig custom resource which is used to generate the Alertmanager configuration. It must be defined in the same namespace as the Alertmanager object. The operator will not enforce a `namespace` label for routes and inhibition rules. minLength: 1 type: string templates: - description: Custom notification templates. + description: templates defines the custom notification templates. items: description: SecretOrConfigMap allows to specify data as a Secret or ConfigMap. Fields are mutually exclusive. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2371,7 +2407,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2399,53 +2436,56 @@ spec: type: object automountServiceAccountToken: description: |- - AutomountServiceAccountToken indicates whether a service account token should be automatically mounted in the pod. + automountServiceAccountToken defines whether a service account token should be automatically mounted in the pod. If the service account has `automountServiceAccountToken: true`, set the field to `false` to opt out of automounting API credentials. type: boolean baseImage: description: |- - Base image that is used to deploy pods, without tag. + baseImage that is used to deploy pods, without tag. Deprecated: use 'image' instead. type: string clusterAdvertiseAddress: description: |- - ClusterAdvertiseAddress is the explicit address to advertise in cluster. + clusterAdvertiseAddress defines the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 type: string clusterGossipInterval: - description: Interval between gossip attempts. + description: clusterGossipInterval defines the interval between gossip + attempts. pattern: ^(0|(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string clusterLabel: description: |- - Defines the identifier that uniquely identifies the Alertmanager cluster. + clusterLabel defines the identifier that uniquely identifies the Alertmanager cluster. You should only set it when the Alertmanager cluster includes Alertmanager instances which are external to this Alertmanager resource. In practice, the addresses of the external instances are provided via the `.spec.additionalPeers` field. type: string clusterPeerTimeout: - description: Timeout for cluster peering. + description: clusterPeerTimeout defines the timeout for cluster peering. pattern: ^(0|(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string clusterPushpullInterval: - description: Interval between pushpull attempts. + description: clusterPushpullInterval defines the interval between + pushpull attempts. pattern: ^(0|(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string clusterTLS: description: |- - Configures the mutual TLS configuration for the Alertmanager cluster's gossip protocol. + clusterTLS defines the mutual TLS configuration for the Alertmanager cluster's gossip protocol. It requires Alertmanager >= 0.24.0. properties: client: - description: Client-side configuration for mutual TLS. + description: client defines the client-side configuration for + mutual TLS. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2468,7 +2508,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2493,11 +2534,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2520,7 +2562,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2545,11 +2588,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2574,7 +2618,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2585,7 +2629,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2595,23 +2639,25 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object server: - description: Server-side configuration for mutual TLS. + description: server defines the server-side configuration for + mutual TLS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2634,7 +2680,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2660,7 +2707,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -2668,7 +2715,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -2678,14 +2725,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2708,7 +2755,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2734,21 +2782,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -2758,7 +2806,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -2766,7 +2814,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -2794,14 +2842,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -2814,7 +2864,7 @@ spec: type: object configMaps: description: |- - ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager + configMaps defines a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. Each ConfigMap is added to the StatefulSet definition as a volume named `configmap-`. The ConfigMaps are mounted into `/etc/alertmanager/configmaps/` in the 'alertmanager' container. @@ -2823,7 +2873,7 @@ spec: type: array configSecret: description: |- - ConfigSecret is the name of a Kubernetes Secret in the same namespace as the + configSecret defines the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains the configuration for this Alertmanager instance. If empty, it defaults to `alertmanager-`. @@ -2838,7 +2888,7 @@ spec: type: string containers: description: |- - Containers allows injecting additional containers. This is meant to + containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge @@ -4366,11 +4416,11 @@ spec: type: object type: array dnsConfig: - description: Defines the DNS configuration for the pods. + description: dnsConfig defines the DNS configuration for the pods. properties: nameservers: description: |- - A list of DNS name server IP addresses. + nameservers defines the list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. items: minLength: 1 @@ -4379,7 +4429,7 @@ spec: x-kubernetes-list-type: set options: description: |- - A list of DNS resolver options. + options defines the list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Resolution options given in Options will override those that appear in the base DNSPolicy. @@ -4388,11 +4438,11 @@ spec: of a pod. properties: name: - description: Name is required and must be unique. + description: name is required and must be unique. minLength: 1 type: string value: - description: Value is optional. + description: value is optional. type: string required: - name @@ -4403,7 +4453,7 @@ spec: x-kubernetes-list-type: map searches: description: |- - A list of DNS search domains for host-name lookup. + searches defines the list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. items: minLength: 1 @@ -4412,7 +4462,7 @@ spec: x-kubernetes-list-type: set type: object dnsPolicy: - description: Defines the DNS policy for the pods. + description: dnsPolicy defines the DNS policy for the pods. enum: - ClusterFirstWithHostNet - ClusterFirst @@ -4421,7 +4471,7 @@ spec: type: string enableFeatures: description: |- - Enable access to Alertmanager feature flags. By default, no features are enabled. + enableFeatures defines the Alertmanager's feature flags. By default, no features are enabled. Enabling features which are disabled by default is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice. @@ -4431,34 +4481,34 @@ spec: type: string type: array enableServiceLinks: - description: Indicates whether information about services should be - injected into pod's environment variables + description: enableServiceLinks defines whether information about + services should be injected into pod's environment variables type: boolean externalUrl: description: |- - The external URL the Alertmanager instances will be available under. This is + externalUrl defines the URL used to access the Alertmanager web service. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. type: string forceEnableClusterMode: description: |- - ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + forceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. type: boolean hostAliases: - description: Pods' hostAliases configuration + description: hostAliases Pods configuration items: description: |- HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. + description: hostnames defines hostnames for the above IP address. items: type: string type: array ip: - description: IP address of the host file entry. + description: ip defines the IP address of the host file entry. type: string required: - hostnames @@ -4470,7 +4520,7 @@ spec: x-kubernetes-list-type: map hostUsers: description: |- - HostUsers supports the user space in Kubernetes. + hostUsers supports the user space in Kubernetes. More info: https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/ @@ -4479,14 +4529,14 @@ spec: type: boolean image: description: |- - Image if specified has precedence over baseImage, tag and sha + image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Alertmanager is being configured. type: string imagePullPolicy: description: |- - Image pull policy for the 'alertmanager', 'init-config-reloader' and 'config-reloader' containers. + imagePullPolicy for the 'alertmanager', 'init-config-reloader' and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. enum: - "" @@ -4496,7 +4546,7 @@ spec: type: string imagePullSecrets: description: |- - An optional list of references to secrets in the same namespace + imagePullSecrets An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ items: @@ -4518,7 +4568,7 @@ spec: type: array initContainers: description: |- - InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. + initContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the Alertmanager configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ InitContainers described here modify an operator @@ -6047,18 +6097,19 @@ spec: type: object type: array limits: - description: Defines the limits command line flags when starting Alertmanager. + description: limits defines the limits command line flags when starting + Alertmanager. properties: maxPerSilenceBytes: description: |- - The maximum size of an individual silence as stored on disk. This corresponds to the Alertmanager's + maxPerSilenceBytes defines the maximum size of an individual silence as stored on disk. This corresponds to the Alertmanager's `--silences.max-per-silence-bytes` flag. It requires Alertmanager >= v0.28.0. pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ type: string maxSilences: description: |- - The maximum number active and pending silences. This corresponds to the + maxSilences defines the maximum number active and pending silences. This corresponds to the Alertmanager's `--silences.max-silences` flag. It requires Alertmanager >= v0.28.0. format: int32 @@ -6067,19 +6118,19 @@ spec: type: object listenLocal: description: |- - ListenLocal makes the Alertmanager server listen on loopback, so that it + listenLocal defines the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. Note this is only for the Alertmanager UI, not the gossip communication. type: boolean logFormat: - description: Log format for Alertmanager to be configured with. + description: logFormat for Alertmanager to be configured with. enum: - "" - logfmt - json type: string logLevel: - description: Log level for Alertmanager to be configured with. + description: logLevel for Alertmanager to be configured with. enum: - "" - debug @@ -6089,7 +6140,7 @@ spec: type: string minReadySeconds: description: |- - Minimum number of seconds for which a newly created pod should be ready + minReadySeconds defines the minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. If unset, pods will be considered available as soon as they are ready. @@ -6099,16 +6150,17 @@ spec: nodeSelector: additionalProperties: type: string - description: Define which Nodes the Pods are scheduled on. + description: nodeSelector defines which Nodes the Pods are scheduled + on. type: object paused: description: |- - If set to true all actions on the underlying managed objects are not + paused if set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. type: boolean persistentVolumeClaimRetentionPolicy: description: |- - The field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. + persistentVolumeClaimRetentionPolicy controls if and how PVCs are deleted during the lifecycle of a StatefulSet. The default behavior is all PVCs are retained. This is an alpha field from kubernetes 1.23 until 1.26 and a beta field from 1.26. It requires enabling the StatefulSetAutoDeletePVC feature gate. @@ -6131,7 +6183,7 @@ spec: type: object podMetadata: description: |- - PodMetadata configures labels and annotations which are propagated to the Alertmanager pods. + podMetadata defines labels and annotations which are propagated to the Alertmanager pods. The following items are reserved and cannot be overridden: * "alertmanager" label, set to the name of the Alertmanager instance. @@ -6145,7 +6197,7 @@ spec: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -6154,14 +6206,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -6172,21 +6224,22 @@ spec: portName: default: web description: |- - Port name used for the pods and governing service. + portName defines the port's name for the pods and governing service. Defaults to `web`. type: string priorityClassName: - description: Priority class assigned to the Pods + description: priorityClassName assigned to the Pods type: string replicas: description: |- - Size is the expected size of the alertmanager cluster. The controller will + replicas defines the expected size of the alertmanager cluster. The controller will eventually make the size of the running cluster equal to the expected size. format: int32 type: integer resources: - description: Define resources requests and limits for single Pods. + description: resources defines the resource requests and limits of + the Pods. properties: claims: description: |- @@ -6247,20 +6300,20 @@ spec: retention: default: 120h description: |- - Time duration Alertmanager shall retain data for. Default is '120h', + retention defines the time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours). pattern: ^(0|(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string routePrefix: description: |- - The route prefix Alertmanager registers HTTP handlers for. This is useful, + routePrefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`. type: string secrets: description: |- - Secrets is a list of Secrets in the same namespace as the Alertmanager + secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. Each Secret is added to the StatefulSet definition as a volume named `secret-`. The Secrets are mounted into `/etc/alertmanager/secrets/` in the 'alertmanager' container. @@ -6269,7 +6322,7 @@ spec: type: array securityContext: description: |- - SecurityContext holds pod-level security attributes and common container settings. + securityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: appArmorProfile: @@ -6503,37 +6556,37 @@ spec: type: object serviceAccountName: description: |- - ServiceAccountName is the name of the ServiceAccount to use to run the + serviceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods. type: string serviceName: description: |- - The name of the service name used by the underlying StatefulSet(s) as the governing service. + serviceName defines the service name used by the underlying StatefulSet(s) as the governing service. If defined, the Service must be created before the Alertmanager resource in the same namespace and it must define a selector that matches the pod labels. - If empty, the operator will create and manage a headless service named `alertmanager-operated` for Alermanager resources. + If empty, the operator will create and manage a headless service named `alertmanager-operated` for Alertmanager resources. When deploying multiple Alertmanager resources in the same namespace, it is recommended to specify a different value for each. See https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id for more details. minLength: 1 type: string sha: description: |- - SHA of Alertmanager container image to be deployed. Defaults to the value of `version`. + sha of Alertmanager container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set. Deprecated: use 'image' instead. The image digest can be specified as part of the image URL. type: string storage: description: |- - Storage is the definition of how storage will be used by the Alertmanager + storage defines the definition of how storage will be used by the Alertmanager instances. properties: disableMountSubPath: - description: 'Deprecated: subPath usage will be removed in a future - release.' + description: 'disableMountSubPath deprecated: subPath usage will + be removed in a future release.' type: boolean emptyDir: description: |- - EmptyDirVolumeSource to be used by the StatefulSet. + emptyDir to be used by the StatefulSet. If specified, it takes precedence over `ephemeral` and `volumeClaimTemplate`. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir properties: @@ -6560,7 +6613,7 @@ spec: type: object ephemeral: description: |- - EphemeralVolumeSource to be used by the StatefulSet. + ephemeral to be used by the StatefulSet. This is a beta field in k8s 1.21 and GA in 1.15. For lower versions, starting with k8s 1.19, it requires enabling the GenericEphemeralVolume feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes @@ -6805,7 +6858,7 @@ spec: type: object volumeClaimTemplate: description: |- - Defines the PVC spec to be used by the Prometheus StatefulSets. + volumeClaimTemplate defines the PVC spec to be used by the Prometheus StatefulSets. The easiest way to use a volume that cannot be automatically provisioned is to use a label selector alongside manually created PersistentVolumes. properties: @@ -6825,14 +6878,14 @@ spec: More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: - description: EmbeddedMetadata contains metadata relevant to - an EmbeddedResource. + description: metadata defines EmbeddedMetadata contains metadata + relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -6841,14 +6894,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -6858,7 +6911,7 @@ spec: type: object spec: description: |- - Defines the desired characteristics of a volume requested by a pod author. + spec defines the specification of the characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: accessModes: @@ -7054,7 +7107,7 @@ spec: type: string type: object status: - description: 'Deprecated: this field is never set.' + description: 'status is deprecated: this field is never set.' properties: accessModes: description: |- @@ -7239,13 +7292,13 @@ spec: type: object tag: description: |- - Tag of Alertmanager container image to be deployed. Defaults to the value of `version`. + tag of Alertmanager container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set. Deprecated: use 'image' instead. The image tag can be specified as part of the image URL. type: string terminationGracePeriodSeconds: description: |- - Optional duration in seconds the pod needs to terminate gracefully. + terminationGracePeriodSeconds defines the Optional duration in seconds the pod needs to terminate gracefully. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down) which may lead to data corruption. @@ -7254,7 +7307,7 @@ spec: minimum: 0 type: integer tolerations: - description: If specified, the pod's tolerations. + description: tolerations defines the pod's tolerations. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -7293,7 +7346,8 @@ spec: type: object type: array topologySpreadConstraints: - description: If specified, the pod's topology spread constraints. + description: topologySpreadConstraints defines the Pod's topology + spread constraints. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. @@ -7467,11 +7521,11 @@ spec: type: object type: array version: - description: Version the cluster should be on. + description: version the cluster should be on. type: string volumeMounts: description: |- - VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. + volumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the alertmanager container, that are generated as a result of StorageSpec objects. items: @@ -7538,7 +7592,7 @@ spec: type: array volumes: description: |- - Volumes allows configuration of additional volumes on the output StatefulSet definition. + volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: @@ -9424,28 +9478,30 @@ spec: type: object type: array web: - description: Defines the web command line flags when starting Alertmanager. + description: web defines the web command line flags when starting + Alertmanager. properties: getConcurrency: description: |- - Maximum number of GET requests processed concurrently. This corresponds to the + getConcurrency defines the maximum number of GET requests processed concurrently. This corresponds to the Alertmanager's `--web.get-concurrency` flag. format: int32 type: integer httpConfig: - description: Defines HTTP parameters for web server. + description: httpConfig defines HTTP parameters for web server. properties: headers: - description: List of headers that can be added to HTTP responses. + description: headers defines a list of headers that can be + added to HTTP responses. properties: contentSecurityPolicy: description: |- - Set the Content-Security-Policy header to HTTP responses. + contentSecurityPolicy defines the Content-Security-Policy header to HTTP responses. Unset if blank. type: string strictTransportSecurity: description: |- - Set the Strict-Transport-Security header to HTTP responses. + strictTransportSecurity defines the Strict-Transport-Security header to HTTP responses. Unset if blank. Please make sure that you use this with care as this header might force browsers to load Prometheus and the other applications hosted on the same @@ -9454,7 +9510,7 @@ spec: type: string xContentTypeOptions: description: |- - Set the X-Content-Type-Options header to HTTP responses. + xContentTypeOptions defines the X-Content-Type-Options header to HTTP responses. Unset if blank. Accepted value is nosniff. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options enum: @@ -9463,7 +9519,7 @@ spec: type: string xFrameOptions: description: |- - Set the X-Frame-Options header to HTTP responses. + xFrameOptions defines the X-Frame-Options header to HTTP responses. Unset if blank. Accepted values are deny and sameorigin. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options enum: @@ -9473,38 +9529,38 @@ spec: type: string xXSSProtection: description: |- - Set the X-XSS-Protection header to all responses. + xXSSProtection defines the X-XSS-Protection header to all responses. Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection type: string type: object http2: description: |- - Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. + http2 enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. When TLSConfig is not configured, HTTP/2 will be disabled. Whenever the value of the field changes, a rolling update will be triggered. type: boolean type: object timeout: description: |- - Timeout for HTTP requests. This corresponds to the Alertmanager's + timeout for HTTP requests. This corresponds to the Alertmanager's `--web.timeout` flag. format: int32 type: integer tlsConfig: - description: Defines the TLS parameters for HTTPS. + description: tlsConfig defines the TLS parameters for HTTPS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9527,7 +9583,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9553,7 +9610,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -9561,7 +9618,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -9571,14 +9628,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9601,7 +9658,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9627,21 +9685,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -9651,7 +9709,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -9659,7 +9717,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -9687,14 +9745,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -9705,18 +9765,19 @@ spec: type: object status: description: |- - Most recent observed status of the Alertmanager cluster. Read-only. + status defines the most recent observed status of the Alertmanager cluster. Read-only. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this Alertmanager cluster. format: int32 type: integer conditions: - description: The current state of the Alertmanager object. + description: conditions defines the current state of the Alertmanager + object. items: description: |- Condition represents the state of the resources associated with the @@ -9728,12 +9789,12 @@ spec: format: date-time type: string message: - description: Human-readable message indicating details for the - condition's last transition. + description: message defines human-readable message indicating + details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the @@ -9741,14 +9802,14 @@ spec: format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: - description: Type of the condition being reported. + description: type of the condition being reported. minLength: 1 type: string required: @@ -9762,36 +9823,30 @@ spec: x-kubernetes-list-type: map paused: description: |- - Represents whether any actions on the underlying managed objects are + paused defines whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: |- - Total number of non-terminated pods targeted by this Alertmanager + replicas defines the total number of non-terminated pods targeted by this Alertmanager object (their labels match the selector). format: int32 type: integer selector: - description: The selector used to match the pods targeted by this - Alertmanager object. + description: selector used to match the pods targeted by this Alertmanager + object. type: string unavailableReplicas: - description: Total number of unavailable pods targeted by this Alertmanager - object. + description: unavailableReplicas defines the total number of unavailable + pods targeted by this Alertmanager object. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this Alertmanager + updatedReplicas defines the total number of non-terminated pods targeted by this Alertmanager object that have the desired version spec. format: int32 type: integer - required: - - availableReplicas - - paused - - replicas - - unavailableReplicas - - updatedReplicas type: object required: - spec diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_podmonitors.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_podmonitors.yaml index c888967e7c4..638e9a8c77c 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_podmonitors.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_podmonitors.yaml @@ -49,19 +49,19 @@ spec: metadata: type: object spec: - description: Specification of desired Pod selection for target discovery - by Prometheus. + description: spec defines the specification of desired Pod selection for + target discovery by Prometheus. properties: attachMetadata: description: |- - `attachMetadata` defines additional metadata which is added to the + attachMetadata defines additional metadata which is added to the discovered targets. It requires Prometheus >= v2.35.0. properties: node: description: |- - When set to true, Prometheus attaches node metadata to the discovered + node when set to true, Prometheus attaches node metadata to the discovered targets. The Prometheus service account must have the `list` and `watch` @@ -70,7 +70,7 @@ spec: type: object bodySizeLimit: description: |- - When defined, bodySizeLimit specifies a job level limit on the size + bodySizeLimit when defined specifies a job level limit on the size of uncompressed response body that will be accepted by Prometheus. It requires Prometheus >= v2.28.0. @@ -78,12 +78,12 @@ spec: type: string convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native histogram with custom buckets. + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.0.0. type: boolean fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It requires Prometheus >= v3.0.0. enum: @@ -95,7 +95,7 @@ spec: type: string jobLabel: description: |- - The label to use to retrieve the job name from. + jobLabel defines the label to use to retrieve the job name from. `jobLabel` selects the label from the associated Kubernetes `Pod` object which will be used as the `job` label for all metrics. @@ -108,7 +108,7 @@ spec: type: string keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -116,44 +116,45 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines the per-scrape limit on number of labels that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer namespaceSelector: description: |- - `namespaceSelector` defines in which namespace(s) Prometheus should discover the pods. + namespaceSelector defines in which namespace(s) Prometheus should discover the pods. By default, the pods are discovered in the same namespace as the `PodMonitor` object but it is possible to select pods across different/all namespaces. properties: any: description: |- - Boolean describing whether all namespaces are selected in contrast to a + any defines the boolean describing whether all namespaces are selected in contrast to a list restricting them. type: boolean matchNames: - description: List of namespace names to select from. + description: matchNames defines the list of namespace names to + select from. items: type: string type: array type: object nativeHistogramBucketLimit: description: |- - If there are more than this many buckets in a native histogram, + nativeHistogramBucketLimit defines ff there are more than this many buckets in a native histogram, buckets will be merged to stay within the limit. It requires Prometheus >= v2.45.0. format: int64 @@ -163,13 +164,14 @@ spec: - type: integer - type: string description: |- - If the growth factor of one bucket to the next is smaller than this, + nativeHistogramMinBucketFactor defines if the growth factor of one bucket to the next is smaller than this, buckets will be merged to increase the factor sufficiently. It requires Prometheus >= v2.50.0. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true podMetricsEndpoints: - description: Defines how to scrape metrics from the selected pods. + description: podMetricsEndpoints defines how to scrape metrics from + the selected pods. items: description: |- PodMetricsEndpoint defines an endpoint serving Prometheus metrics to be scraped by @@ -177,14 +179,14 @@ spec: properties: authorization: description: |- - `authorization` configures the Authorization header credentials to use when - scraping the target. + authorization configures the Authorization header credentials used by + the client. - Cannot be set at the same time as `basicAuth`, or `oauth2`. + Cannot be set at the same time as `basicAuth`, `bearerTokenSecret` or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -209,7 +211,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -218,14 +220,14 @@ spec: type: object basicAuth: description: |- - `basicAuth` configures the Basic Authentication credentials to use when - scraping the target. + basicAuth defines the Basic Authentication credentials used by the + client. - Cannot be set at the same time as `authorization`, or `oauth2`. + Cannot be set at the same time as `authorization`, `bearerTokenSecret` or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -251,7 +253,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -278,9 +280,12 @@ spec: type: object bearerTokenSecret: description: |- - `bearerTokenSecret` specifies a key of a Secret containing the bearer - token for scraping targets. The secret needs to be in the same namespace - as the PodMonitor object and readable by the Prometheus Operator. + bearerTokenSecret defines a key of a Secret containing the bearer token + used by the client for authentication. The secret needs to be in the + same namespace as the custom resource and readable by the Prometheus + Operator. + + Cannot be set at the same time as `authorization`, `basicAuth` or `oauth2`. Deprecated: use `authorization` instead. properties: @@ -306,12 +311,11 @@ spec: type: object x-kubernetes-map-type: atomic enableHttp2: - description: '`enableHttp2` can be used to disable HTTP2 when - scraping the target.' + description: enableHttp2 can be used to disable HTTP2. type: boolean filterRunning: description: |- - When true, the pods which are not running (e.g. either in Failed or + filterRunning when true, the pods which are not running (e.g. either in Failed or Succeeded state) are dropped during the target discovery. If unset, the filtering is enabled. @@ -320,29 +324,29 @@ spec: type: boolean followRedirects: description: |- - `followRedirects` defines whether the scrape requests should follow HTTP - 3xx redirects. + followRedirects defines whether the client should follow HTTP 3xx + redirects. type: boolean honorLabels: description: |- - When true, `honorLabels` preserves the metric's labels when they collide + honorLabels when true preserves the metric's labels when they collide with the target's labels. type: boolean honorTimestamps: description: |- - `honorTimestamps` controls whether Prometheus preserves the timestamps + honorTimestamps defines whether Prometheus preserves the timestamps when exposed by the target. type: boolean interval: description: |- - Interval at which Prometheus scrapes the metrics from the target. + interval at which Prometheus scrapes the metrics from the target. If empty, Prometheus uses the global scrape interval. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string metricRelabelings: description: |- - `metricRelabelings` configures the relabeling rules to apply to the + metricRelabelings defines the relabeling rules to apply to the samples before ingestion. items: description: |- @@ -354,7 +358,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -386,41 +390,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -431,7 +435,7 @@ spec: type: array noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -439,20 +443,20 @@ spec: type: string oauth2: description: |- - `oauth2` configures the OAuth2 settings to use when scraping the target. + oauth2 defines the OAuth2 settings used by the client. It requires Prometheus >= 2.27.0. - Cannot be set at the same time as `authorization`, or `basicAuth`. + Cannot be set at the same time as `authorization`, `basicAuth` or `bearerTokenSecret`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -475,7 +479,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -501,7 +506,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -529,12 +534,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -568,7 +573,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -576,33 +581,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -625,8 +629,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -651,12 +655,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -679,8 +683,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -705,11 +709,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -734,7 +739,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -745,7 +750,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -755,12 +760,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -773,22 +779,23 @@ spec: items: type: string type: array - description: '`params` define optional HTTP URL parameters.' + description: params define optional HTTP URL parameters. type: object path: description: |- - HTTP path from which to scrape for metrics. + path defines the HTTP path from which to scrape for metrics. If empty, Prometheus uses the default value (e.g. `/metrics`). type: string port: description: |- - The `Pod` port name which exposes the endpoint. + port defines the `Pod` port name which exposes the endpoint. It takes precedence over the `portNumber` and `targetPort` fields. type: string portNumber: - description: The `Pod` port number which exposes the endpoint. + description: portNumber defines the `Pod` port number which + exposes the endpoint. format: int32 maximum: 65535 minimum: 1 @@ -821,7 +828,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -829,17 +836,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string relabelings: description: |- - `relabelings` configures the relabeling rules to apply the target's + relabelings defines the relabeling rules to apply the target's metadata labels. The Operator automatically adds relabelings for a few standard Kubernetes fields. @@ -857,7 +864,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -889,41 +896,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -934,7 +941,7 @@ spec: type: array scheme: description: |- - HTTP scheme to use for scraping. + scheme defines the HTTP scheme to use for scraping. `http` and `https` are the expected values unless you rewrite the `__scheme__` label via relabeling. @@ -946,7 +953,7 @@ spec: type: string scrapeTimeout: description: |- - Timeout after which Prometheus considers the scrape to be failed. + scrapeTimeout defines the timeout after which Prometheus considers the scrape to be failed. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used. @@ -958,21 +965,22 @@ spec: - type: integer - type: string description: |- - Name or number of the target port of the `Pod` object behind the Service, the + targetPort defines the name or number of the target port of the `Pod` object behind the Service, the port must be specified with container port property. Deprecated: use 'port' or 'portNumber' instead. x-kubernetes-int-or-string: true tlsConfig: - description: TLS configuration to use when scraping the target. + description: tlsConfig defines the TLS configuration used by + the client. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -995,7 +1003,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1020,11 +1029,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1047,7 +1057,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1072,11 +1083,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1101,7 +1113,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1112,7 +1124,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1122,12 +1134,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object trackTimestampsStaleness: description: |- - `trackTimestampsStaleness` defines whether Prometheus tracks staleness of + trackTimestampsStaleness defines whether Prometheus tracks staleness of the metrics that have an explicit timestamp present in scraped data. Has no effect if `honorTimestamps` is false. @@ -1137,31 +1150,31 @@ spec: type: array podTargetLabels: description: |- - `podTargetLabels` defines the labels which are transferred from the + podTargetLabels defines the labels which are transferred from the associated Kubernetes `Pod` object onto the ingested metrics. items: type: string type: array sampleLimit: description: |- - `sampleLimit` defines a per-scrape limit on the number of scraped samples + sampleLimit defines a per-scrape limit on the number of scraped samples that will be accepted. format: int64 type: integer scrapeClass: - description: The scrape class to apply. + description: scrapeClass defines the scrape class to apply. minLength: 1 type: string scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. It requires Prometheus >= v2.45.0. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. type: boolean scrapeProtocols: description: |- - `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -1186,8 +1199,8 @@ spec: type: array x-kubernetes-list-type: set selector: - description: Label selector to select the Kubernetes `Pod` objects - to scrape metrics from. + description: selector defines the label selector to select the Kubernetes + `Pod` objects to scrape metrics from. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. @@ -1234,7 +1247,7 @@ spec: x-kubernetes-map-type: atomic selectorMechanism: description: |- - Mechanism used to select the endpoints to scrape. + selectorMechanism defines the mechanism used to select the endpoints to scrape. By default, the selection process relies on relabel configurations to filter the discovered targets. Alternatively, you can opt in for role selectors, which may offer better efficiency in large clusters. Which strategy is best for your use case needs to be carefully evaluated. @@ -1246,15 +1259,120 @@ spec: type: string targetLimit: description: |- - `targetLimit` defines a limit on the number of scraped targets that will + targetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer required: - selector type: object + status: + description: |- + status defines the status subresource. It is under active development and is updated only when the + "StatusForConfigurationResources" feature gate is enabled. + + Most recent observed status of the PodMonitor. Read-only. + More info: + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + bindings: + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. + items: + description: WorkloadBinding is a link between a configuration resource + and a workload resource. + properties: + conditions: + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. + items: + description: ConfigResourceCondition describes the status + of configuration resources linked to Prometheus, PrometheusAgent, + Alertmanager or ThanosRuler. + properties: + lastTransitionTime: + description: lastTransitionTime defines the time of the + last update to the current status property. + format: date-time + type: string + message: + description: message defines the human-readable message + indicating details for the condition's last transition. + type: string + observedGeneration: + description: |- + observedGeneration defines the .metadata.generation that the + condition was set based upon. For instance, if `.metadata.generation` is + currently 12, but the `.status.conditions[].observedGeneration` is 9, the + condition is out of date with respect to the current state of the object. + format: int64 + type: integer + reason: + description: reason for the condition's last transition. + type: string + status: + description: status of the condition. + minLength: 1 + type: string + type: + description: |- + type of the condition being reported. + Currently, only "Accepted" is supported. + enum: + - Accepted + minLength: 1 + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + group: + description: group defines the group of the referenced resource. + enum: + - monitoring.coreos.com + type: string + name: + description: name defines the name of the referenced object. + minLength: 1 + type: string + namespace: + description: namespace defines the namespace of the referenced + object. + minLength: 1 + type: string + resource: + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). + enum: + - prometheuses + - prometheusagents + - thanosrulers + - alertmanagers + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map + type: object required: - spec type: object served: true storage: true + subresources: + status: {} diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_probes.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_probes.yaml index 4ac47e50aa1..40424236be3 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_probes.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_probes.yaml @@ -48,15 +48,15 @@ spec: metadata: type: object spec: - description: Specification of desired Ingress selection for target discovery - by Prometheus. + description: spec defines the specification of desired Ingress selection + for target discovery by Prometheus. properties: authorization: - description: Authorization section for this endpoint + description: authorization section for this endpoint properties: credentials: - description: Selects a key of a Secret in the namespace that contains - the credentials for authentication. + description: credentials defines a key of a Secret in the namespace + that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must be @@ -81,7 +81,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -90,12 +90,12 @@ spec: type: object basicAuth: description: |- - BasicAuth allow an endpoint to authenticate over basic authentication. + basicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -121,7 +121,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -148,7 +148,7 @@ spec: type: object bearerTokenSecret: description: |- - Secret to mount to read bearer token for scraping targets. The secret + bearerTokenSecret defines the secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the probe and accessible by the Prometheus Operator. properties: @@ -174,12 +174,12 @@ spec: x-kubernetes-map-type: atomic convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native histogram with custom buckets. + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.0.0. type: boolean fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It requires Prometheus >= v3.0.0. enum: @@ -191,16 +191,16 @@ spec: type: string interval: description: |- - Interval at which targets are probed using the configured prober. + interval at which targets are probed using the configured prober. If not specified Prometheus' global scrape interval is used. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string jobName: - description: The job name assigned to scraped metrics by default. + description: jobName assigned to scraped metrics by default. type: string keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -208,24 +208,25 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines the per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer metricRelabelings: - description: MetricRelabelConfigs to apply to samples before ingestion. + description: metricRelabelings defines the RelabelConfig to apply + to samples before ingestion. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -236,7 +237,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -268,40 +269,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against which + the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated SourceLabels. + description: separator defines the string between concatenated + SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -312,13 +314,13 @@ spec: type: array module: description: |- - The module to use for probing specifying how to probe the target. + module to use for probing specifying how to probe the target. Example module configuring in the blackbox exporter: https://github.com/prometheus/blackbox_exporter/blob/master/example.yml type: string nativeHistogramBucketLimit: description: |- - If there are more than this many buckets in a native histogram, + nativeHistogramBucketLimit defines ff there are more than this many buckets in a native histogram, buckets will be merged to stay within the limit. It requires Prometheus >= v2.45.0. format: int64 @@ -328,22 +330,23 @@ spec: - type: integer - type: string description: |- - If the growth factor of one bucket to the next is smaller than this, + nativeHistogramMinBucketFactor defines if the growth factor of one bucket to the next is smaller than this, buckets will be merged to increase the factor sufficiently. It requires Prometheus >= v2.50.0. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true oauth2: - description: OAuth2 for the URL. Only valid in Prometheus versions + description: oauth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -366,7 +369,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -392,7 +396,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -420,12 +424,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -459,7 +463,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -467,32 +471,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for the - token request.' + description: scopes defines the OAuth2 scopes used for the token + request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -515,7 +519,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -540,11 +545,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -567,7 +573,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -592,11 +599,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -621,7 +629,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -632,7 +640,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -642,12 +650,12 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the token - from.' + description: tokenUrl defines the URL to fetch the token from. minLength: 1 type: string required: @@ -657,7 +665,7 @@ spec: type: object params: description: |- - The list of HTTP query parameters for the scrape. + params defines the list of HTTP query parameters for the scrape. Please note that the `.spec.module` field takes precedence over the `module` parameter from this list when both are defined. The module name must be added using Module under ProbeSpec. items: @@ -665,11 +673,11 @@ spec: for a Probe. properties: name: - description: The parameter name + description: name defines the parameter name minLength: 1 type: string values: - description: The parameter values + description: values defines the parameter values items: minLength: 1 type: string @@ -685,12 +693,12 @@ spec: x-kubernetes-list-type: map prober: description: |- - Specification for the prober to use for probing targets. + prober defines the specification for the prober to use for probing targets. The prober.URL parameter is required. Targets cannot be probed if left empty. properties: noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -699,7 +707,7 @@ spec: path: default: /probe description: |- - Path to collect metrics from. + path to collect metrics from. Defaults to `/probe`. type: string proxyConnectHeader: @@ -730,7 +738,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -738,17 +746,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scheme: description: |- - HTTP scheme to use for scraping. + scheme defines the HTTP scheme to use for scraping. `http` and `https` are the expected values unless you rewrite the `__scheme__` label via relabeling. If empty, Prometheus uses the default value `http`. enum: @@ -756,30 +764,30 @@ spec: - https type: string url: - description: Mandatory URL of the prober. + description: url defines the mandatory URL of the prober. type: string required: - url type: object sampleLimit: - description: SampleLimit defines per-scrape limit on number of scraped + description: sampleLimit defines per-scrape limit on number of scraped samples that will be accepted. format: int64 type: integer scrapeClass: - description: The scrape class to apply. + description: scrapeClass defines the scrape class to apply. minLength: 1 type: string scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. It requires Prometheus >= v2.45.0. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. type: boolean scrapeProtocols: description: |- - `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -805,18 +813,18 @@ spec: x-kubernetes-list-type: set scrapeTimeout: description: |- - Timeout for scraping metrics from the Prometheus exporter. + scrapeTimeout defines the timeout for scraping metrics from the Prometheus exporter. If not specified, the Prometheus global scrape timeout is used. The value cannot be greater than the scrape interval otherwise the operator will reject the resource. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string targetLimit: - description: TargetLimit defines a limit on the number of scraped + description: targetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer targets: - description: Targets defines a set of static or dynamically discovered + description: targets defines a set of static or dynamically discovered targets to probe. properties: ingress: @@ -826,22 +834,24 @@ spec: If `staticConfig` is also defined, `staticConfig` takes precedence. properties: namespaceSelector: - description: From which namespaces to select Ingress objects. + description: namespaceSelector defines from which namespaces + to select Ingress objects. properties: any: description: |- - Boolean describing whether all namespaces are selected in contrast to a + any defines the boolean describing whether all namespaces are selected in contrast to a list restricting them. type: boolean matchNames: - description: List of namespace names to select from. + description: matchNames defines the list of namespace + names to select from. items: type: string type: array type: object relabelingConfigs: description: |- - RelabelConfigs to apply to the label set of the target before it gets + relabelingConfigs to apply to the label set of the target before it gets scraped. The original ingress address is available via the `__tmp_prometheus_ingress_address` label. It can be used to customize the @@ -858,7 +868,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -890,41 +900,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -934,7 +944,7 @@ spec: type: object type: array selector: - description: Selector to select the Ingress objects. + description: selector to select the Ingress objects. properties: matchExpressions: description: matchExpressions is a list of label selector @@ -990,12 +1000,12 @@ spec: labels: additionalProperties: type: string - description: Labels assigned to all metrics scraped from the - targets. + description: labels defines all labels assigned to all metrics + scraped from the targets. type: object relabelingConfigs: description: |- - RelabelConfigs to apply to the label set of the targets before it gets + relabelingConfigs defines relabelings to be apply to the label set of the targets before it gets scraped. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config items: @@ -1008,7 +1018,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -1040,41 +1050,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -1084,21 +1094,23 @@ spec: type: object type: array static: - description: The list of hosts to probe. + description: static defines the list of hosts to probe. items: type: string type: array type: object type: object tlsConfig: - description: TLS configuration to use when scraping the endpoint. + description: tlsConfig defines the TLS configuration to use when scraping + the endpoint. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when verifying + server certificates. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -1121,7 +1133,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1146,10 +1159,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present when + doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -1172,7 +1187,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1197,10 +1213,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must be @@ -1225,7 +1243,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1236,7 +1254,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1246,12 +1264,118 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for the + targets. type: string type: object type: object + status: + description: |- + status defines the status subresource. It is under active development and is updated only when the + "StatusForConfigurationResources" feature gate is enabled. + + Most recent observed status of the Probe. Read-only. + More info: + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + bindings: + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. + items: + description: WorkloadBinding is a link between a configuration resource + and a workload resource. + properties: + conditions: + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. + items: + description: ConfigResourceCondition describes the status + of configuration resources linked to Prometheus, PrometheusAgent, + Alertmanager or ThanosRuler. + properties: + lastTransitionTime: + description: lastTransitionTime defines the time of the + last update to the current status property. + format: date-time + type: string + message: + description: message defines the human-readable message + indicating details for the condition's last transition. + type: string + observedGeneration: + description: |- + observedGeneration defines the .metadata.generation that the + condition was set based upon. For instance, if `.metadata.generation` is + currently 12, but the `.status.conditions[].observedGeneration` is 9, the + condition is out of date with respect to the current state of the object. + format: int64 + type: integer + reason: + description: reason for the condition's last transition. + type: string + status: + description: status of the condition. + minLength: 1 + type: string + type: + description: |- + type of the condition being reported. + Currently, only "Accepted" is supported. + enum: + - Accepted + minLength: 1 + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + group: + description: group defines the group of the referenced resource. + enum: + - monitoring.coreos.com + type: string + name: + description: name defines the name of the referenced object. + minLength: 1 + type: string + namespace: + description: namespace defines the namespace of the referenced + object. + minLength: 1 + type: string + resource: + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). + enum: + - prometheuses + - prometheusagents + - thanosrulers + - alertmanagers + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map + type: object required: - spec type: object served: true storage: true + subresources: + status: {} diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_prometheusagents.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_prometheusagents.yaml index 74e741011b3..993c388a774 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_prometheusagents.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_prometheusagents.yaml @@ -72,12 +72,12 @@ spec: type: object spec: description: |- - Specification of the desired behavior of the Prometheus agent. More info: + spec defines the specification of the desired behavior of the Prometheus agent. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the 'prometheus' container. + additionalArgs allows setting additional arguments for the 'prometheus' container. It is intended for e.g. activating hidden flags which are not supported by the dedicated configuration options yet. The arguments are passed as-is to the @@ -91,12 +91,12 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for name-only - arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. Can + be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name @@ -104,7 +104,7 @@ spec: type: array additionalScrapeConfigs: description: |- - AdditionalScrapeConfigs allows specifying a key of a Secret containing + additionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified @@ -137,7 +137,8 @@ spec: type: object x-kubernetes-map-type: atomic affinity: - description: Defines the Pods' affinity scheduling rules if specified. + description: affinity defines the Pods' affinity scheduling rules + if specified. properties: nodeAffinity: description: Describes node affinity scheduling rules for the @@ -1050,7 +1051,7 @@ spec: type: object apiserverConfig: description: |- - APIServerConfig allows specifying a host and auth methods to access the + apiserverConfig allows specifying a host and auth methods to access the Kuberntees API server. If null, Prometheus is assumed to run inside of the cluster: it will discover the API servers automatically and use the Pod's CA certificate @@ -1058,14 +1059,14 @@ spec: properties: authorization: description: |- - Authorization section for the API server. + authorization section for the API server. Cannot be set at the same time as `basicAuth`, `bearerToken`, or `bearerTokenFile`. properties: credentials: - description: Selects a key of a Secret in the namespace that - contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -1089,12 +1090,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a secret + from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1103,14 +1104,14 @@ spec: type: object basicAuth: description: |- - BasicAuth configuration for the API server. + basicAuth configuration for the API server. Cannot be set at the same time as `authorization`, `bearerToken`, or `bearerTokenFile`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1136,7 +1137,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1163,14 +1164,13 @@ spec: type: object bearerToken: description: |- - *Warning: this field shouldn't be used because the token value appears + bearerToken is deprecated: this will be removed in a future release. + *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File to read bearer token for accessing apiserver. + bearerTokenFile defines the file to read bearer token for accessing apiserver. Cannot be set at the same time as `basicAuth`, `authorization`, or `bearerToken`. @@ -1178,12 +1178,12 @@ spec: type: string host: description: |- - Kubernetes API address consisting of a hostname or IP address followed + host defines the Kubernetes API address consisting of a hostname or IP address followed by an optional port number. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1217,7 +1217,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1225,24 +1225,24 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS Config to use for the API server. + description: tlsConfig to use for the API server. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1265,7 +1265,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1290,15 +1291,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1321,7 +1323,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1346,19 +1349,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1383,7 +1387,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1394,7 +1398,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1404,7 +1408,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -1412,7 +1417,7 @@ spec: type: object arbitraryFSAccessThroughSMs: description: |- - When true, ServiceMonitor, PodMonitor and Probe object are forbidden to + arbitraryFSAccessThroughSMs when true, ServiceMonitor, PodMonitor and Probe object are forbidden to reference arbitrary files on the file system of the 'prometheus' container. When a ServiceMonitor's endpoint specifies a `bearerTokenFile` value @@ -1424,11 +1429,16 @@ spec: `spec.bearerTokenSecret` field. properties: deny: + description: |- + deny prevents service monitors from accessing arbitrary files on the file system. + When true, service monitors cannot use file-based configurations like BearerTokenFile + that could potentially access sensitive files. When false (default), such access is allowed. + Setting this to true enhances security by preventing potential credential theft attacks. type: boolean type: object automountServiceAccountToken: description: |- - AutomountServiceAccountToken indicates whether a service account token should be automatically mounted in the pod. + automountServiceAccountToken defines whether a service account token should be automatically mounted in the pod. If the field isn't set, the operator mounts the service account token by default. **Warning:** be aware that by default, Prometheus requires the service account token for Kubernetes service discovery. @@ -1436,7 +1446,7 @@ spec: type: boolean bodySizeLimit: description: |- - BodySizeLimit defines per-scrape on response body size. + bodySizeLimit defines per-scrape on response body size. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -1445,7 +1455,7 @@ spec: type: string configMaps: description: |- - ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus + configMaps defines a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. Each ConfigMap is added to the StatefulSet definition as a volume named `configmap-`. The ConfigMaps are mounted into /etc/prometheus/configmaps/ in the 'prometheus' container. @@ -1454,7 +1464,7 @@ spec: type: array containers: description: |- - Containers allows injecting additional containers or modifying operator + containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to the Pods or to change the behavior of an operator generated container. Containers described here modify an operator generated @@ -2990,17 +3000,17 @@ spec: type: array convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.4.0. type: boolean dnsConfig: - description: Defines the DNS configuration for the pods. + description: dnsConfig defines the DNS configuration for the pods. properties: nameservers: description: |- - A list of DNS name server IP addresses. + nameservers defines the list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. items: minLength: 1 @@ -3009,7 +3019,7 @@ spec: x-kubernetes-list-type: set options: description: |- - A list of DNS resolver options. + options defines the list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Resolution options given in Options will override those that appear in the base DNSPolicy. @@ -3018,11 +3028,11 @@ spec: of a pod. properties: name: - description: Name is required and must be unique. + description: name is required and must be unique. minLength: 1 type: string value: - description: Value is optional. + description: value is optional. type: string required: - name @@ -3033,7 +3043,7 @@ spec: x-kubernetes-list-type: map searches: description: |- - A list of DNS search domains for host-name lookup. + searches defines the list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. items: minLength: 1 @@ -3042,7 +3052,7 @@ spec: x-kubernetes-list-type: set type: object dnsPolicy: - description: Defines the DNS policy for the pods. + description: dnsPolicy defines the DNS policy for the pods. enum: - ClusterFirstWithHostNet - ClusterFirst @@ -3051,7 +3061,7 @@ spec: type: string enableFeatures: description: |- - Enable access to Prometheus feature flags. By default, no features are enabled. + enableFeatures enables access to Prometheus feature flags. By default, no features are enabled. Enabling features which are disabled by default is entirely outside the scope of what the maintainers will support and by doing so, you accept @@ -3065,7 +3075,7 @@ spec: x-kubernetes-list-type: set enableOTLPReceiver: description: |- - Enable Prometheus to be used as a receiver for the OTLP Metrics protocol. + enableOTLPReceiver defines the Prometheus to be used as a receiver for the OTLP Metrics protocol. Note that the OTLP receiver endpoint is automatically enabled if `.spec.otlpConfig` is defined. @@ -3073,7 +3083,7 @@ spec: type: boolean enableRemoteWriteReceiver: description: |- - Enable Prometheus to be used as a receiver for the Prometheus remote + enableRemoteWriteReceiver defines the Prometheus to be used as a receiver for the Prometheus remote write protocol. WARNING: This is not considered an efficient way of ingesting samples. @@ -3085,12 +3095,12 @@ spec: It requires Prometheus >= v2.33.0. type: boolean enableServiceLinks: - description: Indicates whether information about services should be - injected into pod's environment variables + description: enableServiceLinks defines whether information about + services should be injected into pod's environment variables type: boolean enforcedBodySizeLimit: description: |- - When defined, enforcedBodySizeLimit specifies a global limit on the size + enforcedBodySizeLimit when defined specifies a global limit on the size of uncompressed response body that will be accepted by Prometheus. Targets responding with a body larger than this many bytes will cause the scrape to fail. @@ -3106,7 +3116,7 @@ spec: type: string enforcedKeepDroppedTargets: description: |- - When defined, enforcedKeepDroppedTargets specifies a global limit on the number of targets + enforcedKeepDroppedTargets when defined specifies a global limit on the number of targets dropped by relabeling that will be kept in memory. The value overrides any `spec.keepDroppedTargets` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.keepDroppedTargets` is @@ -3123,7 +3133,7 @@ spec: type: integer enforcedLabelLimit: description: |- - When defined, enforcedLabelLimit specifies a global limit on the number + enforcedLabelLimit when defined specifies a global limit on the number of labels per sample. The value overrides any `spec.labelLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelLimit` is greater than zero and less than `spec.enforcedLabelLimit`. @@ -3139,7 +3149,7 @@ spec: type: integer enforcedLabelNameLengthLimit: description: |- - When defined, enforcedLabelNameLengthLimit specifies a global limit on the length + enforcedLabelNameLengthLimit when defined specifies a global limit on the length of labels name per sample. The value overrides any `spec.labelNameLengthLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelNameLengthLimit` is greater than zero and less than `spec.enforcedLabelNameLengthLimit`. @@ -3155,7 +3165,7 @@ spec: type: integer enforcedLabelValueLengthLimit: description: |- - When not null, enforcedLabelValueLengthLimit defines a global limit on the length + enforcedLabelValueLengthLimit when not null defines a global limit on the length of labels value per sample. The value overrides any `spec.labelValueLengthLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelValueLengthLimit` is greater than zero and less than `spec.enforcedLabelValueLengthLimit`. @@ -3171,7 +3181,7 @@ spec: type: integer enforcedNamespaceLabel: description: |- - When not empty, a label will be added to: + enforcedNamespaceLabel when not empty, a label will be added to: 1. All metrics scraped from `ServiceMonitor`, `PodMonitor`, `Probe` and `ScrapeConfig` objects. 2. All metrics generated from recording rules defined in `PrometheusRule` objects. @@ -3186,7 +3196,7 @@ spec: type: string enforcedSampleLimit: description: |- - When defined, enforcedSampleLimit specifies a global limit on the number + enforcedSampleLimit when defined specifies a global limit on the number of scraped samples that will be accepted. This overrides any `spec.sampleLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.sampleLimit` is greater than zero and less than @@ -3204,7 +3214,7 @@ spec: type: integer enforcedTargetLimit: description: |- - When defined, enforcedTargetLimit specifies a global limit on the number + enforcedTargetLimit when defined specifies a global limit on the number of scraped targets. The value overrides any `spec.targetLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.targetLimit` is greater than zero and less than `spec.enforcedTargetLimit`. @@ -3221,7 +3231,7 @@ spec: type: integer excludedFromEnforcement: description: |- - List of references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + excludedFromEnforcement defines the list of references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects to be excluded from enforcing a namespace label of origin. It is only applicable if `spec.enforcedNamespaceLabel` set to true. @@ -3231,23 +3241,23 @@ spec: properties: group: default: monitoring.coreos.com - description: Group of the referent. When not specified, it defaults + description: group of the referent. When not specified, it defaults to `monitoring.coreos.com` enum: - monitoring.coreos.com type: string name: - description: Name of the referent. When not set, all resources + description: name of the referent. When not set, all resources in the namespace are matched. type: string namespace: description: |- - Namespace of the referent. + namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ minLength: 1 type: string resource: - description: Resource of the referent. + description: resource of the referent. enum: - prometheusrules - servicemonitors @@ -3264,20 +3274,20 @@ spec: additionalProperties: type: string description: |- - The labels to add to any time series or alerts when communicating with + externalLabels defines the labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager). Labels defined by `spec.replicaExternalLabelName` and `spec.prometheusExternalLabelName` take precedence over this list. type: object externalUrl: description: |- - The external URL under which the Prometheus service is externally + externalUrl defines the external URL under which the Prometheus service is externally available. This is necessary to generate correct URLs (for instance if Prometheus is accessible behind an Ingress resource). type: string hostAliases: description: |- - Optional list of hosts and IPs that will be injected into the Pod's + hostAliases defines the optional list of hosts and IPs that will be injected into the Pod's hosts file if specified. items: description: |- @@ -3285,12 +3295,12 @@ spec: pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. + description: hostnames defines hostnames for the above IP address. items: type: string type: array ip: - description: IP address of the host file entry. + description: ip defines the IP address of the host file entry. type: string required: - hostnames @@ -3302,7 +3312,7 @@ spec: x-kubernetes-list-type: map hostNetwork: description: |- - Use the host's network namespace if true. + hostNetwork defines the host's network namespace if true. Make sure to understand the security implications if you want to enable it (https://kubernetes.io/docs/concepts/configuration/overview/ ). @@ -3313,7 +3323,7 @@ spec: type: boolean hostUsers: description: |- - HostUsers supports the user space in Kubernetes. + hostUsers supports the user space in Kubernetes. More info: https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/ @@ -3322,14 +3332,14 @@ spec: type: boolean ignoreNamespaceSelectors: description: |- - When true, `spec.namespaceSelector` from all PodMonitor, ServiceMonitor + ignoreNamespaceSelectors when true, `spec.namespaceSelector` from all PodMonitor, ServiceMonitor and Probe objects will be ignored. They will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object. type: boolean image: description: |- - Container image name for Prometheus. If specified, it takes precedence + image defines the container image name for Prometheus. If specified, it takes precedence over the `spec.baseImage`, `spec.tag` and `spec.sha` fields. Specifying `spec.version` is still necessary to ensure the Prometheus @@ -3341,7 +3351,7 @@ spec: type: string imagePullPolicy: description: |- - Image pull policy for the 'prometheus', 'init-config-reloader' and 'config-reloader' containers. + imagePullPolicy defines the image pull policy for the 'prometheus', 'init-config-reloader' and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. enum: - "" @@ -3351,7 +3361,7 @@ spec: type: string imagePullSecrets: description: |- - An optional list of references to Secrets in the same namespace + imagePullSecrets defines an optional list of references to Secrets in the same namespace to use for pulling images from registries. See http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod items: @@ -3373,7 +3383,7 @@ spec: type: array initContainers: description: |- - InitContainers allows injecting initContainers to the Pod definition. Those + initContainers allows injecting initContainers to the Pod definition. Those can be used to e.g. fetch secrets for injection into the Prometheus configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: @@ -4909,7 +4919,7 @@ spec: type: array keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -4920,7 +4930,7 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -4929,7 +4939,7 @@ spec: type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -4938,7 +4948,7 @@ spec: type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -4947,11 +4957,11 @@ spec: type: integer listenLocal: description: |- - When true, the Prometheus server listens on the loopback address + listenLocal when true, the Prometheus server listens on the loopback address instead of the Pod IP's address. type: boolean logFormat: - description: Log format for Log level for Prometheus and the config-reloader + description: logFormat for Log level for Prometheus and the config-reloader sidecar. enum: - "" @@ -4959,7 +4969,7 @@ spec: - json type: string logLevel: - description: Log level for Prometheus and the config-reloader sidecar. + description: logLevel for Prometheus and the config-reloader sidecar. enum: - "" - debug @@ -4969,14 +4979,14 @@ spec: type: string maximumStartupDurationSeconds: description: |- - Defines the maximum time that the `prometheus` container's startup probe will wait before being considered failed. The startup probe will return success after the WAL replay is complete. - If set, the value should be greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 minutes). + maximumStartupDurationSeconds defines the maximum time that the `prometheus` container's startup probe will wait before being considered failed. The startup probe will return success after the WAL replay is complete. + If set, the value should be greater than 60 (seconds). Otherwise it will be equal to 900 seconds (15 minutes). format: int32 minimum: 60 type: integer minReadySeconds: description: |- - Minimum number of seconds for which a newly created Pod should be ready + minReadySeconds defines the minimum number of seconds for which a newly created Pod should be ready without any of its container crashing for it to be considered available. If unset, pods will be considered available as soon as they are ready. @@ -4985,7 +4995,7 @@ spec: type: integer mode: description: |- - Mode defines how the Prometheus operator deploys the PrometheusAgent pod(s). + mode defines how the Prometheus operator deploys the PrometheusAgent pod(s). (Alpha) Using this field requires the `PrometheusAgentDaemonSet` feature gate to be enabled. enum: @@ -4994,7 +5004,7 @@ spec: type: string nameEscapingScheme: description: |- - Specifies the character escaping scheme that will be requested when scraping + nameEscapingScheme defines the character escaping scheme that will be requested when scraping for metric and label names that do not conform to the legacy Prometheus character set. @@ -5007,7 +5017,7 @@ spec: type: string nameValidationScheme: description: |- - Specifies the validation scheme for metric and label names. + nameValidationScheme defines the validation scheme for metric and label names. It requires Prometheus >= v2.55.0. enum: @@ -5017,21 +5027,21 @@ spec: nodeSelector: additionalProperties: type: string - description: Defines on which Nodes the Pods are scheduled. + description: nodeSelector defines on which Nodes the Pods are scheduled. type: object otlp: description: |- - Settings related to the OTLP receiver feature. + otlp defines the settings related to the OTLP receiver feature. It requires Prometheus >= v2.55.0. properties: convertHistogramsToNHCB: description: |- - Configures optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. + convertHistogramsToNHCB defines optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. It requires Prometheus >= v3.4.0. type: boolean ignoreResourceAttributes: description: |- - List of OpenTelemetry resource attributes to ignore when `promoteAllResourceAttributes` is true. + ignoreResourceAttributes defines the list of OpenTelemetry resource attributes to ignore when `promoteAllResourceAttributes` is true. It requires `promoteAllResourceAttributes` to be true. It requires Prometheus >= v3.5.0. @@ -5043,21 +5053,21 @@ spec: x-kubernetes-list-type: set keepIdentifyingResourceAttributes: description: |- - Enables adding `service.name`, `service.namespace` and `service.instance.id` + keepIdentifyingResourceAttributes enables adding `service.name`, `service.namespace` and `service.instance.id` resource attributes to the `target_info` metric, on top of converting them into the `instance` and `job` labels. It requires Prometheus >= v3.1.0. type: boolean promoteAllResourceAttributes: description: |- - Promote all resource attributes to metric labels except the ones defined in `ignoreResourceAttributes`. + promoteAllResourceAttributes promotes all resource attributes to metric labels except the ones defined in `ignoreResourceAttributes`. Cannot be true when `promoteResourceAttributes` is defined. It requires Prometheus >= v3.5.0. type: boolean promoteResourceAttributes: description: |- - List of OpenTelemetry Attributes that should be promoted to metric labels, defaults to none. + promoteResourceAttributes defines the list of OpenTelemetry Attributes that should be promoted to metric labels, defaults to none. Cannot be defined when `promoteAllResourceAttributes` is true. items: minLength: 1 @@ -5065,39 +5075,46 @@ spec: minItems: 1 type: array x-kubernetes-list-type: set + promoteScopeMetadata: + description: |- + promoteScopeMetadata controls whether to promote OpenTelemetry scope metadata (i.e. name, version, schema URL, and attributes) to metric labels. + As per the OpenTelemetry specification, the aforementioned scope metadata should be identifying, i.e. made into metric labels. + It requires Prometheus >= v3.6.0. + type: boolean translationStrategy: description: |- - Configures how the OTLP receiver endpoint translates the incoming metrics. + translationStrategy defines how the OTLP receiver endpoint translates the incoming metrics. It requires Prometheus >= v3.0.0. enum: - NoUTF8EscapingWithSuffixes - UnderscoreEscapingWithSuffixes - NoTranslation + - UnderscoreEscapingWithoutSuffixes type: string type: object overrideHonorLabels: description: |- - When true, Prometheus resolves label conflicts by renaming the labels in the scraped data + overrideHonorLabels when true, Prometheus resolves label conflicts by renaming the labels in the scraped data to “exported_” for all targets created from ServiceMonitor, PodMonitor and ScrapeConfig objects. Otherwise the HonorLabels field of the service or pod monitor applies. - In practice,`overrideHonorLaels:true` enforces `honorLabels:false` + In practice,`OverrideHonorLabels:true` enforces `honorLabels:false` for all ServiceMonitor, PodMonitor and ScrapeConfig objects. type: boolean overrideHonorTimestamps: description: |- - When true, Prometheus ignores the timestamps for all the targets created + overrideHonorTimestamps when true, Prometheus ignores the timestamps for all the targets created from service and pod monitors. Otherwise the HonorTimestamps field of the service or pod monitor applies. type: boolean paused: description: |- - When a Prometheus deployment is paused, no actions except for deletion + paused defines when a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects. type: boolean persistentVolumeClaimRetentionPolicy: description: |- - The field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. + persistentVolumeClaimRetentionPolicy defines the field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. The default behavior is all PVCs are retained. This is an alpha field from kubernetes 1.23 until 1.26 and a beta field from 1.26. It requires enabling the StatefulSetAutoDeletePVC feature gate. @@ -5120,7 +5137,7 @@ spec: type: object podMetadata: description: |- - PodMetadata configures labels and annotations which are propagated to the Prometheus pods. + podMetadata defines labels and annotations which are propagated to the Prometheus pods. The following items are reserved and cannot be overridden: * "prometheus" label, set to the name of the Prometheus object. @@ -5136,7 +5153,7 @@ spec: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -5145,14 +5162,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -5162,7 +5179,7 @@ spec: type: object podMonitorNamespaceSelector: description: |- - Namespaces to match for PodMonitors discovery. An empty label selector + podMonitorNamespaceSelector defines the namespaces to match for PodMonitors discovery. An empty label selector matches all namespaces. A null label selector (default value) matches the current namespace only. properties: @@ -5211,7 +5228,7 @@ spec: x-kubernetes-map-type: atomic podMonitorSelector: description: |- - PodMonitors to be selected for target discovery. An empty label selector + podMonitorSelector defines the podMonitors to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -5268,7 +5285,7 @@ spec: x-kubernetes-map-type: atomic podTargetLabels: description: |- - PodTargetLabels are appended to the `spec.podTargetLabels` field of all + podTargetLabels are appended to the `spec.podTargetLabels` field of all PodMonitor and ServiceMonitor objects. items: type: string @@ -5276,15 +5293,15 @@ spec: portName: default: web description: |- - Port name used for the pods and governing service. + portName used for the pods and governing service. Default: "web" type: string priorityClassName: - description: Priority class assigned to the Pods. + description: priorityClassName assigned to the Pods. type: string probeNamespaceSelector: description: |- - Namespaces to match for Probe discovery. An empty label + probeNamespaceSelector defines the namespaces to match for Probe discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. properties: @@ -5333,7 +5350,7 @@ spec: x-kubernetes-map-type: atomic probeSelector: description: |- - Probes to be selected for target discovery. An empty label selector + probeSelector defines the probes to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -5390,7 +5407,7 @@ spec: x-kubernetes-map-type: atomic prometheusExternalLabelName: description: |- - Name of Prometheus external label used to denote the Prometheus instance + prometheusExternalLabelName defines the name of Prometheus external label used to denote the Prometheus instance name. The external label will _not_ be added when the field is set to the empty string (`""`). @@ -5398,14 +5415,14 @@ spec: type: string reloadStrategy: description: |- - Defines the strategy used to reload the Prometheus configuration. + reloadStrategy defines the strategy used to reload the Prometheus configuration. If not specified, the configuration is reloaded using the /-/reload HTTP endpoint. enum: - HTTP - ProcessSignal type: string remoteWrite: - description: Defines the list of remote write configurations. + description: remoteWrite defines the list of remote write configurations. items: description: |- RemoteWriteSpec defines the configuration to write samples from Prometheus @@ -5413,15 +5430,15 @@ spec: properties: authorization: description: |- - Authorization section for the URL. + authorization section for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. Cannot be set at the same time as `sigv4`, `basicAuth`, `oauth2`, or `azureAd`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -5445,12 +5462,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -5459,15 +5476,15 @@ spec: type: object azureAd: description: |- - AzureAD for the URL. + azureAd for the URL. It requires Prometheus >= v2.45.0 or Thanos >= v0.31.0. Cannot be set at the same time as `authorization`, `basicAuth`, `oauth2`, or `sigv4`. properties: cloud: - description: The Azure Cloud. Options are 'AzurePublic', - 'AzureChina', or 'AzureGovernment'. + description: cloud defines the Azure Cloud. Options are + 'AzurePublic', 'AzureChina', or 'AzureGovernment'. enum: - AzureChina - AzureGovernment @@ -5475,32 +5492,33 @@ spec: type: string managedIdentity: description: |- - ManagedIdentity defines the Azure User-assigned Managed identity. + managedIdentity defines the Azure User-assigned Managed identity. Cannot be set at the same time as `oauth` or `sdk`. properties: clientId: - description: The client id + description: clientId defines defines the Azure User-assigned + Managed identity. type: string required: - clientId type: object oauth: description: |- - OAuth defines the oauth config that is being used to authenticate. + oauth defines the oauth config that is being used to authenticate. Cannot be set at the same time as `managedIdentity` or `sdk`. It requires Prometheus >= v2.48.0 or Thanos >= v0.31.0. properties: clientId: - description: '`clientID` is the clientId of the Azure + description: clientId defines the clientId of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 type: string clientSecret: - description: '`clientSecret` specifies a key of a Secret + description: clientSecret specifies a key of a Secret containing the client secret of the Azure Active Directory - application that is being used to authenticate.' + application that is being used to authenticate. properties: key: description: The key of the secret to select from. Must @@ -5524,9 +5542,9 @@ spec: type: object x-kubernetes-map-type: atomic tenantId: - description: '`tenantId` is the tenant ID of the Azure + description: tenantId is the tenant ID of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 pattern: ^[0-9a-zA-Z-.]+$ type: string @@ -5537,29 +5555,29 @@ spec: type: object sdk: description: |- - SDK defines the Azure SDK config that is being used to authenticate. + sdk defines the Azure SDK config that is being used to authenticate. See https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication Cannot be set at the same time as `oauth` or `managedIdentity`. It requires Prometheus >= v2.52.0 or Thanos >= v0.36.0. properties: tenantId: - description: '`tenantId` is the tenant ID of the azure + description: tenantId defines the tenant ID of the azure active directory application that is being used to - authenticate.' + authenticate. pattern: ^[0-9a-zA-Z-.]+$ type: string type: object type: object basicAuth: description: |- - BasicAuth configuration for the URL. + basicAuth configuration for the URL. Cannot be set at the same time as `sigv4`, `authorization`, `oauth2`, or `azureAd`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -5585,7 +5603,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -5612,23 +5630,22 @@ spec: type: object bearerToken: description: |- + bearerToken is deprecated: this will be removed in a future release. *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File from which to read bearer token for the URL. + bearerTokenFile defines the file from which to read bearer token for the URL. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. type: boolean @@ -5636,14 +5653,14 @@ spec: additionalProperties: type: string description: |- - Custom HTTP headers to be sent along with each remote write request. + headers defines the custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. It requires Prometheus >= v2.25.0 or Thanos >= v0.24.0. type: object messageVersion: description: |- - The Remote Write message's version to use when writing to the endpoint. + messageVersion defines the Remote Write message's version to use when writing to the endpoint. `Version1.0` corresponds to the `prometheus.WriteRequest` protobuf message introduced in Remote Write 1.0. `Version2.0` corresponds to the `io.prometheus.write.v2.Request` protobuf message introduced in Remote Write 2.0. @@ -5660,37 +5677,37 @@ spec: - V2.0 type: string metadataConfig: - description: MetadataConfig configures the sending of series - metadata to the remote storage. + description: metadataConfig defines how to send a series metadata + to the remote storage. properties: maxSamplesPerSend: description: |- - MaxSamplesPerSend is the maximum number of metadata samples per send. + maxSamplesPerSend defines the maximum number of metadata samples per send. It requires Prometheus >= v2.29.0. format: int32 minimum: -1 type: integer send: - description: Defines whether metric metadata is sent to - the remote storage or not. + description: send defines whether metric metadata is sent + to the remote storage or not. type: boolean sendInterval: - description: Defines how frequently metric metadata is sent - to the remote storage. + description: sendInterval defines how frequently metric + metadata is sent to the remote storage. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object name: description: |- - The name of the remote write queue, it must be unique if specified. The + name of the remote write queue, it must be unique if specified. The name is used in metrics and logging in order to differentiate queues. It requires Prometheus >= v2.15.0 or Thanos >= 0.24.0. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5698,7 +5715,7 @@ spec: type: string oauth2: description: |- - OAuth2 configuration for the URL. + oauth2 configuration for the URL. It requires Prometheus >= v2.27.0 or Thanos >= v0.24.0. @@ -5706,12 +5723,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5734,7 +5751,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5760,7 +5778,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5788,12 +5806,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5827,7 +5845,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5835,33 +5853,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5884,8 +5901,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5910,12 +5927,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5938,8 +5955,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5964,11 +5981,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5993,7 +6011,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6004,7 +6022,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6014,12 +6032,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -6055,7 +6074,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6063,89 +6082,88 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string queueConfig: - description: QueueConfig allows tuning of the remote write queue + description: queueConfig allows tuning of the remote write queue parameters. properties: batchSendDeadline: - description: BatchSendDeadline is the maximum time a sample - will wait in buffer. + description: batchSendDeadline defines the maximum time + a sample will wait in buffer. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string capacity: description: |- - Capacity is the number of samples to buffer per shard before we start + capacity defines the number of samples to buffer per shard before we start dropping them. type: integer maxBackoff: - description: MaxBackoff is the maximum retry delay. + description: maxBackoff defines the maximum retry delay. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string maxRetries: - description: MaxRetries is the maximum number of times to - retry a batch on recoverable errors. + description: maxRetries defines the maximum number of times + to retry a batch on recoverable errors. type: integer maxSamplesPerSend: - description: MaxSamplesPerSend is the maximum number of - samples per send. + description: maxSamplesPerSend defines the maximum number + of samples per send. type: integer maxShards: - description: MaxShards is the maximum number of shards, + description: maxShards defines the maximum number of shards, i.e. amount of concurrency. type: integer minBackoff: - description: MinBackoff is the initial retry delay. Gets - doubled for every retry. + description: minBackoff defines the initial retry delay. + Gets doubled for every retry. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string minShards: - description: MinShards is the minimum number of shards, + description: minShards defines the minimum number of shards, i.e. amount of concurrency. type: integer retryOnRateLimit: description: |- - Retry upon receiving a 429 status code from the remote-write storage. + retryOnRateLimit defines the retry upon receiving a 429 status code from the remote-write storage. This is an *experimental feature*, it may change in any upcoming release in a breaking way. type: boolean sampleAgeLimit: description: |- - SampleAgeLimit drops samples older than the limit. + sampleAgeLimit drops samples older than the limit. It requires Prometheus >= v2.50.0 or Thanos >= v0.32.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object remoteTimeout: - description: Timeout for requests to the remote write endpoint. + description: remoteTimeout defines the timeout for requests + to the remote write endpoint. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string roundRobinDNS: - description: |- - When enabled: - - The remote-write mechanism will resolve the hostname via DNS. - - It will randomly select one of the resolved IP addresses and connect to it. - - When disabled (default behavior): - - The Go standard library will handle hostname resolution. - - It will attempt connections to each resolved IP address sequentially. - - Note: The connection timeout applies to the entire resolution and connection process. - If disabled, the timeout is distributed across all connection attempts. - - It requires Prometheus >= v3.1.0 or Thanos >= v0.38.0. + description: "roundRobinDNS controls the DNS resolution behavior + for remote-write connections.\nWhen enabled:\n - The remote-write + mechanism will resolve the hostname via DNS.\n - It will + randomly select one of the resolved IP addresses and connect + to it.\n\nWhen disabled (default behavior):\n - The Go standard + library will handle hostname resolution.\n - It will attempt + connections to each resolved IP address sequentially.\n\nNote: + The connection timeout applies to the entire resolution and + connection process.\n\n\tIf disabled, the timeout is distributed + across all connection attempts.\n\nIt requires Prometheus + >= v3.1.0 or Thanos >= v0.38.0." type: boolean sendExemplars: description: |- - Enables sending of exemplars over remote write. Note that + sendExemplars enables sending of exemplars over remote write. Note that exemplar-storage itself must be enabled using the `spec.enableFeatures` option for exemplars to be scraped in the first place. @@ -6153,14 +6171,14 @@ spec: type: boolean sendNativeHistograms: description: |- - Enables sending of native histograms, also known as sparse histograms + sendNativeHistograms enables sending of native histograms, also known as sparse histograms over remote write. It requires Prometheus >= v2.40.0 or Thanos >= v0.30.0. type: boolean sigv4: description: |- - Sigv4 allows to configures AWS's Signature Verification 4 for the URL. + sigv4 defines the AWS's Signature Verification 4 for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. @@ -6168,7 +6186,7 @@ spec: properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -6193,18 +6211,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used to authenticate. + description: profile defines the named AWS profile used + to authenticate. type: string region: - description: Region is the AWS region. If blank, the region - from the default credentials chain used. + description: region defines the AWS region. If blank, the + region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used to authenticate. + description: roleArn defines the named AWS profile used + to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -6228,17 +6248,22 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object tlsConfig: - description: TLS Config to use for the URL. + description: tlsConfig to use for the URL. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6261,7 +6286,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6286,15 +6312,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6317,7 +6344,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6342,19 +6370,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6379,7 +6408,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6390,7 +6419,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6400,15 +6429,18 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the endpoint to send samples to. + description: url defines the URL of the endpoint to send samples + to. minLength: 1 type: string writeRelabelConfigs: - description: The list of remote write relabel configurations. + description: writeRelabelConfigs defines the list of remote + write relabel configurations. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -6419,7 +6451,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -6451,41 +6483,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -6500,7 +6532,7 @@ spec: type: array remoteWriteReceiverMessageVersions: description: |- - List of the protobuf message versions to accept when receiving the + remoteWriteReceiverMessageVersions list of the protobuf message versions to accept when receiving the remote writes. It requires Prometheus >= v2.54.0. @@ -6514,7 +6546,7 @@ spec: x-kubernetes-list-type: set replicaExternalLabelName: description: |- - Name of Prometheus external label used to denote the replica name. + replicaExternalLabelName defines the name of Prometheus external label used to denote the replica name. The external label will _not_ be added when the field is set to the empty string (`""`). @@ -6522,7 +6554,7 @@ spec: type: string replicas: description: |- - Number of replicas of each shard to deploy for a Prometheus deployment. + replicas defines the number of replicas of each shard to deploy for a Prometheus deployment. `spec.replicas` multiplied by `spec.shards` is the total number of Pods created. @@ -6530,8 +6562,8 @@ spec: format: int32 type: integer resources: - description: Defines the resources requests and limits of the 'prometheus' - container. + description: resources defines the resources requests and limits of + the 'prometheus' container. properties: claims: description: |- @@ -6591,7 +6623,7 @@ spec: type: object routePrefix: description: |- - The route prefix Prometheus registers HTTP handlers for. + routePrefix defines the route prefix Prometheus registers HTTP handlers for. This is useful when using `spec.externalURL`, and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but @@ -6599,12 +6631,12 @@ spec: for use with `kubectl proxy`. type: string runtime: - description: RuntimeConfig configures the values for the Prometheus - process behavior + description: runtime defines the values for the Prometheus process + behavior properties: goGC: description: |- - The Go garbage collection target percentage. Lowering this number may increase the CPU usage. + goGC defines the Go garbage collection target percentage. Lowering this number may increase the CPU usage. See: https://tip.golang.org/doc/gc-guide#GOGC format: int32 minimum: -1 @@ -6612,7 +6644,7 @@ spec: type: object sampleLimit: description: |- - SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit defines per-scrape limit on number of scraped samples that will be accepted. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -6621,7 +6653,7 @@ spec: type: integer scrapeClasses: description: |- - List of scrape classes to expose to scraping objects such as + scrapeClasses defines the list of scrape classes to expose to scraping objects such as PodMonitors, ServiceMonitors, Probes and ScrapeConfigs. This is an *experimental feature*, it may change in any upcoming release @@ -6630,13 +6662,13 @@ spec: properties: attachMetadata: description: |- - AttachMetadata configures additional metadata to the discovered targets. + attachMetadata defines additional metadata to the discovered targets. When the scrape object defines its own configuration, it takes precedence over the scrape class configuration. properties: node: description: |- - When set to true, Prometheus attaches node metadata to the discovered + node when set to true, Prometheus attaches node metadata to the discovered targets. The Prometheus service account must have the `list` and `watch` @@ -6645,12 +6677,12 @@ spec: type: object authorization: description: |- - Authorization section for the ScrapeClass. + authorization section for the ScrapeClass. It will only apply if the scrape resource doesn't specify any Authorization. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -6674,12 +6706,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6688,14 +6720,14 @@ spec: type: object default: description: |- - Default indicates that the scrape applies to all scrape objects that + default defines that the scrape applies to all scrape objects that don't configure an explicit scrape class name. Only one scrape class can be set as the default. type: boolean fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It will only apply if the scrape resource doesn't specify any FallbackScrapeProtocol It requires Prometheus >= v3.0.0. @@ -6708,7 +6740,7 @@ spec: type: string metricRelabelings: description: |- - MetricRelabelings configures the relabeling rules to apply to all samples before ingestion. + metricRelabelings defines the relabeling rules to apply to all samples before ingestion. The Operator adds the scrape class metric relabelings defined here. Then the Operator adds the target-specific metric relabelings defined in ServiceMonitors, PodMonitors, Probes and ScrapeConfigs. @@ -6725,7 +6757,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -6757,41 +6789,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -6801,12 +6833,12 @@ spec: type: object type: array name: - description: Name of the scrape class. + description: name of the scrape class. minLength: 1 type: string relabelings: description: |- - Relabelings configures the relabeling rules to apply to all scrape targets. + relabelings defines the relabeling rules to apply to all scrape targets. The Operator automatically adds relabelings for a few standard Kubernetes fields like `__meta_kubernetes_namespace` and `__meta_kubernetes_service_name`. @@ -6824,7 +6856,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -6856,41 +6888,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -6901,19 +6933,19 @@ spec: type: array tlsConfig: description: |- - TLSConfig defines the TLS settings to use for the scrape. When the + tlsConfig defines the TLS settings to use for the scrape. When the scrape objects define their own CA, certificate and/or key, they take precedence over the corresponding scrape class fields. For now only the `caFile`, `certFile` and `keyFile` fields are supported. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6936,7 +6968,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6961,15 +6994,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6992,7 +7026,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7017,19 +7052,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7054,7 +7090,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7065,7 +7101,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7075,7 +7111,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -7087,7 +7124,7 @@ spec: x-kubernetes-list-type: map scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. @@ -7095,7 +7132,7 @@ spec: type: boolean scrapeConfigNamespaceSelector: description: |- - Namespaces to match for ScrapeConfig discovery. An empty label selector + scrapeConfigNamespaceSelector defines the namespaces to match for ScrapeConfig discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. @@ -7146,7 +7183,7 @@ spec: x-kubernetes-map-type: atomic scrapeConfigSelector: description: |- - ScrapeConfigs to be selected for target discovery. An empty label + scrapeConfigSelector defines the scrapeConfigs to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -7205,7 +7242,7 @@ spec: x-kubernetes-map-type: atomic scrapeFailureLogFile: description: |- - File to which scrape failures are logged. + scrapeFailureLogFile defines the file to which scrape failures are logged. Reloading the configuration will reopen the file. If the filename has an empty path, e.g. 'file.log', The Prometheus Pods @@ -7218,14 +7255,14 @@ spec: scrapeInterval: default: 30s description: |- - Interval between consecutive scrapes. + scrapeInterval defines interval between consecutive scrapes. Default: "30s" pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string scrapeProtocols: description: |- - The protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -7253,13 +7290,13 @@ spec: x-kubernetes-list-type: set scrapeTimeout: description: |- - Number of seconds to wait until a scrape request times out. + scrapeTimeout defines the number of seconds to wait until a scrape request times out. The value cannot be greater than the scrape interval otherwise the operator will reject the resource. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string secrets: description: |- - Secrets is a list of Secrets in the same namespace as the Prometheus + secrets defines a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. Each Secret is added to the StatefulSet definition as a volume named `secret-`. The Secrets are mounted into /etc/prometheus/secrets/ in the 'prometheus' container. @@ -7269,7 +7306,7 @@ spec: x-kubernetes-list-type: set securityContext: description: |- - SecurityContext holds pod-level security attributes and common container settings. + securityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: appArmorProfile: @@ -7503,12 +7540,12 @@ spec: type: object serviceAccountName: description: |- - ServiceAccountName is the name of the ServiceAccount to use to run the + serviceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods. type: string serviceDiscoveryRole: description: |- - Defines the service discovery role used to discover targets from + serviceDiscoveryRole defines the service discovery role used to discover targets from `ServiceMonitor` objects and Alertmanager endpoints. If set, the value should be either "Endpoints" or "EndpointSlice". @@ -7519,7 +7556,7 @@ spec: type: string serviceMonitorNamespaceSelector: description: |- - Namespaces to match for ServicedMonitors discovery. An empty label selector + serviceMonitorNamespaceSelector defines the namespaces to match for ServicedMonitors discovery. An empty label selector matches all namespaces. A null label selector (default value) matches the current namespace only. properties: @@ -7568,7 +7605,7 @@ spec: x-kubernetes-map-type: atomic serviceMonitorSelector: description: |- - ServiceMonitors to be selected for target discovery. An empty label + serviceMonitorSelector defines the serviceMonitors to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -7625,7 +7662,7 @@ spec: x-kubernetes-map-type: atomic serviceName: description: |- - The name of the service name used by the underlying StatefulSet(s) as the governing service. + serviceName defines the name of the service name used by the underlying StatefulSet(s) as the governing service. If defined, the Service must be created before the Prometheus/PrometheusAgent resource in the same namespace and it must define a selector that matches the pod labels. If empty, the operator will create and manage a headless service named `prometheus-operated` for Prometheus resources, or `prometheus-agent-operated` for PrometheusAgent resources. @@ -7635,7 +7672,7 @@ spec: type: string shards: description: |- - Number of shards to distribute the scraped targets onto. + shards defines the number of shards to distribute the scraped targets onto. `spec.replicas` multiplied by `spec.shards` is the total number of Pods being created. @@ -7664,15 +7701,15 @@ spec: format: int32 type: integer storage: - description: Storage defines the storage used by Prometheus. + description: storage defines the storage used by Prometheus. properties: disableMountSubPath: - description: 'Deprecated: subPath usage will be removed in a future - release.' + description: 'disableMountSubPath deprecated: subPath usage will + be removed in a future release.' type: boolean emptyDir: description: |- - EmptyDirVolumeSource to be used by the StatefulSet. + emptyDir to be used by the StatefulSet. If specified, it takes precedence over `ephemeral` and `volumeClaimTemplate`. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir properties: @@ -7699,7 +7736,7 @@ spec: type: object ephemeral: description: |- - EphemeralVolumeSource to be used by the StatefulSet. + ephemeral to be used by the StatefulSet. This is a beta field in k8s 1.21 and GA in 1.15. For lower versions, starting with k8s 1.19, it requires enabling the GenericEphemeralVolume feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes @@ -7944,7 +7981,7 @@ spec: type: object volumeClaimTemplate: description: |- - Defines the PVC spec to be used by the Prometheus StatefulSets. + volumeClaimTemplate defines the PVC spec to be used by the Prometheus StatefulSets. The easiest way to use a volume that cannot be automatically provisioned is to use a label selector alongside manually created PersistentVolumes. properties: @@ -7964,14 +8001,14 @@ spec: More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: - description: EmbeddedMetadata contains metadata relevant to - an EmbeddedResource. + description: metadata defines EmbeddedMetadata contains metadata + relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -7980,14 +8017,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -7997,7 +8034,7 @@ spec: type: object spec: description: |- - Defines the desired characteristics of a volume requested by a pod author. + spec defines the specification of the characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: accessModes: @@ -8193,7 +8230,7 @@ spec: type: string type: object status: - description: 'Deprecated: this field is never set.' + description: 'status is deprecated: this field is never set.' properties: accessModes: description: |- @@ -8378,7 +8415,7 @@ spec: type: object targetLimit: description: |- - TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit defines a limit on the number of scraped targets that will be accepted. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -8387,7 +8424,7 @@ spec: type: integer terminationGracePeriodSeconds: description: |- - Optional duration in seconds the pod needs to terminate gracefully. + terminationGracePeriodSeconds defines the optional duration in seconds the pod needs to terminate gracefully. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down) which may lead to data corruption. @@ -8396,7 +8433,7 @@ spec: minimum: 0 type: integer tolerations: - description: Defines the Pods' tolerations if specified. + description: tolerations defines the Pods' tolerations if specified. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -8435,12 +8472,14 @@ spec: type: object type: array topologySpreadConstraints: - description: Defines the pod's topology spread constraints if specified. + description: topologySpreadConstraints defines the pod's topology + spread constraints if specified. items: properties: additionalLabelSelectors: - description: Defines what Prometheus Operator managed labels - should be added to labelSelector on the topologySpreadConstraint. + description: additionalLabelSelectors Defines what Prometheus + Operator managed labels should be added to labelSelector on + the topologySpreadConstraint. enum: - OnResource - OnShard @@ -8615,61 +8654,62 @@ spec: type: array tracingConfig: description: |- - TracingConfig configures tracing in Prometheus. + tracingConfig defines tracing in Prometheus. This is an *experimental feature*, it may change in any upcoming release in a breaking way. properties: clientType: - description: Client used to export the traces. Supported values - are `http` or `grpc`. + description: clientType defines the client used to export the + traces. Supported values are `http` or `grpc`. enum: - http - grpc type: string compression: - description: Compression key for supported compression types. + description: compression key for supported compression types. The only supported value is `gzip`. enum: - gzip type: string endpoint: - description: Endpoint to send the traces to. Should be provided + description: endpoint to send the traces to. Should be provided in format :. minLength: 1 type: string headers: additionalProperties: type: string - description: Key-value pairs to be used as headers associated - with gRPC or HTTP requests. + description: headers defines the key-value pairs to be used as + headers associated with gRPC or HTTP requests. type: object insecure: - description: If disabled, the client will use a secure connection. + description: insecure if disabled, the client will use a secure + connection. type: boolean samplingFraction: anyOf: - type: integer - type: string - description: Sets the probability a given trace will be sampled. - Must be a float from 0 through 1. + description: samplingFraction defines the probability a given + trace will be sampled. Must be a float from 0 through 1. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true timeout: - description: Maximum time the exporter will wait for each batch - export. + description: timeout defines the maximum time the exporter will + wait for each batch export. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS Config to use when sending traces. + description: tlsConfig to use when sending traces. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8692,7 +8732,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8717,15 +8758,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8748,7 +8790,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8773,19 +8816,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8810,7 +8854,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8821,7 +8865,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8831,7 +8875,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -8839,12 +8884,12 @@ spec: type: object tsdb: description: |- - Defines the runtime reloadable configuration of the timeseries database(TSDB). + tsdb defines the runtime reloadable configuration of the timeseries database(TSDB). It requires Prometheus >= v2.39.0 or PrometheusAgent >= v2.54.0. properties: outOfOrderTimeWindow: description: |- - Configures how old an out-of-order/out-of-bounds sample can be with + outOfOrderTimeWindow defines how old an out-of-order/out-of-bounds sample can be with respect to the TSDB max time. An out-of-order/out-of-bounds sample is ingested into the TSDB as long as @@ -8859,7 +8904,7 @@ spec: type: object version: description: |- - Version of Prometheus being deployed. The operator uses this information + version of Prometheus being deployed. The operator uses this information to generate the Prometheus StatefulSet + configuration files. If not specified, the operator assumes the latest upstream version of @@ -8868,7 +8913,7 @@ spec: type: string volumeMounts: description: |- - VolumeMounts allows the configuration of additional VolumeMounts. + volumeMounts allows the configuration of additional VolumeMounts. VolumeMounts will be appended to other VolumeMounts in the 'prometheus' container, that are generated as a result of StorageSpec objects. @@ -8936,7 +8981,7 @@ spec: type: array volumes: description: |- - Volumes allows the configuration of additional volumes on the output + volumes allows the configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: @@ -10823,29 +10868,30 @@ spec: type: array walCompression: description: |- - Configures compression of the write-ahead log (WAL) using Snappy. + walCompression defines the compression of the write-ahead log (WAL) using Snappy. WAL compression is enabled by default for Prometheus >= 2.20.0 Requires Prometheus v2.11.0 and above. type: boolean web: - description: Defines the configuration of the Prometheus web server. + description: web defines the configuration of the Prometheus web server. properties: httpConfig: - description: Defines HTTP parameters for web server. + description: httpConfig defines HTTP parameters for web server. properties: headers: - description: List of headers that can be added to HTTP responses. + description: headers defines a list of headers that can be + added to HTTP responses. properties: contentSecurityPolicy: description: |- - Set the Content-Security-Policy header to HTTP responses. + contentSecurityPolicy defines the Content-Security-Policy header to HTTP responses. Unset if blank. type: string strictTransportSecurity: description: |- - Set the Strict-Transport-Security header to HTTP responses. + strictTransportSecurity defines the Strict-Transport-Security header to HTTP responses. Unset if blank. Please make sure that you use this with care as this header might force browsers to load Prometheus and the other applications hosted on the same @@ -10854,7 +10900,7 @@ spec: type: string xContentTypeOptions: description: |- - Set the X-Content-Type-Options header to HTTP responses. + xContentTypeOptions defines the X-Content-Type-Options header to HTTP responses. Unset if blank. Accepted value is nosniff. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options enum: @@ -10863,7 +10909,7 @@ spec: type: string xFrameOptions: description: |- - Set the X-Frame-Options header to HTTP responses. + xFrameOptions defines the X-Frame-Options header to HTTP responses. Unset if blank. Accepted values are deny and sameorigin. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options enum: @@ -10873,42 +10919,42 @@ spec: type: string xXSSProtection: description: |- - Set the X-XSS-Protection header to all responses. + xXSSProtection defines the X-XSS-Protection header to all responses. Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection type: string type: object http2: description: |- - Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. + http2 enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. When TLSConfig is not configured, HTTP/2 will be disabled. Whenever the value of the field changes, a rolling update will be triggered. type: boolean type: object maxConnections: description: |- - Defines the maximum number of simultaneous connections + maxConnections defines the maximum number of simultaneous connections A zero value means that Prometheus doesn't accept any incoming connection. format: int32 minimum: 0 type: integer pageTitle: - description: The prometheus web page title. + description: pageTitle defines the prometheus web page title. type: string tlsConfig: - description: Defines the TLS parameters for HTTPS. + description: tlsConfig defines the TLS parameters for HTTPS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10931,7 +10977,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10957,7 +11004,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -10965,7 +11012,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -10975,14 +11022,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11005,7 +11052,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11031,21 +11079,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -11055,7 +11103,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -11063,7 +11111,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -11091,14 +11139,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -11124,18 +11174,19 @@ spec: rule: '!(has(self.mode) && self.mode == ''DaemonSet'' && has(self.probeSelector))' status: description: |- - Most recent observed status of the Prometheus cluster. Read-only. + status defines the most recent observed status of the Prometheus cluster. Read-only. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment. format: int32 type: integer conditions: - description: The current state of the Prometheus deployment. + description: conditions defines the current state of the Prometheus + deployment. items: description: |- Condition represents the state of the resources associated with the @@ -11147,12 +11198,12 @@ spec: format: date-time type: string message: - description: Human-readable message indicating details for the - condition's last transition. + description: message defines human-readable message indicating + details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the @@ -11160,14 +11211,14 @@ spec: format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: - description: Type of the condition being reported. + description: type of the condition being reported. minLength: 1 type: string required: @@ -11181,45 +11232,46 @@ spec: x-kubernetes-list-type: map paused: description: |- - Represents whether any actions on the underlying managed objects are + paused defines whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: |- - Total number of non-terminated pods targeted by this Prometheus deployment + replicas defines the total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector). format: int32 type: integer selector: - description: The selector used to match the pods targeted by this - Prometheus resource. + description: selector used to match the pods targeted by this Prometheus + resource. type: string shardStatuses: - description: The list has one entry per shard. Each entry provides - a summary of the shard status. + description: shardStatuses defines the list has one entry per shard. + Each entry provides a summary of the shard status. items: properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this shard. format: int32 type: integer replicas: - description: Total number of pods targeted by this shard. + description: replicas defines the total number of pods targeted + by this shard. format: int32 type: integer shardID: - description: Identifier of the shard. + description: shardID defines the identifier of the shard. type: string unavailableReplicas: - description: Total number of unavailable pods targeted by this - shard. + description: unavailableReplicas defines the Total number of + unavailable pods targeted by this shard. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this shard + updatedReplicas defines the total number of non-terminated pods targeted by this shard that have the desired spec. format: int32 type: integer @@ -11235,26 +11287,20 @@ spec: - shardID x-kubernetes-list-type: map shards: - description: Shards is the most recently observed number of shards. + description: shards defines the most recently observed number of shards. format: int32 type: integer unavailableReplicas: - description: Total number of unavailable pods targeted by this Prometheus - deployment. + description: unavailableReplicas defines the total number of unavailable + pods targeted by this Prometheus deployment. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this Prometheus deployment + updatedReplicas defines the total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec. format: int32 type: integer - required: - - availableReplicas - - paused - - replicas - - unavailableReplicas - - updatedReplicas type: object required: - spec diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_prometheuses.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_prometheuses.yaml index c5420605b20..0c2d581d96b 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_prometheuses.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_prometheuses.yaml @@ -76,12 +76,12 @@ spec: type: object spec: description: |- - Specification of the desired behavior of the Prometheus cluster. More info: + spec defines the specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: additionalAlertManagerConfigs: description: |- - AdditionalAlertManagerConfigs specifies a key of a Secret containing + additionalAlertManagerConfigs defines a key of a Secret containing additional Prometheus Alertmanager configurations. The Alertmanager configurations are appended to the configuration generated by the Prometheus Operator. They must be formatted according to the official @@ -118,7 +118,7 @@ spec: x-kubernetes-map-type: atomic additionalAlertRelabelConfigs: description: |- - AdditionalAlertRelabelConfigs specifies a key of a Secret containing + additionalAlertRelabelConfigs defines a key of a Secret containing additional Prometheus alert relabel configurations. The alert relabel configurations are appended to the configuration generated by the Prometheus Operator. They must be formatted according to the official @@ -155,7 +155,7 @@ spec: x-kubernetes-map-type: atomic additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the 'prometheus' container. + additionalArgs allows setting additional arguments for the 'prometheus' container. It is intended for e.g. activating hidden flags which are not supported by the dedicated configuration options yet. The arguments are passed as-is to the @@ -169,12 +169,12 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for name-only - arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. Can + be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name @@ -182,7 +182,7 @@ spec: type: array additionalScrapeConfigs: description: |- - AdditionalScrapeConfigs allows specifying a key of a Secret containing + additionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified @@ -215,7 +215,8 @@ spec: type: object x-kubernetes-map-type: atomic affinity: - description: Defines the Pods' affinity scheduling rules if specified. + description: affinity defines the Pods' affinity scheduling rules + if specified. properties: nodeAffinity: description: Describes node affinity scheduling rules for the @@ -1127,10 +1128,10 @@ spec: type: object type: object alerting: - description: Defines the settings related to Alertmanager. + description: alerting defines the settings related to Alertmanager. properties: alertmanagers: - description: Alertmanager endpoints where Prometheus should send + description: alertmanagers endpoints where Prometheus should send alerts to. items: description: |- @@ -1139,7 +1140,7 @@ spec: properties: alertRelabelings: description: |- - Relabeling configs applied before sending alerts to a specific Alertmanager. + alertRelabelings defines the relabeling configs applied before sending alerts to a specific Alertmanager. It requires Prometheus >= v2.51.0. items: description: |- @@ -1151,7 +1152,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -1183,41 +1184,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the - extracted value is matched. + description: regex defines the regular expression + against which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated - SourceLabels. + description: separator defines the string between + concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -1228,7 +1229,7 @@ spec: type: array apiVersion: description: |- - Version of the Alertmanager API that Prometheus uses to send alerts. + apiVersion defines the version of the Alertmanager API that Prometheus uses to send alerts. It can be "V1" or "V2". The field has no effect for Prometheus >= v3.0.0 because only the v2 API is supported. enum: @@ -1239,13 +1240,13 @@ spec: type: string authorization: description: |- - Authorization section for Alertmanager. + authorization section for Alertmanager. Cannot be set at the same time as `basicAuth`, `bearerTokenFile` or `sigv4`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in + the namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -1270,7 +1271,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1279,13 +1280,13 @@ spec: type: object basicAuth: description: |- - BasicAuth configuration for Alertmanager. + basicAuth configuration for Alertmanager. Cannot be set at the same time as `bearerTokenFile`, `authorization` or `sigv4`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1311,7 +1312,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1338,22 +1339,22 @@ spec: type: object bearerTokenFile: description: |- - File to read bearer token for Alertmanager. + bearerTokenFile defines the file to read bearer token for Alertmanager. Cannot be set at the same time as `basicAuth`, `authorization`, or `sigv4`. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string enableHttp2: - description: Whether to enable HTTP2. + description: enableHttp2 defines whether to enable HTTP2. type: boolean name: - description: Name of the Endpoints object in the namespace. + description: name of the Endpoints object in the namespace. minLength: 1 type: string namespace: description: |- - Namespace of the Endpoints object. + namespace of the Endpoints object. If not set, the object will be discovered in the namespace of the Prometheus object. @@ -1361,21 +1362,21 @@ spec: type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string pathPrefix: - description: Prefix for the HTTP path alerts are pushed - to. + description: pathPrefix defines the prefix for the HTTP + path alerts are pushed to. type: string port: anyOf: - type: integer - type: string - description: Port on which the Alertmanager API is exposed. + description: port on which the Alertmanager API is exposed. x-kubernetes-int-or-string: true proxyConnectHeader: additionalProperties: @@ -1405,7 +1406,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1413,18 +1414,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string relabelings: - description: Relabel configuration applied to the discovered - Alertmanagers. + description: relabelings defines the relabel configuration + applied to the discovered Alertmanagers. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -1435,7 +1435,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -1467,41 +1467,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the - extracted value is matched. + description: regex defines the regular expression + against which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated - SourceLabels. + description: separator defines the string between + concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -1511,11 +1511,11 @@ spec: type: object type: array scheme: - description: Scheme to use when firing alerts. + description: scheme to use when firing alerts. type: string sigv4: description: |- - Sigv4 allows to configures AWS's Signature Verification 4 for the URL. + sigv4 defines AWS's Signature Verification 4 for the URL. It requires Prometheus >= v2.48.0. @@ -1523,7 +1523,7 @@ spec: properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -1548,20 +1548,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used to - authenticate. + description: profile defines the named AWS profile used + to authenticate. type: string region: - description: Region is the AWS region. If blank, the - region from the default credentials chain used. + description: region defines the AWS region. If blank, + the region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used to - authenticate. + description: roleArn defines the named AWS profile used + to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -1585,22 +1585,27 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object timeout: - description: Timeout is a per-target Alertmanager timeout + description: timeout defines a per-target Alertmanager timeout when pushing alerts. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS Config to use for Alertmanager. + description: tlsConfig to use for Alertmanager. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1623,8 +1628,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1649,16 +1654,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert + in the Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1681,8 +1686,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1707,19 +1712,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client + cert file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client + key file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1744,7 +1750,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1755,7 +1761,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1765,7 +1771,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object required: @@ -1778,14 +1785,14 @@ spec: type: object allowOverlappingBlocks: description: |- - AllowOverlappingBlocks enables vertical compaction and vertical query + allowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. Deprecated: this flag has no effect for Prometheus >= 2.39.0 where overlapping blocks are enabled by default. type: boolean apiserverConfig: description: |- - APIServerConfig allows specifying a host and auth methods to access the + apiserverConfig allows specifying a host and auth methods to access the Kuberntees API server. If null, Prometheus is assumed to run inside of the cluster: it will discover the API servers automatically and use the Pod's CA certificate @@ -1793,14 +1800,14 @@ spec: properties: authorization: description: |- - Authorization section for the API server. + authorization section for the API server. Cannot be set at the same time as `basicAuth`, `bearerToken`, or `bearerTokenFile`. properties: credentials: - description: Selects a key of a Secret in the namespace that - contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -1824,12 +1831,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a secret + from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1838,14 +1845,14 @@ spec: type: object basicAuth: description: |- - BasicAuth configuration for the API server. + basicAuth configuration for the API server. Cannot be set at the same time as `authorization`, `bearerToken`, or `bearerTokenFile`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1871,7 +1878,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1898,14 +1905,13 @@ spec: type: object bearerToken: description: |- - *Warning: this field shouldn't be used because the token value appears + bearerToken is deprecated: this will be removed in a future release. + *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File to read bearer token for accessing apiserver. + bearerTokenFile defines the file to read bearer token for accessing apiserver. Cannot be set at the same time as `basicAuth`, `authorization`, or `bearerToken`. @@ -1913,12 +1919,12 @@ spec: type: string host: description: |- - Kubernetes API address consisting of a hostname or IP address followed + host defines the Kubernetes API address consisting of a hostname or IP address followed by an optional port number. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1952,7 +1958,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1960,24 +1966,24 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS Config to use for the API server. + description: tlsConfig to use for the API server. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2000,7 +2006,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2025,15 +2032,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2056,7 +2064,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2081,19 +2090,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2118,7 +2128,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2129,7 +2139,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2139,7 +2149,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -2147,7 +2158,7 @@ spec: type: object arbitraryFSAccessThroughSMs: description: |- - When true, ServiceMonitor, PodMonitor and Probe object are forbidden to + arbitraryFSAccessThroughSMs when true, ServiceMonitor, PodMonitor and Probe object are forbidden to reference arbitrary files on the file system of the 'prometheus' container. When a ServiceMonitor's endpoint specifies a `bearerTokenFile` value @@ -2159,22 +2170,27 @@ spec: `spec.bearerTokenSecret` field. properties: deny: + description: |- + deny prevents service monitors from accessing arbitrary files on the file system. + When true, service monitors cannot use file-based configurations like BearerTokenFile + that could potentially access sensitive files. When false (default), such access is allowed. + Setting this to true enhances security by preventing potential credential theft attacks. type: boolean type: object automountServiceAccountToken: description: |- - AutomountServiceAccountToken indicates whether a service account token should be automatically mounted in the pod. + automountServiceAccountToken defines whether a service account token should be automatically mounted in the pod. If the field isn't set, the operator mounts the service account token by default. **Warning:** be aware that by default, Prometheus requires the service account token for Kubernetes service discovery. It is possible to use strategic merge patch to project the service account token into the 'prometheus' container. type: boolean baseImage: - description: 'Deprecated: use ''spec.image'' instead.' + description: 'baseImage is deprecated: use ''spec.image'' instead.' type: string bodySizeLimit: description: |- - BodySizeLimit defines per-scrape on response body size. + bodySizeLimit defines per-scrape on response body size. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -2183,7 +2199,7 @@ spec: type: string configMaps: description: |- - ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus + configMaps defines a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. Each ConfigMap is added to the StatefulSet definition as a volume named `configmap-`. The ConfigMaps are mounted into /etc/prometheus/configmaps/ in the 'prometheus' container. @@ -2192,7 +2208,7 @@ spec: type: array containers: description: |- - Containers allows injecting additional containers or modifying operator + containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to the Pods or to change the behavior of an operator generated container. Containers described here modify an operator generated @@ -3728,23 +3744,23 @@ spec: type: array convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.4.0. type: boolean disableCompaction: description: |- - When true, the Prometheus compaction is disabled. + disableCompaction when true, the Prometheus compaction is disabled. When `spec.thanos.objectStorageConfig` or `spec.objectStorageConfigFile` are defined, the operator automatically disables block compaction to avoid race conditions during block uploads (as the Thanos documentation recommends). type: boolean dnsConfig: - description: Defines the DNS configuration for the pods. + description: dnsConfig defines the DNS configuration for the pods. properties: nameservers: description: |- - A list of DNS name server IP addresses. + nameservers defines the list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. items: minLength: 1 @@ -3753,7 +3769,7 @@ spec: x-kubernetes-list-type: set options: description: |- - A list of DNS resolver options. + options defines the list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Resolution options given in Options will override those that appear in the base DNSPolicy. @@ -3762,11 +3778,11 @@ spec: of a pod. properties: name: - description: Name is required and must be unique. + description: name is required and must be unique. minLength: 1 type: string value: - description: Value is optional. + description: value is optional. type: string required: - name @@ -3777,7 +3793,7 @@ spec: x-kubernetes-list-type: map searches: description: |- - A list of DNS search domains for host-name lookup. + searches defines the list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. items: minLength: 1 @@ -3786,7 +3802,7 @@ spec: x-kubernetes-list-type: set type: object dnsPolicy: - description: Defines the DNS policy for the pods. + description: dnsPolicy defines the DNS policy for the pods. enum: - ClusterFirstWithHostNet - ClusterFirst @@ -3795,7 +3811,7 @@ spec: type: string enableAdminAPI: description: |- - Enables access to the Prometheus web admin API. + enableAdminAPI defines access to the Prometheus web admin API. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the @@ -3807,7 +3823,7 @@ spec: type: boolean enableFeatures: description: |- - Enable access to Prometheus feature flags. By default, no features are enabled. + enableFeatures enables access to Prometheus feature flags. By default, no features are enabled. Enabling features which are disabled by default is entirely outside the scope of what the maintainers will support and by doing so, you accept @@ -3821,7 +3837,7 @@ spec: x-kubernetes-list-type: set enableOTLPReceiver: description: |- - Enable Prometheus to be used as a receiver for the OTLP Metrics protocol. + enableOTLPReceiver defines the Prometheus to be used as a receiver for the OTLP Metrics protocol. Note that the OTLP receiver endpoint is automatically enabled if `.spec.otlpConfig` is defined. @@ -3829,7 +3845,7 @@ spec: type: boolean enableRemoteWriteReceiver: description: |- - Enable Prometheus to be used as a receiver for the Prometheus remote + enableRemoteWriteReceiver defines the Prometheus to be used as a receiver for the Prometheus remote write protocol. WARNING: This is not considered an efficient way of ingesting samples. @@ -3841,12 +3857,12 @@ spec: It requires Prometheus >= v2.33.0. type: boolean enableServiceLinks: - description: Indicates whether information about services should be - injected into pod's environment variables + description: enableServiceLinks defines whether information about + services should be injected into pod's environment variables type: boolean enforcedBodySizeLimit: description: |- - When defined, enforcedBodySizeLimit specifies a global limit on the size + enforcedBodySizeLimit when defined specifies a global limit on the size of uncompressed response body that will be accepted by Prometheus. Targets responding with a body larger than this many bytes will cause the scrape to fail. @@ -3862,7 +3878,7 @@ spec: type: string enforcedKeepDroppedTargets: description: |- - When defined, enforcedKeepDroppedTargets specifies a global limit on the number of targets + enforcedKeepDroppedTargets when defined specifies a global limit on the number of targets dropped by relabeling that will be kept in memory. The value overrides any `spec.keepDroppedTargets` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.keepDroppedTargets` is @@ -3879,7 +3895,7 @@ spec: type: integer enforcedLabelLimit: description: |- - When defined, enforcedLabelLimit specifies a global limit on the number + enforcedLabelLimit when defined specifies a global limit on the number of labels per sample. The value overrides any `spec.labelLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelLimit` is greater than zero and less than `spec.enforcedLabelLimit`. @@ -3895,7 +3911,7 @@ spec: type: integer enforcedLabelNameLengthLimit: description: |- - When defined, enforcedLabelNameLengthLimit specifies a global limit on the length + enforcedLabelNameLengthLimit when defined specifies a global limit on the length of labels name per sample. The value overrides any `spec.labelNameLengthLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelNameLengthLimit` is greater than zero and less than `spec.enforcedLabelNameLengthLimit`. @@ -3911,7 +3927,7 @@ spec: type: integer enforcedLabelValueLengthLimit: description: |- - When not null, enforcedLabelValueLengthLimit defines a global limit on the length + enforcedLabelValueLengthLimit when not null defines a global limit on the length of labels value per sample. The value overrides any `spec.labelValueLengthLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelValueLengthLimit` is greater than zero and less than `spec.enforcedLabelValueLengthLimit`. @@ -3927,7 +3943,7 @@ spec: type: integer enforcedNamespaceLabel: description: |- - When not empty, a label will be added to: + enforcedNamespaceLabel when not empty, a label will be added to: 1. All metrics scraped from `ServiceMonitor`, `PodMonitor`, `Probe` and `ScrapeConfig` objects. 2. All metrics generated from recording rules defined in `PrometheusRule` objects. @@ -3942,7 +3958,7 @@ spec: type: string enforcedSampleLimit: description: |- - When defined, enforcedSampleLimit specifies a global limit on the number + enforcedSampleLimit when defined specifies a global limit on the number of scraped samples that will be accepted. This overrides any `spec.sampleLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.sampleLimit` is greater than zero and less than @@ -3960,7 +3976,7 @@ spec: type: integer enforcedTargetLimit: description: |- - When defined, enforcedTargetLimit specifies a global limit on the number + enforcedTargetLimit when defined specifies a global limit on the number of scraped targets. The value overrides any `spec.targetLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.targetLimit` is greater than zero and less than `spec.enforcedTargetLimit`. @@ -3978,13 +3994,13 @@ spec: evaluationInterval: default: 30s description: |- - Interval between rule evaluations. + evaluationInterval defines the interval between rule evaluations. Default: "30s" pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string excludedFromEnforcement: description: |- - List of references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + excludedFromEnforcement defines the list of references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects to be excluded from enforcing a namespace label of origin. It is only applicable if `spec.enforcedNamespaceLabel` set to true. @@ -3994,23 +4010,23 @@ spec: properties: group: default: monitoring.coreos.com - description: Group of the referent. When not specified, it defaults + description: group of the referent. When not specified, it defaults to `monitoring.coreos.com` enum: - monitoring.coreos.com type: string name: - description: Name of the referent. When not set, all resources + description: name of the referent. When not set, all resources in the namespace are matched. type: string namespace: description: |- - Namespace of the referent. + namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ minLength: 1 type: string resource: - description: Resource of the referent. + description: resource of the referent. enum: - prometheusrules - servicemonitors @@ -4025,12 +4041,12 @@ spec: type: array exemplars: description: |- - Exemplars related settings that are runtime reloadable. + exemplars related settings that are runtime reloadable. It requires to enable the `exemplar-storage` feature flag to be effective. properties: maxSize: description: |- - Maximum number of exemplars stored in memory for all series. + maxSize defines the maximum number of exemplars stored in memory for all series. exemplar-storage itself must be enabled using the `spec.enableFeature` option for exemplars to be scraped in the first place. @@ -4044,20 +4060,20 @@ spec: additionalProperties: type: string description: |- - The labels to add to any time series or alerts when communicating with + externalLabels defines the labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager). Labels defined by `spec.replicaExternalLabelName` and `spec.prometheusExternalLabelName` take precedence over this list. type: object externalUrl: description: |- - The external URL under which the Prometheus service is externally + externalUrl defines the external URL under which the Prometheus service is externally available. This is necessary to generate correct URLs (for instance if Prometheus is accessible behind an Ingress resource). type: string hostAliases: description: |- - Optional list of hosts and IPs that will be injected into the Pod's + hostAliases defines the optional list of hosts and IPs that will be injected into the Pod's hosts file if specified. items: description: |- @@ -4065,12 +4081,12 @@ spec: pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. + description: hostnames defines hostnames for the above IP address. items: type: string type: array ip: - description: IP address of the host file entry. + description: ip defines the IP address of the host file entry. type: string required: - hostnames @@ -4082,7 +4098,7 @@ spec: x-kubernetes-list-type: map hostNetwork: description: |- - Use the host's network namespace if true. + hostNetwork defines the host's network namespace if true. Make sure to understand the security implications if you want to enable it (https://kubernetes.io/docs/concepts/configuration/overview/ ). @@ -4093,7 +4109,7 @@ spec: type: boolean hostUsers: description: |- - HostUsers supports the user space in Kubernetes. + hostUsers supports the user space in Kubernetes. More info: https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/ @@ -4102,14 +4118,14 @@ spec: type: boolean ignoreNamespaceSelectors: description: |- - When true, `spec.namespaceSelector` from all PodMonitor, ServiceMonitor + ignoreNamespaceSelectors when true, `spec.namespaceSelector` from all PodMonitor, ServiceMonitor and Probe objects will be ignored. They will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object. type: boolean image: description: |- - Container image name for Prometheus. If specified, it takes precedence + image defines the container image name for Prometheus. If specified, it takes precedence over the `spec.baseImage`, `spec.tag` and `spec.sha` fields. Specifying `spec.version` is still necessary to ensure the Prometheus @@ -4121,7 +4137,7 @@ spec: type: string imagePullPolicy: description: |- - Image pull policy for the 'prometheus', 'init-config-reloader' and 'config-reloader' containers. + imagePullPolicy defines the image pull policy for the 'prometheus', 'init-config-reloader' and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. enum: - "" @@ -4131,7 +4147,7 @@ spec: type: string imagePullSecrets: description: |- - An optional list of references to Secrets in the same namespace + imagePullSecrets defines an optional list of references to Secrets in the same namespace to use for pulling images from registries. See http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod items: @@ -4153,7 +4169,7 @@ spec: type: array initContainers: description: |- - InitContainers allows injecting initContainers to the Pod definition. Those + initContainers allows injecting initContainers to the Pod definition. Those can be used to e.g. fetch secrets for injection into the Prometheus configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: @@ -5689,7 +5705,7 @@ spec: type: array keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -5700,7 +5716,7 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -5709,7 +5725,7 @@ spec: type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -5718,7 +5734,7 @@ spec: type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -5727,11 +5743,11 @@ spec: type: integer listenLocal: description: |- - When true, the Prometheus server listens on the loopback address + listenLocal when true, the Prometheus server listens on the loopback address instead of the Pod IP's address. type: boolean logFormat: - description: Log format for Log level for Prometheus and the config-reloader + description: logFormat for Log level for Prometheus and the config-reloader sidecar. enum: - "" @@ -5739,7 +5755,7 @@ spec: - json type: string logLevel: - description: Log level for Prometheus and the config-reloader sidecar. + description: logLevel for Prometheus and the config-reloader sidecar. enum: - "" - debug @@ -5749,14 +5765,14 @@ spec: type: string maximumStartupDurationSeconds: description: |- - Defines the maximum time that the `prometheus` container's startup probe will wait before being considered failed. The startup probe will return success after the WAL replay is complete. - If set, the value should be greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 minutes). + maximumStartupDurationSeconds defines the maximum time that the `prometheus` container's startup probe will wait before being considered failed. The startup probe will return success after the WAL replay is complete. + If set, the value should be greater than 60 (seconds). Otherwise it will be equal to 900 seconds (15 minutes). format: int32 minimum: 60 type: integer minReadySeconds: description: |- - Minimum number of seconds for which a newly created Pod should be ready + minReadySeconds defines the minimum number of seconds for which a newly created Pod should be ready without any of its container crashing for it to be considered available. If unset, pods will be considered available as soon as they are ready. @@ -5765,7 +5781,7 @@ spec: type: integer nameEscapingScheme: description: |- - Specifies the character escaping scheme that will be requested when scraping + nameEscapingScheme defines the character escaping scheme that will be requested when scraping for metric and label names that do not conform to the legacy Prometheus character set. @@ -5778,7 +5794,7 @@ spec: type: string nameValidationScheme: description: |- - Specifies the validation scheme for metric and label names. + nameValidationScheme defines the validation scheme for metric and label names. It requires Prometheus >= v2.55.0. enum: @@ -5788,21 +5804,21 @@ spec: nodeSelector: additionalProperties: type: string - description: Defines on which Nodes the Pods are scheduled. + description: nodeSelector defines on which Nodes the Pods are scheduled. type: object otlp: description: |- - Settings related to the OTLP receiver feature. + otlp defines the settings related to the OTLP receiver feature. It requires Prometheus >= v2.55.0. properties: convertHistogramsToNHCB: description: |- - Configures optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. + convertHistogramsToNHCB defines optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. It requires Prometheus >= v3.4.0. type: boolean ignoreResourceAttributes: description: |- - List of OpenTelemetry resource attributes to ignore when `promoteAllResourceAttributes` is true. + ignoreResourceAttributes defines the list of OpenTelemetry resource attributes to ignore when `promoteAllResourceAttributes` is true. It requires `promoteAllResourceAttributes` to be true. It requires Prometheus >= v3.5.0. @@ -5814,21 +5830,21 @@ spec: x-kubernetes-list-type: set keepIdentifyingResourceAttributes: description: |- - Enables adding `service.name`, `service.namespace` and `service.instance.id` + keepIdentifyingResourceAttributes enables adding `service.name`, `service.namespace` and `service.instance.id` resource attributes to the `target_info` metric, on top of converting them into the `instance` and `job` labels. It requires Prometheus >= v3.1.0. type: boolean promoteAllResourceAttributes: description: |- - Promote all resource attributes to metric labels except the ones defined in `ignoreResourceAttributes`. + promoteAllResourceAttributes promotes all resource attributes to metric labels except the ones defined in `ignoreResourceAttributes`. Cannot be true when `promoteResourceAttributes` is defined. It requires Prometheus >= v3.5.0. type: boolean promoteResourceAttributes: description: |- - List of OpenTelemetry Attributes that should be promoted to metric labels, defaults to none. + promoteResourceAttributes defines the list of OpenTelemetry Attributes that should be promoted to metric labels, defaults to none. Cannot be defined when `promoteAllResourceAttributes` is true. items: minLength: 1 @@ -5836,39 +5852,46 @@ spec: minItems: 1 type: array x-kubernetes-list-type: set + promoteScopeMetadata: + description: |- + promoteScopeMetadata controls whether to promote OpenTelemetry scope metadata (i.e. name, version, schema URL, and attributes) to metric labels. + As per the OpenTelemetry specification, the aforementioned scope metadata should be identifying, i.e. made into metric labels. + It requires Prometheus >= v3.6.0. + type: boolean translationStrategy: description: |- - Configures how the OTLP receiver endpoint translates the incoming metrics. + translationStrategy defines how the OTLP receiver endpoint translates the incoming metrics. It requires Prometheus >= v3.0.0. enum: - NoUTF8EscapingWithSuffixes - UnderscoreEscapingWithSuffixes - NoTranslation + - UnderscoreEscapingWithoutSuffixes type: string type: object overrideHonorLabels: description: |- - When true, Prometheus resolves label conflicts by renaming the labels in the scraped data + overrideHonorLabels when true, Prometheus resolves label conflicts by renaming the labels in the scraped data to “exported_” for all targets created from ServiceMonitor, PodMonitor and ScrapeConfig objects. Otherwise the HonorLabels field of the service or pod monitor applies. - In practice,`overrideHonorLaels:true` enforces `honorLabels:false` + In practice,`OverrideHonorLabels:true` enforces `honorLabels:false` for all ServiceMonitor, PodMonitor and ScrapeConfig objects. type: boolean overrideHonorTimestamps: description: |- - When true, Prometheus ignores the timestamps for all the targets created + overrideHonorTimestamps when true, Prometheus ignores the timestamps for all the targets created from service and pod monitors. Otherwise the HonorTimestamps field of the service or pod monitor applies. type: boolean paused: description: |- - When a Prometheus deployment is paused, no actions except for deletion + paused defines when a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects. type: boolean persistentVolumeClaimRetentionPolicy: description: |- - The field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. + persistentVolumeClaimRetentionPolicy defines the field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. The default behavior is all PVCs are retained. This is an alpha field from kubernetes 1.23 until 1.26 and a beta field from 1.26. It requires enabling the StatefulSetAutoDeletePVC feature gate. @@ -5891,7 +5914,7 @@ spec: type: object podMetadata: description: |- - PodMetadata configures labels and annotations which are propagated to the Prometheus pods. + podMetadata defines labels and annotations which are propagated to the Prometheus pods. The following items are reserved and cannot be overridden: * "prometheus" label, set to the name of the Prometheus object. @@ -5907,7 +5930,7 @@ spec: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -5916,14 +5939,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -5933,7 +5956,7 @@ spec: type: object podMonitorNamespaceSelector: description: |- - Namespaces to match for PodMonitors discovery. An empty label selector + podMonitorNamespaceSelector defines the namespaces to match for PodMonitors discovery. An empty label selector matches all namespaces. A null label selector (default value) matches the current namespace only. properties: @@ -5982,7 +6005,7 @@ spec: x-kubernetes-map-type: atomic podMonitorSelector: description: |- - PodMonitors to be selected for target discovery. An empty label selector + podMonitorSelector defines the podMonitors to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -6039,7 +6062,7 @@ spec: x-kubernetes-map-type: atomic podTargetLabels: description: |- - PodTargetLabels are appended to the `spec.podTargetLabels` field of all + podTargetLabels are appended to the `spec.podTargetLabels` field of all PodMonitor and ServiceMonitor objects. items: type: string @@ -6047,15 +6070,15 @@ spec: portName: default: web description: |- - Port name used for the pods and governing service. + portName used for the pods and governing service. Default: "web" type: string priorityClassName: - description: Priority class assigned to the Pods. + description: priorityClassName assigned to the Pods. type: string probeNamespaceSelector: description: |- - Namespaces to match for Probe discovery. An empty label + probeNamespaceSelector defines the namespaces to match for Probe discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. properties: @@ -6104,7 +6127,7 @@ spec: x-kubernetes-map-type: atomic probeSelector: description: |- - Probes to be selected for target discovery. An empty label selector + probeSelector defines the probes to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -6161,7 +6184,7 @@ spec: x-kubernetes-map-type: atomic prometheusExternalLabelName: description: |- - Name of Prometheus external label used to denote the Prometheus instance + prometheusExternalLabelName defines the name of Prometheus external label used to denote the Prometheus instance name. The external label will _not_ be added when the field is set to the empty string (`""`). @@ -6169,7 +6192,7 @@ spec: type: string prometheusRulesExcludedFromEnforce: description: |- - Defines the list of PrometheusRule objects to which the namespace label + prometheusRulesExcludedFromEnforce defines the list of PrometheusRule objects to which the namespace label enforcement doesn't apply. This is only relevant when `spec.enforcedNamespaceLabel` is set to true. Deprecated: use `spec.excludedFromEnforcement` instead. @@ -6180,10 +6203,12 @@ spec: namespace label for alerts and metrics. properties: ruleName: - description: Name of the excluded PrometheusRule object. + description: ruleName defines the name of the excluded PrometheusRule + object. type: string ruleNamespace: - description: Namespace of the excluded PrometheusRule object. + description: ruleNamespace defines the namespace of the excluded + PrometheusRule object. type: string required: - ruleName @@ -6191,27 +6216,29 @@ spec: type: object type: array query: - description: QuerySpec defines the configuration of the Promethus - query service. + description: query defines the configuration of the Prometheus query + service. properties: lookbackDelta: - description: The delta difference allowed for retrieving metrics - during expression evaluations. + description: lookbackDelta defines the delta difference allowed + for retrieving metrics during expression evaluations. type: string maxConcurrency: - description: Number of concurrent queries that can be run at once. + description: maxConcurrency defines the number of concurrent queries + that can be run at once. format: int32 minimum: 1 type: integer maxSamples: description: |- - Maximum number of samples a single query can load into memory. Note that + maxSamples defines the maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return. format: int32 type: integer timeout: - description: Maximum time a query may take before being aborted. + description: timeout defines the maximum time a query may take + before being aborted. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object @@ -6231,14 +6258,14 @@ spec: type: string reloadStrategy: description: |- - Defines the strategy used to reload the Prometheus configuration. + reloadStrategy defines the strategy used to reload the Prometheus configuration. If not specified, the configuration is reloaded using the /-/reload HTTP endpoint. enum: - HTTP - ProcessSignal type: string remoteRead: - description: Defines the list of remote read configurations. + description: remoteRead defines the list of remote read configurations. items: description: |- RemoteReadSpec defines the configuration for Prometheus to read back samples @@ -6246,15 +6273,15 @@ spec: properties: authorization: description: |- - Authorization section for the URL. + authorization section for the URL. It requires Prometheus >= v2.26.0. Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -6278,12 +6305,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6292,13 +6319,13 @@ spec: type: object basicAuth: description: |- - BasicAuth configuration for the URL. + basicAuth configuration for the URL. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -6324,7 +6351,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -6351,26 +6378,25 @@ spec: type: object bearerToken: description: |- + bearerToken is deprecated: this will be removed in a future release. *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File from which to read the bearer token for the URL. + bearerTokenFile defines the file from which to read the bearer token for the URL. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string filterExternalLabels: description: |- - Whether to use the external labels as selectors for the remote read endpoint. + filterExternalLabels defines whether to use the external labels as selectors for the remote read endpoint. It requires Prometheus >= v2.34.0. type: boolean followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.26.0. type: boolean @@ -6378,13 +6404,13 @@ spec: additionalProperties: type: string description: |- - Custom HTTP headers to be sent along with each remote read request. + headers defines the custom HTTP headers to be sent along with each remote read request. Be aware that headers that are set by Prometheus itself can't be overwritten. Only valid in Prometheus versions 2.26.0 and newer. type: object name: description: |- - The name of the remote read queue, it must be unique if specified. The + name of the remote read queue, it must be unique if specified. The name is used in metrics and logging in order to differentiate read configurations. @@ -6392,7 +6418,7 @@ spec: type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6400,7 +6426,7 @@ spec: type: string oauth2: description: |- - OAuth2 configuration for the URL. + oauth2 configuration for the URL. It requires Prometheus >= v2.27.0. @@ -6408,12 +6434,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6436,7 +6462,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6462,7 +6489,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -6490,12 +6517,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6529,7 +6556,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6537,33 +6564,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6586,8 +6612,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -6612,12 +6638,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6640,8 +6666,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -6666,11 +6692,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6695,7 +6722,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6706,7 +6733,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6716,12 +6743,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -6757,7 +6785,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6765,40 +6793,41 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string readRecent: description: |- - Whether reads should be made for queries for time ranges that + readRecent defines whether reads should be made for queries for time ranges that the local storage should have complete data for. type: boolean remoteTimeout: - description: Timeout for requests to the remote read endpoint. + description: remoteTimeout defines the timeout for requests + to the remote read endpoint. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string requiredMatchers: additionalProperties: type: string description: |- - An optional list of equality matchers which have to be present + requiredMatchers defines an optional list of equality matchers which have to be present in a selector to query the remote read endpoint. type: object tlsConfig: - description: TLS Config to use for the URL. + description: tlsConfig to use for the URL. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6821,7 +6850,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6846,15 +6876,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6877,7 +6908,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6902,19 +6934,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6939,7 +6972,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6950,7 +6983,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6960,18 +6993,19 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the endpoint to query from. + description: url defines the URL of the endpoint to query from. type: string required: - url type: object type: array remoteWrite: - description: Defines the list of remote write configurations. + description: remoteWrite defines the list of remote write configurations. items: description: |- RemoteWriteSpec defines the configuration to write samples from Prometheus @@ -6979,15 +7013,15 @@ spec: properties: authorization: description: |- - Authorization section for the URL. + authorization section for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. Cannot be set at the same time as `sigv4`, `basicAuth`, `oauth2`, or `azureAd`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -7011,12 +7045,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -7025,15 +7059,15 @@ spec: type: object azureAd: description: |- - AzureAD for the URL. + azureAd for the URL. It requires Prometheus >= v2.45.0 or Thanos >= v0.31.0. Cannot be set at the same time as `authorization`, `basicAuth`, `oauth2`, or `sigv4`. properties: cloud: - description: The Azure Cloud. Options are 'AzurePublic', - 'AzureChina', or 'AzureGovernment'. + description: cloud defines the Azure Cloud. Options are + 'AzurePublic', 'AzureChina', or 'AzureGovernment'. enum: - AzureChina - AzureGovernment @@ -7041,32 +7075,33 @@ spec: type: string managedIdentity: description: |- - ManagedIdentity defines the Azure User-assigned Managed identity. + managedIdentity defines the Azure User-assigned Managed identity. Cannot be set at the same time as `oauth` or `sdk`. properties: clientId: - description: The client id + description: clientId defines defines the Azure User-assigned + Managed identity. type: string required: - clientId type: object oauth: description: |- - OAuth defines the oauth config that is being used to authenticate. + oauth defines the oauth config that is being used to authenticate. Cannot be set at the same time as `managedIdentity` or `sdk`. It requires Prometheus >= v2.48.0 or Thanos >= v0.31.0. properties: clientId: - description: '`clientID` is the clientId of the Azure + description: clientId defines the clientId of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 type: string clientSecret: - description: '`clientSecret` specifies a key of a Secret + description: clientSecret specifies a key of a Secret containing the client secret of the Azure Active Directory - application that is being used to authenticate.' + application that is being used to authenticate. properties: key: description: The key of the secret to select from. Must @@ -7090,9 +7125,9 @@ spec: type: object x-kubernetes-map-type: atomic tenantId: - description: '`tenantId` is the tenant ID of the Azure + description: tenantId is the tenant ID of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 pattern: ^[0-9a-zA-Z-.]+$ type: string @@ -7103,29 +7138,29 @@ spec: type: object sdk: description: |- - SDK defines the Azure SDK config that is being used to authenticate. + sdk defines the Azure SDK config that is being used to authenticate. See https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication Cannot be set at the same time as `oauth` or `managedIdentity`. It requires Prometheus >= v2.52.0 or Thanos >= v0.36.0. properties: tenantId: - description: '`tenantId` is the tenant ID of the azure + description: tenantId defines the tenant ID of the azure active directory application that is being used to - authenticate.' + authenticate. pattern: ^[0-9a-zA-Z-.]+$ type: string type: object type: object basicAuth: description: |- - BasicAuth configuration for the URL. + basicAuth configuration for the URL. Cannot be set at the same time as `sigv4`, `authorization`, `oauth2`, or `azureAd`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -7151,7 +7186,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -7178,23 +7213,22 @@ spec: type: object bearerToken: description: |- + bearerToken is deprecated: this will be removed in a future release. *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File from which to read bearer token for the URL. + bearerTokenFile defines the file from which to read bearer token for the URL. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. type: boolean @@ -7202,14 +7236,14 @@ spec: additionalProperties: type: string description: |- - Custom HTTP headers to be sent along with each remote write request. + headers defines the custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. It requires Prometheus >= v2.25.0 or Thanos >= v0.24.0. type: object messageVersion: description: |- - The Remote Write message's version to use when writing to the endpoint. + messageVersion defines the Remote Write message's version to use when writing to the endpoint. `Version1.0` corresponds to the `prometheus.WriteRequest` protobuf message introduced in Remote Write 1.0. `Version2.0` corresponds to the `io.prometheus.write.v2.Request` protobuf message introduced in Remote Write 2.0. @@ -7226,37 +7260,37 @@ spec: - V2.0 type: string metadataConfig: - description: MetadataConfig configures the sending of series - metadata to the remote storage. + description: metadataConfig defines how to send a series metadata + to the remote storage. properties: maxSamplesPerSend: description: |- - MaxSamplesPerSend is the maximum number of metadata samples per send. + maxSamplesPerSend defines the maximum number of metadata samples per send. It requires Prometheus >= v2.29.0. format: int32 minimum: -1 type: integer send: - description: Defines whether metric metadata is sent to - the remote storage or not. + description: send defines whether metric metadata is sent + to the remote storage or not. type: boolean sendInterval: - description: Defines how frequently metric metadata is sent - to the remote storage. + description: sendInterval defines how frequently metric + metadata is sent to the remote storage. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object name: description: |- - The name of the remote write queue, it must be unique if specified. The + name of the remote write queue, it must be unique if specified. The name is used in metrics and logging in order to differentiate queues. It requires Prometheus >= v2.15.0 or Thanos >= 0.24.0. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7264,7 +7298,7 @@ spec: type: string oauth2: description: |- - OAuth2 configuration for the URL. + oauth2 configuration for the URL. It requires Prometheus >= v2.27.0 or Thanos >= v0.24.0. @@ -7272,12 +7306,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7300,7 +7334,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7326,7 +7361,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -7354,12 +7389,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7393,7 +7428,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7401,33 +7436,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7450,8 +7484,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7476,12 +7510,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7504,8 +7538,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7530,11 +7564,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7559,7 +7594,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7570,7 +7605,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7580,12 +7615,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -7621,7 +7657,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7629,89 +7665,88 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string queueConfig: - description: QueueConfig allows tuning of the remote write queue + description: queueConfig allows tuning of the remote write queue parameters. properties: batchSendDeadline: - description: BatchSendDeadline is the maximum time a sample - will wait in buffer. + description: batchSendDeadline defines the maximum time + a sample will wait in buffer. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string capacity: description: |- - Capacity is the number of samples to buffer per shard before we start + capacity defines the number of samples to buffer per shard before we start dropping them. type: integer maxBackoff: - description: MaxBackoff is the maximum retry delay. + description: maxBackoff defines the maximum retry delay. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string maxRetries: - description: MaxRetries is the maximum number of times to - retry a batch on recoverable errors. + description: maxRetries defines the maximum number of times + to retry a batch on recoverable errors. type: integer maxSamplesPerSend: - description: MaxSamplesPerSend is the maximum number of - samples per send. + description: maxSamplesPerSend defines the maximum number + of samples per send. type: integer maxShards: - description: MaxShards is the maximum number of shards, + description: maxShards defines the maximum number of shards, i.e. amount of concurrency. type: integer minBackoff: - description: MinBackoff is the initial retry delay. Gets - doubled for every retry. + description: minBackoff defines the initial retry delay. + Gets doubled for every retry. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string minShards: - description: MinShards is the minimum number of shards, + description: minShards defines the minimum number of shards, i.e. amount of concurrency. type: integer retryOnRateLimit: description: |- - Retry upon receiving a 429 status code from the remote-write storage. + retryOnRateLimit defines the retry upon receiving a 429 status code from the remote-write storage. This is an *experimental feature*, it may change in any upcoming release in a breaking way. type: boolean sampleAgeLimit: description: |- - SampleAgeLimit drops samples older than the limit. + sampleAgeLimit drops samples older than the limit. It requires Prometheus >= v2.50.0 or Thanos >= v0.32.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object remoteTimeout: - description: Timeout for requests to the remote write endpoint. + description: remoteTimeout defines the timeout for requests + to the remote write endpoint. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string roundRobinDNS: - description: |- - When enabled: - - The remote-write mechanism will resolve the hostname via DNS. - - It will randomly select one of the resolved IP addresses and connect to it. - - When disabled (default behavior): - - The Go standard library will handle hostname resolution. - - It will attempt connections to each resolved IP address sequentially. - - Note: The connection timeout applies to the entire resolution and connection process. - If disabled, the timeout is distributed across all connection attempts. - - It requires Prometheus >= v3.1.0 or Thanos >= v0.38.0. + description: "roundRobinDNS controls the DNS resolution behavior + for remote-write connections.\nWhen enabled:\n - The remote-write + mechanism will resolve the hostname via DNS.\n - It will + randomly select one of the resolved IP addresses and connect + to it.\n\nWhen disabled (default behavior):\n - The Go standard + library will handle hostname resolution.\n - It will attempt + connections to each resolved IP address sequentially.\n\nNote: + The connection timeout applies to the entire resolution and + connection process.\n\n\tIf disabled, the timeout is distributed + across all connection attempts.\n\nIt requires Prometheus + >= v3.1.0 or Thanos >= v0.38.0." type: boolean sendExemplars: description: |- - Enables sending of exemplars over remote write. Note that + sendExemplars enables sending of exemplars over remote write. Note that exemplar-storage itself must be enabled using the `spec.enableFeatures` option for exemplars to be scraped in the first place. @@ -7719,14 +7754,14 @@ spec: type: boolean sendNativeHistograms: description: |- - Enables sending of native histograms, also known as sparse histograms + sendNativeHistograms enables sending of native histograms, also known as sparse histograms over remote write. It requires Prometheus >= v2.40.0 or Thanos >= v0.30.0. type: boolean sigv4: description: |- - Sigv4 allows to configures AWS's Signature Verification 4 for the URL. + sigv4 defines the AWS's Signature Verification 4 for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. @@ -7734,7 +7769,7 @@ spec: properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -7759,18 +7794,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used to authenticate. + description: profile defines the named AWS profile used + to authenticate. type: string region: - description: Region is the AWS region. If blank, the region - from the default credentials chain used. + description: region defines the AWS region. If blank, the + region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used to authenticate. + description: roleArn defines the named AWS profile used + to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -7794,17 +7831,22 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object tlsConfig: - description: TLS Config to use for the URL. + description: tlsConfig to use for the URL. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7827,7 +7869,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7852,15 +7895,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7883,7 +7927,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7908,19 +7953,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7945,7 +7991,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7956,7 +8002,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7966,15 +8012,18 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the endpoint to send samples to. + description: url defines the URL of the endpoint to send samples + to. minLength: 1 type: string writeRelabelConfigs: - description: The list of remote write relabel configurations. + description: writeRelabelConfigs defines the list of remote + write relabel configurations. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -7985,7 +8034,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -8017,41 +8066,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -8066,7 +8115,7 @@ spec: type: array remoteWriteReceiverMessageVersions: description: |- - List of the protobuf message versions to accept when receiving the + remoteWriteReceiverMessageVersions list of the protobuf message versions to accept when receiving the remote writes. It requires Prometheus >= v2.54.0. @@ -8080,7 +8129,7 @@ spec: x-kubernetes-list-type: set replicaExternalLabelName: description: |- - Name of Prometheus external label used to denote the replica name. + replicaExternalLabelName defines the name of Prometheus external label used to denote the replica name. The external label will _not_ be added when the field is set to the empty string (`""`). @@ -8088,7 +8137,7 @@ spec: type: string replicas: description: |- - Number of replicas of each shard to deploy for a Prometheus deployment. + replicas defines the number of replicas of each shard to deploy for a Prometheus deployment. `spec.replicas` multiplied by `spec.shards` is the total number of Pods created. @@ -8096,8 +8145,8 @@ spec: format: int32 type: integer resources: - description: Defines the resources requests and limits of the 'prometheus' - container. + description: resources defines the resources requests and limits of + the 'prometheus' container. properties: claims: description: |- @@ -8157,18 +8206,19 @@ spec: type: object retention: description: |- - How long to retain the Prometheus data. + retention defines how long to retain the Prometheus data. Default: "24h" if `spec.retention` and `spec.retentionSize` are empty. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string retentionSize: - description: Maximum number of bytes used by the Prometheus data. + description: retentionSize defines the maximum number of bytes used + by the Prometheus data. pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ type: string routePrefix: description: |- - The route prefix Prometheus registers HTTP handlers for. + routePrefix defines the route prefix Prometheus registers HTTP handlers for. This is useful when using `spec.externalURL`, and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but @@ -8177,7 +8227,7 @@ spec: type: string ruleNamespaceSelector: description: |- - Namespaces to match for PrometheusRule discovery. An empty label selector + ruleNamespaceSelector defines the namespaces to match for PrometheusRule discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. properties: @@ -8226,13 +8276,13 @@ spec: x-kubernetes-map-type: atomic ruleQueryOffset: description: |- - Defines the offset the rule evaluation timestamp of this particular group by the specified duration into the past. + ruleQueryOffset defines the offset the rule evaluation timestamp of this particular group by the specified duration into the past. It requires Prometheus >= v2.53.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string ruleSelector: description: |- - PrometheusRule objects to be selected for rule evaluation. An empty + ruleSelector defines the prometheusRule objects to be selected for rule evaluation. An empty label selector matches all objects. A null label selector matches no objects. properties: @@ -8280,40 +8330,41 @@ spec: type: object x-kubernetes-map-type: atomic rules: - description: Defines the configuration of the Prometheus rules' engine. + description: rules defines the configuration of the Prometheus rules' + engine. properties: alert: description: |- - Defines the parameters of the Prometheus rules' engine. + alert defines the parameters of the Prometheus rules' engine. Any update to these parameters trigger a restart of the pods. properties: forGracePeriod: description: |- - Minimum duration between alert and restored 'for' state. + forGracePeriod defines the minimum duration between alert and restored 'for' state. This is maintained only for alerts with a configured 'for' time greater than the grace period. type: string forOutageTolerance: description: |- - Max time to tolerate prometheus outage for restoring 'for' state of + forOutageTolerance defines the max time to tolerate prometheus outage for restoring 'for' state of alert. type: string resendDelay: description: |- - Minimum amount of time to wait before resending an alert to + resendDelay defines the minimum amount of time to wait before resending an alert to Alertmanager. type: string type: object type: object runtime: - description: RuntimeConfig configures the values for the Prometheus - process behavior + description: runtime defines the values for the Prometheus process + behavior properties: goGC: description: |- - The Go garbage collection target percentage. Lowering this number may increase the CPU usage. + goGC defines the Go garbage collection target percentage. Lowering this number may increase the CPU usage. See: https://tip.golang.org/doc/gc-guide#GOGC format: int32 minimum: -1 @@ -8321,7 +8372,7 @@ spec: type: object sampleLimit: description: |- - SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit defines per-scrape limit on number of scraped samples that will be accepted. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -8330,7 +8381,7 @@ spec: type: integer scrapeClasses: description: |- - List of scrape classes to expose to scraping objects such as + scrapeClasses defines the list of scrape classes to expose to scraping objects such as PodMonitors, ServiceMonitors, Probes and ScrapeConfigs. This is an *experimental feature*, it may change in any upcoming release @@ -8339,13 +8390,13 @@ spec: properties: attachMetadata: description: |- - AttachMetadata configures additional metadata to the discovered targets. + attachMetadata defines additional metadata to the discovered targets. When the scrape object defines its own configuration, it takes precedence over the scrape class configuration. properties: node: description: |- - When set to true, Prometheus attaches node metadata to the discovered + node when set to true, Prometheus attaches node metadata to the discovered targets. The Prometheus service account must have the `list` and `watch` @@ -8354,12 +8405,12 @@ spec: type: object authorization: description: |- - Authorization section for the ScrapeClass. + authorization section for the ScrapeClass. It will only apply if the scrape resource doesn't specify any Authorization. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -8383,12 +8434,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -8397,14 +8448,14 @@ spec: type: object default: description: |- - Default indicates that the scrape applies to all scrape objects that + default defines that the scrape applies to all scrape objects that don't configure an explicit scrape class name. Only one scrape class can be set as the default. type: boolean fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It will only apply if the scrape resource doesn't specify any FallbackScrapeProtocol It requires Prometheus >= v3.0.0. @@ -8417,7 +8468,7 @@ spec: type: string metricRelabelings: description: |- - MetricRelabelings configures the relabeling rules to apply to all samples before ingestion. + metricRelabelings defines the relabeling rules to apply to all samples before ingestion. The Operator adds the scrape class metric relabelings defined here. Then the Operator adds the target-specific metric relabelings defined in ServiceMonitors, PodMonitors, Probes and ScrapeConfigs. @@ -8434,7 +8485,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -8466,41 +8517,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -8510,12 +8561,12 @@ spec: type: object type: array name: - description: Name of the scrape class. + description: name of the scrape class. minLength: 1 type: string relabelings: description: |- - Relabelings configures the relabeling rules to apply to all scrape targets. + relabelings defines the relabeling rules to apply to all scrape targets. The Operator automatically adds relabelings for a few standard Kubernetes fields like `__meta_kubernetes_namespace` and `__meta_kubernetes_service_name`. @@ -8533,7 +8584,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -8565,41 +8616,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -8610,19 +8661,19 @@ spec: type: array tlsConfig: description: |- - TLSConfig defines the TLS settings to use for the scrape. When the + tlsConfig defines the TLS settings to use for the scrape. When the scrape objects define their own CA, certificate and/or key, they take precedence over the corresponding scrape class fields. For now only the `caFile`, `certFile` and `keyFile` fields are supported. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8645,7 +8696,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8670,15 +8722,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8701,7 +8754,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8726,19 +8780,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8763,7 +8818,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8774,7 +8829,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8784,7 +8839,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -8796,7 +8852,7 @@ spec: x-kubernetes-list-type: map scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. @@ -8804,7 +8860,7 @@ spec: type: boolean scrapeConfigNamespaceSelector: description: |- - Namespaces to match for ScrapeConfig discovery. An empty label selector + scrapeConfigNamespaceSelector defines the namespaces to match for ScrapeConfig discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. @@ -8855,7 +8911,7 @@ spec: x-kubernetes-map-type: atomic scrapeConfigSelector: description: |- - ScrapeConfigs to be selected for target discovery. An empty label + scrapeConfigSelector defines the scrapeConfigs to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -8914,7 +8970,7 @@ spec: x-kubernetes-map-type: atomic scrapeFailureLogFile: description: |- - File to which scrape failures are logged. + scrapeFailureLogFile defines the file to which scrape failures are logged. Reloading the configuration will reopen the file. If the filename has an empty path, e.g. 'file.log', The Prometheus Pods @@ -8927,14 +8983,14 @@ spec: scrapeInterval: default: 30s description: |- - Interval between consecutive scrapes. + scrapeInterval defines interval between consecutive scrapes. Default: "30s" pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string scrapeProtocols: description: |- - The protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -8962,13 +9018,13 @@ spec: x-kubernetes-list-type: set scrapeTimeout: description: |- - Number of seconds to wait until a scrape request times out. + scrapeTimeout defines the number of seconds to wait until a scrape request times out. The value cannot be greater than the scrape interval otherwise the operator will reject the resource. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string secrets: description: |- - Secrets is a list of Secrets in the same namespace as the Prometheus + secrets defines a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. Each Secret is added to the StatefulSet definition as a volume named `secret-`. The Secrets are mounted into /etc/prometheus/secrets/ in the 'prometheus' container. @@ -8978,7 +9034,7 @@ spec: x-kubernetes-list-type: set securityContext: description: |- - SecurityContext holds pod-level security attributes and common container settings. + securityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: appArmorProfile: @@ -9212,12 +9268,12 @@ spec: type: object serviceAccountName: description: |- - ServiceAccountName is the name of the ServiceAccount to use to run the + serviceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods. type: string serviceDiscoveryRole: description: |- - Defines the service discovery role used to discover targets from + serviceDiscoveryRole defines the service discovery role used to discover targets from `ServiceMonitor` objects and Alertmanager endpoints. If set, the value should be either "Endpoints" or "EndpointSlice". @@ -9228,7 +9284,7 @@ spec: type: string serviceMonitorNamespaceSelector: description: |- - Namespaces to match for ServicedMonitors discovery. An empty label selector + serviceMonitorNamespaceSelector defines the namespaces to match for ServicedMonitors discovery. An empty label selector matches all namespaces. A null label selector (default value) matches the current namespace only. properties: @@ -9277,7 +9333,7 @@ spec: x-kubernetes-map-type: atomic serviceMonitorSelector: description: |- - ServiceMonitors to be selected for target discovery. An empty label + serviceMonitorSelector defines the serviceMonitors to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -9334,7 +9390,7 @@ spec: x-kubernetes-map-type: atomic serviceName: description: |- - The name of the service name used by the underlying StatefulSet(s) as the governing service. + serviceName defines the name of the service name used by the underlying StatefulSet(s) as the governing service. If defined, the Service must be created before the Prometheus/PrometheusAgent resource in the same namespace and it must define a selector that matches the pod labels. If empty, the operator will create and manage a headless service named `prometheus-operated` for Prometheus resources, or `prometheus-agent-operated` for PrometheusAgent resources. @@ -9343,12 +9399,12 @@ spec: minLength: 1 type: string sha: - description: 'Deprecated: use ''spec.image'' instead. The image''s + description: 'sha is deprecated: use ''spec.image'' instead. The image''s digest can be specified as part of the image name.' type: string shardRetentionPolicy: description: |- - ShardRetentionPolicy defines the retention policy for the Prometheus shards. + shardRetentionPolicy defines the retention policy for the Prometheus shards. (Alpha) Using this field requires the 'PrometheusShardRetentionPolicy' feature gate to be enabled. The final goals for this feature can be seen at https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/proposals/202310-shard-autoscaling.md#graceful-scale-down-of-prometheus-servers, @@ -9357,14 +9413,12 @@ spec: properties: retain: description: |- - Defines the config for retention when the retention policy is set to `Retain`. + retain defines the config for retention when the retention policy is set to `Retain`. This field is ineffective as of now. properties: retentionPeriod: - description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` + description: retentionPeriod defines the retentionPeriod for + shard retention policy. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string required: @@ -9372,7 +9426,7 @@ spec: type: object whenScaled: description: |- - Defines the retention policy when the Prometheus shards are scaled down. + whenScaled defines the retention policy when the Prometheus shards are scaled down. * `Delete`, the operator will delete the pods from the scaled-down shard(s). * `Retain`, the operator will keep the pods from the scaled-down shard(s), so the data can still be queried. @@ -9384,7 +9438,7 @@ spec: type: object shards: description: |- - Number of shards to distribute the scraped targets onto. + shards defines the number of shards to distribute the scraped targets onto. `spec.replicas` multiplied by `spec.shards` is the total number of Pods being created. @@ -9413,15 +9467,15 @@ spec: format: int32 type: integer storage: - description: Storage defines the storage used by Prometheus. + description: storage defines the storage used by Prometheus. properties: disableMountSubPath: - description: 'Deprecated: subPath usage will be removed in a future - release.' + description: 'disableMountSubPath deprecated: subPath usage will + be removed in a future release.' type: boolean emptyDir: description: |- - EmptyDirVolumeSource to be used by the StatefulSet. + emptyDir to be used by the StatefulSet. If specified, it takes precedence over `ephemeral` and `volumeClaimTemplate`. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir properties: @@ -9448,7 +9502,7 @@ spec: type: object ephemeral: description: |- - EphemeralVolumeSource to be used by the StatefulSet. + ephemeral to be used by the StatefulSet. This is a beta field in k8s 1.21 and GA in 1.15. For lower versions, starting with k8s 1.19, it requires enabling the GenericEphemeralVolume feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes @@ -9693,7 +9747,7 @@ spec: type: object volumeClaimTemplate: description: |- - Defines the PVC spec to be used by the Prometheus StatefulSets. + volumeClaimTemplate defines the PVC spec to be used by the Prometheus StatefulSets. The easiest way to use a volume that cannot be automatically provisioned is to use a label selector alongside manually created PersistentVolumes. properties: @@ -9713,14 +9767,14 @@ spec: More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: - description: EmbeddedMetadata contains metadata relevant to - an EmbeddedResource. + description: metadata defines EmbeddedMetadata contains metadata + relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -9729,14 +9783,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -9746,7 +9800,7 @@ spec: type: object spec: description: |- - Defines the desired characteristics of a volume requested by a pod author. + spec defines the specification of the characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: accessModes: @@ -9942,7 +9996,7 @@ spec: type: string type: object status: - description: 'Deprecated: this field is never set.' + description: 'status is deprecated: this field is never set.' properties: accessModes: description: |- @@ -10126,12 +10180,12 @@ spec: type: object type: object tag: - description: 'Deprecated: use ''spec.image'' instead. The image''s + description: 'tag is deprecated: use ''spec.image'' instead. The image''s tag can be specified as part of the image name.' type: string targetLimit: description: |- - TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit defines a limit on the number of scraped targets that will be accepted. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -10140,7 +10194,7 @@ spec: type: integer terminationGracePeriodSeconds: description: |- - Optional duration in seconds the pod needs to terminate gracefully. + terminationGracePeriodSeconds defines the optional duration in seconds the pod needs to terminate gracefully. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down) which may lead to data corruption. @@ -10149,11 +10203,12 @@ spec: minimum: 0 type: integer thanos: - description: Defines the configuration of the optional Thanos sidecar. + description: thanos defines the configuration of the optional Thanos + sidecar. properties: additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the Thanos container. + additionalArgs allows setting additional arguments for the Thanos container. The arguments are passed as-is to the Thanos container which may cause issues if they are invalid or not supported the given Thanos version. In case of an argument conflict (e.g. an argument which is already set by the @@ -10163,24 +10218,24 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for - name-only arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. + Can be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name type: object type: array baseImage: - description: 'Deprecated: use ''image'' instead.' + description: 'baseImage is deprecated: use ''image'' instead.' type: string blockSize: default: 2h description: |- - BlockDuration controls the size of TSDB blocks produced by Prometheus. + blockSize controls the size of TSDB blocks produced by Prometheus. The default value is 2h to match the upstream Prometheus defaults. WARNING: Changing the block duration can impact the performance and @@ -10191,34 +10246,35 @@ spec: pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string getConfigInterval: - description: How often to retrieve the Prometheus configuration. + description: getConfigInterval defines how often to retrieve the + Prometheus configuration. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string getConfigTimeout: - description: Maximum time to wait when retrieving the Prometheus - configuration. + description: getConfigTimeout defines the maximum time to wait + when retrieving the Prometheus configuration. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string grpcListenLocal: description: |- - When true, the Thanos sidecar listens on the loopback interface instead + grpcListenLocal defines when true, the Thanos sidecar listens on the loopback interface instead of the Pod IP's address for the gRPC endpoints. It has no effect if `listenLocal` is true. type: boolean grpcServerTlsConfig: description: |- - Configures the TLS parameters for the gRPC server providing the StoreAPI. + grpcServerTlsConfig defines the TLS parameters for the gRPC server providing the StoreAPI. Note: Currently only the `caFile`, `certFile`, and `keyFile` fields are supported. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10241,7 +10297,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10266,15 +10323,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10297,7 +10355,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10322,19 +10381,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -10359,7 +10419,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10370,7 +10430,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10380,19 +10440,20 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object httpListenLocal: description: |- - When true, the Thanos sidecar listens on the loopback interface instead + httpListenLocal when true, the Thanos sidecar listens on the loopback interface instead of the Pod IP's address for the HTTP endpoints. It has no effect if `listenLocal` is true. type: boolean image: description: |- - Container image name for Thanos. If specified, it takes precedence over + image defines the container image name for Thanos. If specified, it takes precedence over the `spec.thanos.baseImage`, `spec.thanos.tag` and `spec.thanos.sha` fields. @@ -10404,18 +10465,18 @@ spec: the time when the operator was released. type: string listenLocal: - description: 'Deprecated: use `grpcListenLocal` and `httpListenLocal` - instead.' + description: 'listenLocal is deprecated: use `grpcListenLocal` + and `httpListenLocal` instead.' type: boolean logFormat: - description: Log format for the Thanos sidecar. + description: logFormat for the Thanos sidecar. enum: - "" - logfmt - json type: string logLevel: - description: Log level for the Thanos sidecar. + description: logLevel for the Thanos sidecar. enum: - "" - debug @@ -10425,14 +10486,14 @@ spec: type: string minTime: description: |- - Defines the start of time range limit served by the Thanos sidecar's StoreAPI. + minTime defines the start of time range limit served by the Thanos sidecar's StoreAPI. The field's value should be a constant time in RFC3339 format or a time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y. type: string objectStorageConfig: description: |- - Defines the Thanos sidecar's configuration to upload TSDB blocks to object storage. + objectStorageConfig defines the Thanos sidecar's configuration to upload TSDB blocks to object storage. More info: https://thanos.io/tip/thanos/storage.md/ @@ -10461,7 +10522,7 @@ spec: x-kubernetes-map-type: atomic objectStorageConfigFile: description: |- - Defines the Thanos sidecar's configuration file to upload TSDB blocks to object storage. + objectStorageConfigFile defines the Thanos sidecar's configuration file to upload TSDB blocks to object storage. More info: https://thanos.io/tip/thanos/storage.md/ @@ -10469,13 +10530,13 @@ spec: type: string readyTimeout: description: |- - ReadyTimeout is the maximum time that the Thanos sidecar will wait for + readyTimeout defines the maximum time that the Thanos sidecar will wait for Prometheus to start. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string resources: - description: Defines the resources requests and limits of the - Thanos sidecar. + description: resources defines the resources requests and limits + of the Thanos sidecar. properties: claims: description: |- @@ -10534,16 +10595,16 @@ spec: type: object type: object sha: - description: 'Deprecated: use ''image'' instead. The image digest - can be specified as part of the image name.' + description: 'sha is deprecated: use ''image'' instead. The image + digest can be specified as part of the image name.' type: string tag: - description: 'Deprecated: use ''image'' instead. The image''s + description: 'tag is deprecated: use ''image'' instead. The image''s tag can be specified as as part of the image name.' type: string tracingConfig: description: |- - Defines the tracing configuration for the Thanos sidecar. + tracingConfig defines the tracing configuration for the Thanos sidecar. `tracingConfigFile` takes precedence over this field. @@ -10575,7 +10636,7 @@ spec: x-kubernetes-map-type: atomic tracingConfigFile: description: |- - Defines the tracing configuration file for the Thanos sidecar. + tracingConfigFile defines the tracing configuration file for the Thanos sidecar. This field takes precedence over `tracingConfig`. @@ -10586,7 +10647,7 @@ spec: type: string version: description: |- - Version of Thanos being deployed. The operator uses this information + version of Thanos being deployed. The operator uses this information to generate the Prometheus StatefulSet + configuration files. If not specified, the operator assumes the latest upstream release of @@ -10595,7 +10656,7 @@ spec: type: string volumeMounts: description: |- - VolumeMounts allows configuration of additional VolumeMounts for Thanos. + volumeMounts allows configuration of additional VolumeMounts for Thanos. VolumeMounts specified will be appended to other VolumeMounts in the 'thanos-sidecar' container. items: @@ -10662,7 +10723,7 @@ spec: type: array type: object tolerations: - description: Defines the Pods' tolerations if specified. + description: tolerations defines the Pods' tolerations if specified. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -10701,12 +10762,14 @@ spec: type: object type: array topologySpreadConstraints: - description: Defines the pod's topology spread constraints if specified. + description: topologySpreadConstraints defines the pod's topology + spread constraints if specified. items: properties: additionalLabelSelectors: - description: Defines what Prometheus Operator managed labels - should be added to labelSelector on the topologySpreadConstraint. + description: additionalLabelSelectors Defines what Prometheus + Operator managed labels should be added to labelSelector on + the topologySpreadConstraint. enum: - OnResource - OnShard @@ -10881,61 +10944,62 @@ spec: type: array tracingConfig: description: |- - TracingConfig configures tracing in Prometheus. + tracingConfig defines tracing in Prometheus. This is an *experimental feature*, it may change in any upcoming release in a breaking way. properties: clientType: - description: Client used to export the traces. Supported values - are `http` or `grpc`. + description: clientType defines the client used to export the + traces. Supported values are `http` or `grpc`. enum: - http - grpc type: string compression: - description: Compression key for supported compression types. + description: compression key for supported compression types. The only supported value is `gzip`. enum: - gzip type: string endpoint: - description: Endpoint to send the traces to. Should be provided + description: endpoint to send the traces to. Should be provided in format :. minLength: 1 type: string headers: additionalProperties: type: string - description: Key-value pairs to be used as headers associated - with gRPC or HTTP requests. + description: headers defines the key-value pairs to be used as + headers associated with gRPC or HTTP requests. type: object insecure: - description: If disabled, the client will use a secure connection. + description: insecure if disabled, the client will use a secure + connection. type: boolean samplingFraction: anyOf: - type: integer - type: string - description: Sets the probability a given trace will be sampled. - Must be a float from 0 through 1. + description: samplingFraction defines the probability a given + trace will be sampled. Must be a float from 0 through 1. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true timeout: - description: Maximum time the exporter will wait for each batch - export. + description: timeout defines the maximum time the exporter will + wait for each batch export. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS Config to use when sending traces. + description: tlsConfig to use when sending traces. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10958,7 +11022,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10983,15 +11048,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11014,7 +11080,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11039,19 +11106,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -11076,7 +11144,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11087,7 +11155,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11097,7 +11165,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -11105,12 +11174,12 @@ spec: type: object tsdb: description: |- - Defines the runtime reloadable configuration of the timeseries database(TSDB). + tsdb defines the runtime reloadable configuration of the timeseries database(TSDB). It requires Prometheus >= v2.39.0 or PrometheusAgent >= v2.54.0. properties: outOfOrderTimeWindow: description: |- - Configures how old an out-of-order/out-of-bounds sample can be with + outOfOrderTimeWindow defines how old an out-of-order/out-of-bounds sample can be with respect to the TSDB max time. An out-of-order/out-of-bounds sample is ingested into the TSDB as long as @@ -11125,7 +11194,7 @@ spec: type: object version: description: |- - Version of Prometheus being deployed. The operator uses this information + version of Prometheus being deployed. The operator uses this information to generate the Prometheus StatefulSet + configuration files. If not specified, the operator assumes the latest upstream version of @@ -11134,7 +11203,7 @@ spec: type: string volumeMounts: description: |- - VolumeMounts allows the configuration of additional VolumeMounts. + volumeMounts allows the configuration of additional VolumeMounts. VolumeMounts will be appended to other VolumeMounts in the 'prometheus' container, that are generated as a result of StorageSpec objects. @@ -11202,7 +11271,7 @@ spec: type: array volumes: description: |- - Volumes allows the configuration of additional volumes on the output + volumes allows the configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: @@ -13089,29 +13158,30 @@ spec: type: array walCompression: description: |- - Configures compression of the write-ahead log (WAL) using Snappy. + walCompression defines the compression of the write-ahead log (WAL) using Snappy. WAL compression is enabled by default for Prometheus >= 2.20.0 Requires Prometheus v2.11.0 and above. type: boolean web: - description: Defines the configuration of the Prometheus web server. + description: web defines the configuration of the Prometheus web server. properties: httpConfig: - description: Defines HTTP parameters for web server. + description: httpConfig defines HTTP parameters for web server. properties: headers: - description: List of headers that can be added to HTTP responses. + description: headers defines a list of headers that can be + added to HTTP responses. properties: contentSecurityPolicy: description: |- - Set the Content-Security-Policy header to HTTP responses. + contentSecurityPolicy defines the Content-Security-Policy header to HTTP responses. Unset if blank. type: string strictTransportSecurity: description: |- - Set the Strict-Transport-Security header to HTTP responses. + strictTransportSecurity defines the Strict-Transport-Security header to HTTP responses. Unset if blank. Please make sure that you use this with care as this header might force browsers to load Prometheus and the other applications hosted on the same @@ -13120,7 +13190,7 @@ spec: type: string xContentTypeOptions: description: |- - Set the X-Content-Type-Options header to HTTP responses. + xContentTypeOptions defines the X-Content-Type-Options header to HTTP responses. Unset if blank. Accepted value is nosniff. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options enum: @@ -13129,7 +13199,7 @@ spec: type: string xFrameOptions: description: |- - Set the X-Frame-Options header to HTTP responses. + xFrameOptions defines the X-Frame-Options header to HTTP responses. Unset if blank. Accepted values are deny and sameorigin. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options enum: @@ -13139,42 +13209,42 @@ spec: type: string xXSSProtection: description: |- - Set the X-XSS-Protection header to all responses. + xXSSProtection defines the X-XSS-Protection header to all responses. Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection type: string type: object http2: description: |- - Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. + http2 enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. When TLSConfig is not configured, HTTP/2 will be disabled. Whenever the value of the field changes, a rolling update will be triggered. type: boolean type: object maxConnections: description: |- - Defines the maximum number of simultaneous connections + maxConnections defines the maximum number of simultaneous connections A zero value means that Prometheus doesn't accept any incoming connection. format: int32 minimum: 0 type: integer pageTitle: - description: The prometheus web page title. + description: pageTitle defines the prometheus web page title. type: string tlsConfig: - description: Defines the TLS parameters for HTTPS. + description: tlsConfig defines the TLS parameters for HTTPS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -13197,7 +13267,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -13223,7 +13294,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -13231,7 +13302,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -13241,14 +13312,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -13271,7 +13342,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -13297,21 +13369,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -13321,7 +13393,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -13329,7 +13401,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -13357,14 +13429,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -13375,18 +13449,19 @@ spec: type: object status: description: |- - Most recent observed status of the Prometheus cluster. Read-only. + status defines the most recent observed status of the Prometheus cluster. Read-only. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment. format: int32 type: integer conditions: - description: The current state of the Prometheus deployment. + description: conditions defines the current state of the Prometheus + deployment. items: description: |- Condition represents the state of the resources associated with the @@ -13398,12 +13473,12 @@ spec: format: date-time type: string message: - description: Human-readable message indicating details for the - condition's last transition. + description: message defines human-readable message indicating + details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the @@ -13411,14 +13486,14 @@ spec: format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: - description: Type of the condition being reported. + description: type of the condition being reported. minLength: 1 type: string required: @@ -13432,45 +13507,46 @@ spec: x-kubernetes-list-type: map paused: description: |- - Represents whether any actions on the underlying managed objects are + paused defines whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: |- - Total number of non-terminated pods targeted by this Prometheus deployment + replicas defines the total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector). format: int32 type: integer selector: - description: The selector used to match the pods targeted by this - Prometheus resource. + description: selector used to match the pods targeted by this Prometheus + resource. type: string shardStatuses: - description: The list has one entry per shard. Each entry provides - a summary of the shard status. + description: shardStatuses defines the list has one entry per shard. + Each entry provides a summary of the shard status. items: properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this shard. format: int32 type: integer replicas: - description: Total number of pods targeted by this shard. + description: replicas defines the total number of pods targeted + by this shard. format: int32 type: integer shardID: - description: Identifier of the shard. + description: shardID defines the identifier of the shard. type: string unavailableReplicas: - description: Total number of unavailable pods targeted by this - shard. + description: unavailableReplicas defines the Total number of + unavailable pods targeted by this shard. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this shard + updatedReplicas defines the total number of non-terminated pods targeted by this shard that have the desired spec. format: int32 type: integer @@ -13486,26 +13562,20 @@ spec: - shardID x-kubernetes-list-type: map shards: - description: Shards is the most recently observed number of shards. + description: shards defines the most recently observed number of shards. format: int32 type: integer unavailableReplicas: - description: Total number of unavailable pods targeted by this Prometheus - deployment. + description: unavailableReplicas defines the total number of unavailable + pods targeted by this Prometheus deployment. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this Prometheus deployment + updatedReplicas defines the total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec. format: int32 type: integer - required: - - availableReplicas - - paused - - replicas - - unavailableReplicas - - updatedReplicas type: object required: - spec diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_prometheusrules.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_prometheusrules.yaml index f4800ee9ed9..87285176ca8 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_prometheusrules.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_prometheusrules.yaml @@ -44,24 +44,25 @@ spec: metadata: type: object spec: - description: Specification of desired alerting rule definitions for Prometheus. + description: spec defines the specification of desired alerting rule definitions + for Prometheus. properties: groups: - description: Content of Prometheus rule file + description: groups defines the content of Prometheus rule file items: description: RuleGroup is a list of sequentially evaluated recording and alerting rules. properties: interval: - description: Interval determines how often rules in the group - are evaluated. + description: interval defines how often rules in the group are + evaluated. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string labels: additionalProperties: type: string description: |- - Labels to add or overwrite before storing the result for its rules. + labels define the labels to add or overwrite before storing the result for its rules. The labels defined at the rule level take precedence. It requires Prometheus >= 3.0.0. @@ -69,31 +70,32 @@ spec: type: object limit: description: |- - Limit the number of alerts an alerting rule and series a recording + limit defines the number of alerts an alerting rule and series a recording rule can produce. Limit is supported starting with Prometheus >= 2.31 and Thanos Ruler >= 0.24. type: integer name: - description: Name of the rule group. + description: name defines the name of the rule group. minLength: 1 type: string partial_response_strategy: description: |- - PartialResponseStrategy is only used by ThanosRuler and will + partial_response_strategy is only used by ThanosRuler and will be ignored by Prometheus instances. More info: https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response pattern: ^(?i)(abort|warn)?$ type: string query_offset: description: |- - Defines the offset the rule evaluation timestamp of this particular group by the specified duration into the past. + query_offset defines the offset the rule evaluation timestamp of this particular group by the specified duration into the past. It requires Prometheus >= v2.53.0. It is not supported for ThanosRuler. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string rules: - description: List of alerting and recording rules. + description: rules defines the list of alerting and recording + rules. items: description: |- Rule describes an alerting or recording rule @@ -101,42 +103,42 @@ spec: properties: alert: description: |- - Name of the alert. Must be a valid label value. + alert defines the name of the alert. Must be a valid label value. Only one of `record` and `alert` must be set. type: string annotations: additionalProperties: type: string description: |- - Annotations to add to each alert. + annotations defines annotations to add to each alert. Only valid for alerting rules. type: object expr: anyOf: - type: integer - type: string - description: PromQL expression to evaluate. + description: expr defines the PromQL expression to evaluate. x-kubernetes-int-or-string: true for: - description: Alerts are considered firing once they have - been returned for this long. + description: for defines how alerts are considered firing + once they have been returned for this long. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string keep_firing_for: - description: KeepFiringFor defines how long an alert will - continue firing after the condition that triggered it - has cleared. + description: keep_firing_for defines how long an alert + will continue firing after the condition that triggered + it has cleared. minLength: 1 pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string labels: additionalProperties: type: string - description: Labels to add or overwrite. + description: labels defines labels to add or overwrite. type: object record: description: |- - Name of the time series to output to. Must be a valid metric name. + record defines the name of the time series to output to. Must be a valid metric name. Only one of `record` and `alert` must be set. type: string required: @@ -151,8 +153,113 @@ spec: - name x-kubernetes-list-type: map type: object + status: + description: |- + status defines the status subresource. It is under active development and is updated only when the + "StatusForConfigurationResources" feature gate is enabled. + + Most recent observed status of the PrometheusRule. Read-only. + More info: + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + bindings: + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. + items: + description: WorkloadBinding is a link between a configuration resource + and a workload resource. + properties: + conditions: + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. + items: + description: ConfigResourceCondition describes the status + of configuration resources linked to Prometheus, PrometheusAgent, + Alertmanager or ThanosRuler. + properties: + lastTransitionTime: + description: lastTransitionTime defines the time of the + last update to the current status property. + format: date-time + type: string + message: + description: message defines the human-readable message + indicating details for the condition's last transition. + type: string + observedGeneration: + description: |- + observedGeneration defines the .metadata.generation that the + condition was set based upon. For instance, if `.metadata.generation` is + currently 12, but the `.status.conditions[].observedGeneration` is 9, the + condition is out of date with respect to the current state of the object. + format: int64 + type: integer + reason: + description: reason for the condition's last transition. + type: string + status: + description: status of the condition. + minLength: 1 + type: string + type: + description: |- + type of the condition being reported. + Currently, only "Accepted" is supported. + enum: + - Accepted + minLength: 1 + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + group: + description: group defines the group of the referenced resource. + enum: + - monitoring.coreos.com + type: string + name: + description: name defines the name of the referenced object. + minLength: 1 + type: string + namespace: + description: namespace defines the namespace of the referenced + object. + minLength: 1 + type: string + resource: + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). + enum: + - prometheuses + - prometheusagents + - thanosrulers + - alertmanagers + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map + type: object required: - spec type: object served: true storage: true + subresources: + status: {} diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_scrapeconfigs.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_scrapeconfigs.yaml index b6a61bb8bad..00f58e2c2e9 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_scrapeconfigs.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_scrapeconfigs.yaml @@ -43,15 +43,15 @@ spec: metadata: type: object spec: - description: ScrapeConfigSpec is a specification of the desired configuration - for a scrape configuration. + description: spec defines the specification of ScrapeConfigSpec. properties: authorization: - description: Authorization header to use on every scrape request. + description: authorization defines the header to use on every scrape + request. properties: credentials: - description: Selects a key of a Secret in the namespace that contains - the credentials for authentication. + description: credentials defines a key of a Secret in the namespace + that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must be @@ -76,7 +76,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -84,7 +84,7 @@ spec: type: string type: object azureSDConfigs: - description: AzureSDConfigs defines a list of Azure service discovery + description: azureSDConfigs defines a list of Azure service discovery configurations. items: description: |- @@ -93,7 +93,7 @@ spec: properties: authenticationMethod: description: |- - # The authentication method, either `OAuth` or `ManagedIdentity` or `SDK`. + authenticationMethod defines the authentication method, either `OAuth` or `ManagedIdentity` or `SDK`. See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview SDK authentication method uses environment variables by default. See https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication @@ -104,12 +104,12 @@ spec: type: string authorization: description: |- - Authorization header configuration to authenticate against the target HTTP endpoint. + authorization defines the authorization header configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `oAuth2`, or `basicAuth`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -134,7 +134,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -143,13 +143,13 @@ spec: type: object basicAuth: description: |- - BasicAuth information to authenticate against the target HTTP endpoint. + basicAuth defines the information to authenticate against the target HTTP endpoint. More info: https://prometheus.io/docs/operating/configuration/#endpoints Cannot be set at the same time as `authorization`, or `oAuth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -175,7 +175,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -201,13 +201,13 @@ spec: x-kubernetes-map-type: atomic type: object clientID: - description: Optional client ID. Only required with the OAuth - authentication method. + description: clientID defines client ID. Only required with + the OAuth authentication method. minLength: 1 type: string clientSecret: - description: Optional client secret. Only required with the - OAuth authentication method. + description: clientSecret defines client secret. Only required + with the OAuth authentication method. properties: key: description: The key of the secret to select from. Must @@ -231,37 +231,36 @@ spec: type: object x-kubernetes-map-type: atomic enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean environment: - description: The Azure environment. + description: environment defines the Azure environment. minLength: 1 type: string followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. - Cannot be set at the same time as `authorization`, or `basicAuth`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -284,7 +283,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -310,7 +310,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -338,12 +338,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -377,7 +377,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -385,33 +385,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -434,8 +433,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -460,12 +459,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -488,8 +487,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -514,11 +513,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -543,7 +543,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -554,7 +554,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -564,12 +564,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -579,7 +580,7 @@ spec: type: object port: description: |- - The port to scrape metrics from. If using the public IP address, this must + port defines the port to scrape metrics from. If using the public IP address, this must instead be specified in the relabeling rule. format: int32 maximum: 65535 @@ -613,7 +614,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -621,44 +622,47 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: RefreshInterval configures the refresh interval - at which Prometheus will re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string resourceGroup: description: |- - Optional resource group name. Limits discovery to this resource group. + resourceGroup defines resource group name. Limits discovery to this resource group. Requires Prometheus v2.35.0 and above minLength: 1 type: string subscriptionID: - description: The subscription ID. Always required. + description: subscriptionID defines subscription ID. Always + required. minLength: 1 type: string tenantID: - description: Optional tenant ID. Only required with the OAuth - authentication method. + description: tenantID defines tenant ID. Only required with + the OAuth authentication method. minLength: 1 type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defies the TLS configuration applying + to the target HTTP endpoint. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -681,7 +685,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -706,11 +711,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -733,7 +739,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -758,11 +765,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -787,7 +795,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -798,7 +806,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -808,7 +816,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -816,11 +825,12 @@ spec: type: object type: array basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -846,7 +856,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -872,7 +882,7 @@ spec: x-kubernetes-map-type: atomic type: object consulSDConfigs: - description: ConsulSDConfigs defines a list of Consul service discovery + description: consulSDConfigs defines a list of Consul service discovery configurations. items: description: |- @@ -881,17 +891,17 @@ spec: properties: allowStale: description: |- - Allow stale Consul results (see https://www.consul.io/api/features/consistency.html). Will reduce load on Consul. + allowStale Consul results (see https://www.consul.io/api/features/consistency.html). Will reduce load on Consul. If unset, Prometheus uses its default value. type: boolean authorization: description: |- - Optional Authorization header configuration to authenticate against the Consul Server. + authorization defines the header configuration to authenticate against the Consul Server. Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -916,7 +926,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -925,13 +935,13 @@ spec: type: object basicAuth: description: |- - Optional BasicAuth information to authenticate against the Consul Server. + basicAuth defines the information to authenticate against the Consul Server. More info: https://prometheus.io/docs/operating/configuration/#endpoints Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -957,7 +967,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -983,37 +993,34 @@ spec: x-kubernetes-map-type: atomic type: object datacenter: - description: Consul Datacenter name, if not provided it will - use the local Consul Agent Datacenter. + description: datacenter defines the consul Datacenter name, + if not provided it will use the local Consul Agent Datacenter. minLength: 1 type: string enableHTTP2: - description: |- - Whether to enable HTTP2. - If unset, Prometheus uses its default value. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean filter: description: |- - Filter expression used to filter the catalog results. + filter defines the filter expression used to filter the catalog results. See https://www.consul.io/api-docs/catalog#list-services It requires Prometheus >= 3.0.0. minLength: 1 type: string followRedirects: - description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. - If unset, Prometheus uses its default value. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean namespace: description: |- - Namespaces are only supported in Consul Enterprise. + namespace are only supported in Consul Enterprise. It requires Prometheus >= 2.28.0. minLength: 1 type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1023,23 +1030,23 @@ spec: additionalProperties: type: string description: |- - Node metadata key/value pairs to filter nodes for a given service. + nodeMeta defines the node metadata key/value pairs to filter nodes for a given service. Starting with Consul 1.14, it is recommended to use `filter` with the `NodeMeta` selector instead. type: object x-kubernetes-map-type: atomic oauth2: description: |- - Optional OAuth2.0 configuration. - Cannot be set at the same time as `basicAuth`, or `authorization`. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1062,7 +1069,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1088,7 +1096,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -1116,12 +1124,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1155,7 +1163,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1163,33 +1171,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1212,8 +1219,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1238,12 +1245,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1266,8 +1273,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1292,11 +1299,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1321,7 +1329,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1332,7 +1340,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1342,12 +1350,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -1356,12 +1365,13 @@ spec: - tokenUrl type: object partition: - description: Admin Partitions are only supported in Consul Enterprise. + description: partition defines the admin Partitions are only + supported in Consul Enterprise. minLength: 1 type: string pathPrefix: description: |- - Prefix for URIs for when consul is behind an API gateway (reverse proxy). + pathPrefix defines the prefix for URIs for when consul is behind an API gateway (reverse proxy). It requires Prometheus >= 2.45.0. minLength: 1 @@ -1394,7 +1404,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1402,63 +1412,64 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: description: |- - The time after which the provided names are refreshed. - On large setup it might be a good idea to increase this value because the catalog will change all the time. - If unset, Prometheus uses its default value. + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string scheme: - description: HTTP Scheme default "http" + description: scheme defines the HTTP Scheme default "http" enum: - HTTP - HTTPS type: string server: - description: Consul server address. A valid string consisting - of a hostname or IP followed by an optional port number. + description: server defines the consul server address. A valid + string consisting of a hostname or IP followed by an optional + port number. minLength: 1 type: string services: - description: A list of services for which targets are retrieved. - If omitted, all services are scraped. + description: services defines a list of services for which targets + are retrieved. If omitted, all services are scraped. items: type: string type: array x-kubernetes-list-type: set tagSeparator: description: |- - The string by which Consul tags are joined into the tag label. + tagSeparator defines the string by which Consul tags are joined into the tag label. If unset, Prometheus uses its default value. minLength: 1 type: string tags: description: |- - An optional list of tags used to filter nodes for a given service. Services must contain all tags in the list. + tags defines an optional list of tags used to filter nodes for a given service. Services must contain all tags in the list. Starting with Consul 1.14, it is recommended to use `filter` with the `ServiceTags` selector instead. items: type: string type: array x-kubernetes-list-type: set tlsConfig: - description: TLS configuration to connect to the Consul API. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1481,7 +1492,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1506,11 +1518,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1533,7 +1546,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1558,11 +1572,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1587,7 +1602,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1598,7 +1613,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1608,12 +1623,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object tokenRef: - description: Consul ACL TokenRef, if not provided it will use - the ACL from the local Consul Agent. + description: tokenRef defines the consul ACL TokenRef, if not + provided it will use the ACL from the local Consul Agent. properties: key: description: The key of the secret to select from. Must @@ -1642,11 +1658,11 @@ spec: type: array convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native histogram with custom buckets. + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.0.0. type: boolean digitalOceanSDConfigs: - description: DigitalOceanSDConfigs defines a list of DigitalOcean + description: digitalOceanSDConfigs defines a list of DigitalOcean service discovery configurations. items: description: |- @@ -1656,12 +1672,12 @@ spec: properties: authorization: description: |- - Authorization header configuration to authenticate against the DigitalOcean API. + authorization defines the header configuration to authenticate against the DigitalOcean API. Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -1686,7 +1702,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1694,33 +1710,32 @@ spec: type: string type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1743,7 +1758,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1769,7 +1785,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -1797,12 +1813,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1836,7 +1852,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1844,33 +1860,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1893,8 +1908,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1919,12 +1934,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1947,8 +1962,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1973,11 +1988,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2002,7 +2018,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2013,7 +2029,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2023,12 +2039,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -2037,7 +2054,8 @@ spec: - tokenUrl type: object port: - description: The port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -2070,7 +2088,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2078,28 +2096,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2122,7 +2143,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2147,11 +2169,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2174,7 +2197,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2199,11 +2223,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2228,7 +2253,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2239,7 +2264,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2249,13 +2274,14 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object type: object type: array dnsSDConfigs: - description: DNSSDConfigs defines a list of DNS service discovery + description: dnsSDConfigs defines a list of DNS service discovery configurations. items: description: |- @@ -2264,7 +2290,8 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config properties: names: - description: A list of DNS domain names to be queried. + description: names defines a list of DNS domain names to be + queried. items: minLength: 1 type: string @@ -2272,7 +2299,7 @@ spec: type: array port: description: |- - The port number used if the query type is not SRV + port defines the port to scrape metrics from. If using the public IP address, this must Ignored for SRV records format: int32 maximum: 65535 @@ -2280,13 +2307,13 @@ spec: type: integer refreshInterval: description: |- - RefreshInterval configures the time after which the provided names are refreshed. + refreshInterval defines the time after which the provided names are refreshed. If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: description: |- - The type of DNS query to perform. One of SRV, A, AAAA, MX or NS. + type defines the type of DNS query to perform. One of SRV, A, AAAA, MX or NS. If not set, Prometheus uses its default value. When set to NS, it requires Prometheus >= v2.49.0. @@ -2303,7 +2330,7 @@ spec: type: object type: array dockerSDConfigs: - description: DockerSDConfigs defines a list of Docker service discovery + description: dockerSDConfigs defines a list of Docker service discovery configurations. items: description: |- @@ -2314,12 +2341,12 @@ spec: properties: authorization: description: |- - Authorization header configuration to authenticate against the Docker API. + authorization defines the header configuration to authenticate against the DigitalOcean API. Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -2344,7 +2371,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -2352,11 +2379,12 @@ spec: type: string type: object basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -2382,7 +2410,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -2408,20 +2436,20 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean filters: - description: Optional filters to limit the discovery process - to a subset of the available resources. + description: filters defines filters to limit the discovery + process to a subset of the available resources. items: description: Filter name and value pairs to limit the discovery process to a subset of available resources. properties: name: - description: Name of the Filter. + description: name of the Filter. type: string values: - description: Value to filter on. + description: values defines values to filter on. items: minLength: 1 type: string @@ -2437,45 +2465,44 @@ spec: - name x-kubernetes-list-type: map followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean host: - description: Address of the docker daemon + description: host defines the address of the docker daemon minLength: 1 type: string hostNetworkingHost: - description: The host to use if the container is in host networking - mode. + description: hostNetworkingHost defines the host to use if the + container is in host networking mode. minLength: 1 type: string matchFirstNetwork: description: |- - Configure whether to match the first network if the container has multiple networks defined. + matchFirstNetwork defines whether to match the first network if the container has multiple networks defined. If unset, Prometheus uses true by default. It requires Prometheus >= v2.54.1. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2498,7 +2525,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2524,7 +2552,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -2552,12 +2580,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -2591,7 +2619,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2599,33 +2627,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2648,8 +2675,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -2674,12 +2701,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2702,8 +2729,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -2728,11 +2755,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2757,7 +2785,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2768,7 +2796,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2778,12 +2806,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -2792,7 +2821,8 @@ spec: - tokenUrl type: object port: - description: The port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -2825,7 +2855,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2833,28 +2863,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Time after which the container is refreshed. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2877,7 +2910,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2902,11 +2936,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2929,7 +2964,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2954,11 +2990,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2983,7 +3020,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2994,7 +3031,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3004,7 +3041,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -3012,7 +3050,7 @@ spec: type: object type: array dockerSwarmSDConfigs: - description: DockerswarmSDConfigs defines a list of Dockerswarm service + description: dockerSwarmSDConfigs defines a list of Dockerswarm service discovery configurations. items: description: |- @@ -3020,12 +3058,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config properties: authorization: - description: Authorization header configuration to authenticate - against the target HTTP endpoint. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -3050,7 +3089,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -3058,11 +3097,12 @@ spec: type: string type: object basicAuth: - description: Optional HTTP basic authentication information. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -3088,7 +3128,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -3114,11 +3154,11 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean filters: description: |- - Optional filters to limit the discovery process to a subset of available + filters defines the filters to limit the discovery process to a subset of available resources. The available filters are listed in the upstream documentation: Services: https://docs.docker.com/engine/api/v1.40/#operation/ServiceList @@ -3129,10 +3169,10 @@ spec: process to a subset of available resources. properties: name: - description: Name of the Filter. + description: name of the Filter. type: string values: - description: Value to filter on. + description: values defines values to filter on. items: minLength: 1 type: string @@ -3148,16 +3188,16 @@ spec: - name x-kubernetes-list-type: map followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean host: - description: Address of the Docker daemon + description: host defines the address of the Docker daemon pattern: ^[a-zA-Z][a-zA-Z0-9+.-]*://.+$ type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3165,17 +3205,17 @@ spec: type: string oauth2: description: |- - Optional OAuth 2.0 configuration. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3198,7 +3238,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -3224,7 +3265,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -3252,12 +3293,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3291,7 +3332,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3299,33 +3340,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3348,8 +3388,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -3374,12 +3414,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3402,8 +3442,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -3428,11 +3468,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -3457,7 +3498,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -3468,7 +3509,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3478,12 +3519,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -3493,7 +3535,7 @@ spec: type: object port: description: |- - The port to scrape metrics from, when `role` is nodes, and for discovered + port defines the port to scrape metrics from. If using the public IP address, this must tasks and services that don't have published ports. format: int32 maximum: 65535 @@ -3527,7 +3569,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3535,21 +3577,22 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: The time after which the service discovery data - is refreshed. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string role: - description: Role of the targets to retrieve. Must be `Services`, + description: role of the targets to retrieve. Must be `Services`, `Tasks`, or `Nodes`. enum: - Services @@ -3557,15 +3600,16 @@ spec: - Nodes type: string tlsConfig: - description: TLS configuration to use on every scrape request + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3588,7 +3632,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -3613,11 +3658,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3640,7 +3686,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -3665,11 +3712,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -3694,7 +3742,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -3705,7 +3753,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3715,7 +3763,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -3724,7 +3773,7 @@ spec: type: object type: array ec2SDConfigs: - description: EC2SDConfigs defines a list of EC2 service discovery + description: ec2SDConfigs defines a list of EC2 service discovery configurations. items: description: |- @@ -3737,7 +3786,7 @@ spec: BasicAuth, Authorization and OAuth2 fields are not present on purpose. properties: accessKey: - description: AccessKey is the AWS API key. + description: accessKey defines the AWS API key. properties: key: description: The key of the secret to select from. Must @@ -3762,12 +3811,12 @@ spec: x-kubernetes-map-type: atomic enableHTTP2: description: |- - Whether to enable HTTP2. + enableHTTP2 defines whether to enable HTTP2. It requires Prometheus >= v2.41.0 type: boolean filters: description: |- - Filters can be used optionally to filter the instance list by other criteria. + filters can be used optionally to filter the instance list by other criteria. Available filter criteria can be found here: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html Filter API documentation: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Filter.html @@ -3777,10 +3826,10 @@ spec: process to a subset of available resources. properties: name: - description: Name of the Filter. + description: name of the Filter. type: string values: - description: Value to filter on. + description: values defines values to filter on. items: minLength: 1 type: string @@ -3797,12 +3846,12 @@ spec: x-kubernetes-list-type: map followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.41.0 type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3810,7 +3859,7 @@ spec: type: string port: description: |- - The port to scrape metrics from. If using the public IP address, this must + port defines the port to scrape metrics from. If using the public IP address, this must instead be specified in the relabeling rule. format: int32 maximum: 65535 @@ -3844,7 +3893,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3852,29 +3901,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: RefreshInterval configures the refresh interval - at which Prometheus will re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: - description: The AWS region. + description: region defines the AWS region. minLength: 1 type: string roleARN: - description: AWS Role ARN, an alternative to using AWS API keys. + description: roleARN defines an alternative to using AWS API + keys. minLength: 1 type: string secretKey: - description: SecretKey is the AWS API secret. + description: secretKey defines the AWS API secret. properties: key: description: The key of the secret to select from. Must @@ -3899,16 +3950,16 @@ spec: x-kubernetes-map-type: atomic tlsConfig: description: |- - TLS configuration to connect to the AWS EC2 API. + tlsConfig defines the TLS configuration to connect to the Consul API. It requires Prometheus >= v2.41.0 properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3931,7 +3982,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -3956,11 +4008,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3983,7 +4036,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -4008,11 +4062,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -4037,7 +4092,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4048,7 +4103,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4058,24 +4113,25 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object type: object type: array enableCompression: description: |- - When false, Prometheus will request uncompressed response from the scraped target. + enableCompression when false, Prometheus will request uncompressed response from the scraped target. It requires Prometheus >= v2.49.0. If unset, Prometheus uses true by default. type: boolean enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean eurekaSDConfigs: - description: EurekaSDConfigs defines a list of Eureka service discovery + description: eurekaSDConfigs defines a list of Eureka service discovery configurations. items: description: |- @@ -4084,11 +4140,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config properties: authorization: - description: Authorization header to use on every scrape request. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -4113,7 +4171,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -4121,11 +4179,12 @@ spec: type: string type: object basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines the BasicAuth information to + use on every scrape request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -4151,7 +4210,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -4177,33 +4236,32 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization` or `basic_auth`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4226,7 +4284,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -4252,7 +4311,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -4280,12 +4339,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -4319,7 +4378,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4327,33 +4386,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4376,8 +4434,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -4402,12 +4460,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4430,8 +4488,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -4456,11 +4514,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -4485,7 +4544,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4496,7 +4555,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4506,12 +4565,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -4547,7 +4607,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4555,32 +4615,37 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string server: - description: The URL to connect to the Eureka server. + description: server defines the URL to connect to the Eureka + server. minLength: 1 + pattern: ^http(s)?://.+$ type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4603,7 +4668,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -4628,11 +4694,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4655,7 +4722,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -4680,11 +4748,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -4709,7 +4778,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4720,7 +4789,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4730,7 +4799,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -4739,7 +4809,7 @@ spec: type: array fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It requires Prometheus >= v3.0.0. enum: @@ -4750,7 +4820,7 @@ spec: - PrometheusText1.0.0 type: string fileSDConfigs: - description: FileSDConfigs defines a list of file service discovery + description: fileSDConfigs defines a list of file service discovery configurations. items: description: |- @@ -4759,7 +4829,7 @@ spec: properties: files: description: |- - List of files to be used for file discovery. Recommendation: use absolute paths. While relative paths work, the + files defines the list of files to be used for file discovery. Recommendation: use absolute paths. While relative paths work, the prometheus-operator project makes no guarantees about the working directory where the configuration file is stored. Files must be mounted using Prometheus.ConfigMaps or Prometheus.Secrets. @@ -4771,8 +4841,9 @@ spec: type: array x-kubernetes-list-type: set refreshInterval: - description: RefreshInterval configures the refresh interval - at which Prometheus will reload the content of the files. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string required: @@ -4780,7 +4851,7 @@ spec: type: object type: array gceSDConfigs: - description: GCESDConfigs defines a list of GCE service discovery + description: gceSDConfigs defines a list of GCE service discovery configurations. items: description: |- @@ -4800,36 +4871,37 @@ spec: properties: filter: description: |- - Filter can be used optionally to filter the instance list by other criteria + filter defines the filter that can be used optionally to filter the instance list by other criteria Syntax of this filter is described in the filter query parameter section: https://cloud.google.com/compute/docs/reference/latest/instances/list minLength: 1 type: string port: description: |- - The port to scrape metrics from. If using the public IP address, this must + port defines the port to scrape metrics from. If using the public IP address, this must instead be specified in the relabeling rule. format: int32 maximum: 65535 minimum: 0 type: integer project: - description: The Google Cloud Project ID + description: project defines the Google Cloud Project ID minLength: 1 type: string refreshInterval: - description: RefreshInterval configures the refresh interval - at which Prometheus will re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tagSeparator: - description: The tag separator is used to separate the tags - on concatenation + description: tagSeparator defines the tag separator is used + to separate the tags on concatenation minLength: 1 type: string zone: - description: The zone of the scrape targets. If you need multiple - zones use multiple GCESDConfigs. + description: zone defines the zone of the scrape targets. If + you need multiple zones use multiple GCESDConfigs. minLength: 1 type: string required: @@ -4838,7 +4910,7 @@ spec: type: object type: array hetznerSDConfigs: - description: HetznerSDConfigs defines a list of Hetzner service discovery + description: hetznerSDConfigs defines a list of Hetzner service discovery configurations. items: description: |- @@ -4848,12 +4920,12 @@ spec: properties: authorization: description: |- - Authorization header configuration, required when role is hcloud. - Role robot does not support bearer token authentication. + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -4878,7 +4950,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -4886,13 +4958,12 @@ spec: type: string type: object basicAuth: - description: |- - BasicAuth information to use on every scrape request, required when role is robot. - Role hcloud does not support basic auth. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -4918,7 +4989,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -4944,39 +5015,38 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean labelSelector: description: |- - Label selector used to filter the servers when fetching them from the API. + labelSelector defines the label selector used to filter the servers when fetching them from the API. It requires Prometheus >= v3.5.0. minLength: 1 type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be used at the same time as `basic_auth` or `authorization`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4999,7 +5069,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5025,7 +5096,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5053,12 +5124,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5092,7 +5163,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5100,33 +5171,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5149,8 +5219,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5175,12 +5245,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5203,8 +5273,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5229,11 +5299,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5258,7 +5329,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5269,7 +5340,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5279,12 +5350,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -5293,7 +5365,11 @@ spec: - tokenUrl type: object port: - description: The port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must + format: int32 + maximum: 65535 + minimum: 0 type: integer proxyConnectHeader: additionalProperties: @@ -5323,7 +5399,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5331,20 +5407,23 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: The time after which the servers are refreshed. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string role: - description: The Hetzner role of entities that should be discovered. + description: role defines the Hetzner role of entities that + should be discovered. enum: - hcloud - Hcloud @@ -5352,15 +5431,16 @@ spec: - Robot type: string tlsConfig: - description: TLS configuration to use on every scrape request. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5383,7 +5463,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5408,11 +5489,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5435,7 +5517,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5460,11 +5543,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5489,7 +5573,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5500,7 +5584,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5510,7 +5594,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -5518,15 +5603,17 @@ spec: type: object type: array honorLabels: - description: HonorLabels chooses the metric's labels on collisions - with target labels. + description: |- + honorLabels defines when true the metric's labels when they collide + with the target's labels. type: boolean honorTimestamps: - description: HonorTimestamps controls whether Prometheus respects - the timestamps present in scraped data. + description: |- + honorTimestamps defines whether Prometheus preserves the timestamps + when exposed by the target. type: boolean httpSDConfigs: - description: HTTPSDConfigs defines a list of HTTP service discovery + description: httpSDConfigs defines a list of HTTP service discovery configurations. items: description: |- @@ -5535,12 +5622,12 @@ spec: properties: authorization: description: |- - Authorization header configuration to authenticate against the target HTTP endpoint. + authorization defines the authorization header configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `oAuth2`, or `basicAuth`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -5565,7 +5652,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -5574,13 +5661,13 @@ spec: type: object basicAuth: description: |- - BasicAuth information to authenticate against the target HTTP endpoint. + basicAuth defines information to use on every scrape request. More info: https://prometheus.io/docs/operating/configuration/#endpoints Cannot be set at the same time as `authorization`, or `oAuth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -5606,7 +5693,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -5632,15 +5719,15 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5648,17 +5735,17 @@ spec: type: string oauth2: description: |- - Optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5681,7 +5768,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5707,7 +5795,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5735,12 +5823,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5774,7 +5862,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5782,33 +5870,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5831,8 +5918,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5857,12 +5944,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5885,8 +5972,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5911,11 +5998,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5940,7 +6028,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5951,7 +6039,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5961,12 +6049,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -6002,7 +6091,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6010,30 +6099,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: description: |- - RefreshInterval configures the refresh interval at which Prometheus will re-query the - endpoint to update the target list. + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration applying + to the target HTTP endpoint. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6056,7 +6146,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6081,11 +6172,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6108,7 +6200,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6133,11 +6226,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6162,7 +6256,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6173,7 +6267,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6183,11 +6277,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: URL from which the targets are fetched. + description: url defines the URL from which the targets are + fetched. minLength: 1 pattern: ^http(s)?://.+$ type: string @@ -6196,7 +6292,7 @@ spec: type: object type: array ionosSDConfigs: - description: IonosSDConfigs defines a list of IONOS service discovery + description: ionosSDConfigs defines a list of IONOS service discovery configurations. items: description: |- @@ -6204,12 +6300,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ionos_sd_config properties: authorization: - description: Authorization` header configuration, required when - using IONOS. + description: |- + authorization defines the header configuration to authenticate against the IONOS. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -6234,7 +6331,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6242,35 +6339,37 @@ spec: type: string type: object datacenterID: - description: The unique ID of the IONOS data center. + description: datacenterID defines the unique ID of the IONOS + data center. minLength: 1 type: string enableHTTP2: - description: Configure whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether the HTTP requests should follow + description: followRedirects defines whether HTTP requests follow HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: Configure whether to enable OAuth2. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6293,7 +6392,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6319,7 +6419,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -6347,12 +6447,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6386,7 +6486,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6394,33 +6494,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6443,8 +6542,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -6469,12 +6568,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6497,8 +6596,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -6523,11 +6622,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6552,7 +6652,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6563,7 +6663,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6573,12 +6673,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -6587,7 +6688,8 @@ spec: - tokenUrl type: object port: - description: Port to scrape the metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -6620,7 +6722,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6628,29 +6730,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the list of resources. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration to use when connecting to the - IONOS API. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6673,7 +6777,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6698,11 +6803,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6725,7 +6831,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6750,11 +6857,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6779,7 +6887,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6790,7 +6898,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6800,7 +6908,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -6810,7 +6919,7 @@ spec: type: array jobName: description: |- - The value of the `job` label assigned to the scraped metrics by default. + jobName defines the value of the `job` label assigned to the scraped metrics by default. The `job_name` field in the rendered scrape configuration is always controlled by the operator to prevent duplicate job names, which Prometheus does not allow. Instead the @@ -6819,14 +6928,14 @@ spec: type: string keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. format: int64 type: integer kubernetesSDConfigs: - description: KubernetesSDConfigs defines a list of Kubernetes service + description: kubernetesSDConfigs defines a list of Kubernetes service discovery configurations. items: description: |- @@ -6835,7 +6944,7 @@ spec: properties: apiServer: description: |- - The API server address consisting of a hostname or IP address followed + apiServer defines the API server address consisting of a hostname or IP address followed by an optional port number. If left empty, Prometheus is assumed to run inside of the cluster. It will discover API servers automatically and use the pod's @@ -6844,13 +6953,13 @@ spec: type: string attachMetadata: description: |- - Optional metadata to attach to discovered targets. + attachMetadata defines the metadata to attach to discovered targets. It requires Prometheus >= v2.35.0 when using the `Pod` role and Prometheus >= v2.37.0 for `Endpoints` and `Endpointslice` roles. properties: node: description: |- - Attaches node metadata to discovered targets. + node attaches node metadata to discovered targets. When set to true, Prometheus must have the `get` permission on the `Nodes` objects. Only valid for Pod, Endpoint and Endpointslice roles. @@ -6858,12 +6967,12 @@ spec: type: object authorization: description: |- - Authorization header to use on every scrape request. + authorization defines the authorization header to use on every scrape request. Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -6888,7 +6997,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6897,12 +7006,12 @@ spec: type: object basicAuth: description: |- - BasicAuth information to use on every scrape request. + basicAuth defines information to use on every scrape request. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -6928,7 +7037,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -6954,32 +7063,32 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean namespaces: - description: Optional namespace discovery. If omitted, Prometheus - discovers targets across all namespaces. + description: namespaces defines the namespace discovery. If + omitted, Prometheus discovers targets across all namespaces. properties: names: description: |- - List of namespaces where to watch for resources. + names defines a list of namespaces where to watch for resources. If empty and `ownNamespace` isn't true, Prometheus watches for resources in all namespaces. items: type: string type: array x-kubernetes-list-type: set ownNamespace: - description: Includes the namespace in which the Prometheus - pod runs to the list of watched namespaces. + description: ownNamespace includes the namespace in which + the Prometheus pod runs to the list of watched namespaces. type: boolean type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6987,17 +7096,17 @@ spec: type: string oauth2: description: |- - Optional OAuth 2.0 configuration. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7020,7 +7129,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7046,7 +7156,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -7074,12 +7184,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7113,7 +7223,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7121,33 +7231,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7170,8 +7279,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7196,12 +7305,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7224,8 +7333,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7250,11 +7359,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7279,7 +7389,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7290,7 +7400,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7300,12 +7410,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -7341,7 +7452,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7349,17 +7460,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string role: description: |- - Role of the Kubernetes entities that should be discovered. + role defines the Kubernetes role of the entities that should be discovered. Role `Endpointslice` requires Prometheus >= v2.21.0 enum: - Pod @@ -7371,26 +7482,26 @@ spec: type: string selectors: description: |- - Selector to select objects. + selectors defines the selector to select objects. It requires Prometheus >= v2.17.0 items: description: K8SSelectorConfig is Kubernetes Selector Config properties: field: description: |- - An optional field selector to limit the service discovery to resources which have fields with specific values. + field defines an optional field selector to limit the service discovery to resources which have fields with specific values. e.g: `metadata.name=foobar` minLength: 1 type: string label: description: |- - An optional label selector to limit the service discovery to resources with specific labels and label values. + label defines an optional label selector to limit the service discovery to resources with specific labels and label values. e.g: `node.kubernetes.io/instance-type=master` minLength: 1 type: string role: description: |- - Role specifies the type of Kubernetes resource to limit the service discovery to. + role defines the type of Kubernetes resource to limit the service discovery to. Accepted values are: Node, Pod, Endpoints, EndpointSlice, Service, Ingress. enum: - Pod @@ -7408,16 +7519,16 @@ spec: - role x-kubernetes-list-type: map tlsConfig: - description: TLS configuration to connect to the Kubernetes - API. + description: tlsConfig defines the TLS configuration to connect + to the Kubernetes API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7440,7 +7551,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7465,11 +7577,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7492,7 +7605,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7517,11 +7631,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7546,7 +7661,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7557,7 +7672,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7567,7 +7682,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -7575,7 +7691,7 @@ spec: type: object type: array kumaSDConfigs: - description: KumaSDConfigs defines a list of Kuma service discovery + description: kumaSDConfigs defines a list of Kuma service discovery configurations. items: description: |- @@ -7583,11 +7699,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kuma_sd_config properties: authorization: - description: Authorization header to use on every scrape request. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -7612,7 +7730,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -7620,11 +7738,12 @@ spec: type: string type: object basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -7650,7 +7769,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -7676,42 +7795,43 @@ spec: x-kubernetes-map-type: atomic type: object clientID: - description: Client id is used by Kuma Control Plane to compute - Monitoring Assignment for specific Prometheus backend. + description: |- + clientID is used by Kuma Control Plane to compute Monitoring Assignment for specific Prometheus backend. + It requires Prometheus >= v2.50.0. + minLength: 1 type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean fetchTimeout: - description: The time after which the monitoring assignments - are refreshed. + description: fetchTimeout defines the time after which the monitoring + assignments are refreshed. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization`, or `basicAuth`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7734,7 +7854,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7760,7 +7881,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -7788,12 +7909,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7827,7 +7948,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7835,33 +7956,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7884,8 +8004,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7910,12 +8030,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7938,8 +8058,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7964,11 +8084,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7993,7 +8114,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8004,7 +8125,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8014,12 +8135,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -8055,7 +8177,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8063,32 +8185,36 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: The time to wait between polling update requests. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string server: - description: Address of the Kuma Control Plane's MADS xDS server. - minLength: 1 + description: server defines the address of the Kuma Control + Plane's MADS xDS server. + pattern: ^https?://.+$ type: string tlsConfig: - description: TLS configuration to use on every scrape request + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8111,7 +8237,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8136,11 +8263,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8163,7 +8291,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8188,11 +8317,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8217,7 +8347,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8228,7 +8358,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8238,7 +8368,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -8247,24 +8378,24 @@ spec: type: array labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines the per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer lightSailSDConfigs: - description: LightsailSDConfigs defines a list of Lightsail service + description: lightSailSDConfigs defines a list of Lightsail service discovery configurations. items: description: |- @@ -8272,7 +8403,7 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#lightsail_sd_config properties: accessKey: - description: AccessKey is the AWS API key. + description: accessKey defines the AWS API key. properties: key: description: The key of the secret to select from. Must @@ -8297,12 +8428,12 @@ spec: x-kubernetes-map-type: atomic authorization: description: |- - Optional `authorization` HTTP header configuration. - Cannot be set at the same time as `basicAuth`, or `oauth2`. + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -8327,7 +8458,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -8336,12 +8467,12 @@ spec: type: object basicAuth: description: |- - Optional HTTP basic authentication information. + basicAuth defines information to use on every scrape request. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -8367,7 +8498,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -8393,19 +8524,19 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Configure whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean endpoint: - description: Custom endpoint to be used. + description: endpoint defines the custom endpoint to be used. minLength: 1 type: string followRedirects: - description: Configure whether the HTTP requests should follow + description: followRedirects defines whether HTTP requests follow HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -8413,17 +8544,17 @@ spec: type: string oauth2: description: |- - Optional OAuth2.0 configuration. - Cannot be set at the same time as `basicAuth`, or `authorization`. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8446,7 +8577,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8472,7 +8604,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -8500,12 +8632,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -8539,7 +8671,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8547,33 +8679,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8596,8 +8727,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -8622,12 +8753,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8650,8 +8781,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -8676,11 +8807,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8705,7 +8837,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8716,7 +8848,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8726,12 +8858,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -8740,9 +8873,8 @@ spec: - tokenUrl type: object port: - description: |- - Port to scrape the metrics from. - If using the public IP address, this must instead be specified in the relabeling rule. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -8775,7 +8907,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8783,27 +8915,30 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the list of instances. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: - description: The AWS region. + description: region defines the AWS region. minLength: 1 type: string roleARN: - description: AWS Role ARN, an alternative to using AWS API keys. + description: roleARN defines the AWS Role ARN, an alternative + to using AWS API keys. type: string secretKey: - description: SecretKey is the AWS API secret. + description: secretKey defines the AWS API secret. properties: key: description: The key of the secret to select from. Must @@ -8827,15 +8962,16 @@ spec: type: object x-kubernetes-map-type: atomic tlsConfig: - description: TLS configuration to connect to the Puppet DB. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8858,7 +8994,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8883,11 +9020,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8910,7 +9048,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8935,11 +9074,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8964,7 +9104,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8975,7 +9115,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8985,13 +9125,14 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object type: object type: array linodeSDConfigs: - description: LinodeSDConfigs defines a list of Linode service discovery + description: linodeSDConfigs defines a list of Linode service discovery configurations. items: description: |- @@ -8999,11 +9140,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#linode_sd_config properties: authorization: - description: Authorization header configuration. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -9028,7 +9171,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -9036,15 +9179,15 @@ spec: type: string type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -9052,17 +9195,17 @@ spec: type: string oauth2: description: |- - Optional OAuth 2.0 configuration. - Cannot be used at the same time as `authorization`. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9085,7 +9228,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9111,7 +9255,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -9139,12 +9283,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -9178,7 +9322,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9186,33 +9330,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9235,8 +9378,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -9261,12 +9404,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9289,8 +9432,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -9315,11 +9458,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -9344,7 +9488,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9355,7 +9499,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9365,12 +9509,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -9379,7 +9524,8 @@ spec: - tokenUrl type: object port: - description: Default port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -9412,7 +9558,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9420,37 +9566,40 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Time after which the linode instances are refreshed. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: - description: Optional region to filter on. + description: region defines the region to filter on. minLength: 1 type: string tagSeparator: - description: The string by which Linode Instance tags are joined - into the tag label. + description: tagSeparator defines the string by which Linode + Instance tags are joined into the tag label.el. minLength: 1 type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9473,7 +9622,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9498,11 +9648,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9525,7 +9676,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9550,11 +9702,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -9579,7 +9732,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9590,7 +9743,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9600,13 +9753,15 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object type: object type: array metricRelabelings: - description: MetricRelabelConfigs to apply to samples before ingestion. + description: metricRelabelings defines the metricRelabelings to apply + to samples before ingestion. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -9617,7 +9772,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -9649,40 +9804,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against which + the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated SourceLabels. + description: separator defines the string between concatenated + SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -9693,13 +9849,13 @@ spec: minItems: 1 type: array metricsPath: - description: MetricsPath HTTP path to scrape for metrics. If empty, - Prometheus uses the default value (e.g. /metrics). + description: metricsPath defines the HTTP path to scrape for metrics. + If empty, Prometheus uses the default value (e.g. /metrics). minLength: 1 type: string nameEscapingScheme: description: |- - Metric name escaping mode to request through content negotiation. + nameEscapingScheme defines the metric name escaping mode to request through content negotiation. It requires Prometheus >= v3.4.0. enum: @@ -9710,7 +9866,7 @@ spec: type: string nameValidationScheme: description: |- - Specifies the validation scheme for metric and label names. + nameValidationScheme defines the validation scheme for metric and label names. It requires Prometheus >= v3.0.0. enum: @@ -9719,7 +9875,7 @@ spec: type: string nativeHistogramBucketLimit: description: |- - If there are more than this many buckets in a native histogram, + nativeHistogramBucketLimit defines ff there are more than this many buckets in a native histogram, buckets will be merged to stay within the limit. It requires Prometheus >= v2.45.0. format: int64 @@ -9729,21 +9885,21 @@ spec: - type: integer - type: string description: |- - If the growth factor of one bucket to the next is smaller than this, + nativeHistogramMinBucketFactor defines if the growth factor of one bucket to the next is smaller than this, buckets will be merged to increase the factor sufficiently. It requires Prometheus >= v2.50.0. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string nomadSDConfigs: - description: NomadSDConfigs defines a list of Nomad service discovery + description: nomadSDConfigs defines a list of Nomad service discovery configurations. items: description: |- @@ -9752,15 +9908,17 @@ spec: properties: allowStale: description: |- - The information to access the Nomad API. It is to be defined + allowStale defines the information to access the Nomad API. It is to be defined as the Nomad documentation requires. type: boolean authorization: - description: Authorization header to use on every scrape request. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -9785,7 +9943,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -9793,11 +9951,12 @@ spec: type: string type: object basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -9823,7 +9982,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -9849,35 +10008,37 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean namespace: + description: |- + namespace defines the Nomad namespace to query for service discovery. + When specified, only resources within this namespace will be discovered. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization` or `basic_auth`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9900,7 +10061,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9926,7 +10088,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -9954,12 +10116,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -9993,7 +10155,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10001,33 +10163,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10050,8 +10211,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -10076,12 +10237,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10104,8 +10265,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -10130,11 +10291,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -10159,7 +10321,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10170,7 +10332,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10180,12 +10342,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -10221,7 +10384,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10229,38 +10392,47 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: + description: |- + region defines the Nomad region to query for service discovery. + When specified, only resources within this region will be discovered. type: string server: + description: |- + server defines the Nomad server address to connect to for service discovery. + This should be the full URL including protocol (e.g., "https://nomad.example.com:4646"). minLength: 1 type: string tagSeparator: + description: |- + tagSeparator defines the separator used to join multiple tags. + This determines how Nomad service tags are concatenated into Prometheus labels. type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10283,7 +10455,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10308,11 +10481,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10335,7 +10509,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10360,11 +10535,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -10389,7 +10565,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10400,7 +10576,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10410,7 +10586,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -10418,15 +10595,17 @@ spec: type: object type: array oauth2: - description: OAuth2 configuration to use on every scrape request. + description: oauth2 defines the configuration to use on every scrape + request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -10449,7 +10628,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10475,7 +10655,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -10503,12 +10683,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -10542,7 +10722,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10550,32 +10730,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for the - token request.' + description: scopes defines the OAuth2 scopes used for the token + request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10598,7 +10778,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10623,11 +10804,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10650,7 +10832,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10675,11 +10858,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -10704,7 +10888,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10715,7 +10899,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10725,12 +10909,12 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the token - from.' + description: tokenUrl defines the URL to fetch the token from. minLength: 1 type: string required: @@ -10739,7 +10923,7 @@ spec: - tokenUrl type: object openstackSDConfigs: - description: OpenStackSDConfigs defines a list of OpenStack service + description: openstackSDConfigs defines a list of OpenStack service discovery configurations. items: description: |- @@ -10748,15 +10932,15 @@ spec: properties: allTenants: description: |- - Whether the service discovery should list all instances for all projects. + allTenants defines whether the service discovery should list all instances for all projects. It is only relevant for the 'instance' role and usually requires admin permissions. type: boolean applicationCredentialId: - description: ApplicationCredentialID + description: applicationCredentialId defines the OpenStack applicationCredentialId. type: string applicationCredentialName: description: |- - The ApplicationCredentialID or ApplicationCredentialName fields are + applicationCredentialName defines the ApplicationCredentialID or ApplicationCredentialName fields are required if using an application credential to authenticate. Some providers allow you to create an application credential to authenticate rather than a password. @@ -10764,7 +10948,7 @@ spec: type: string applicationCredentialSecret: description: |- - The applicationCredentialSecret field is required if using an application + applicationCredentialSecret defines the required field if using an application credential to authenticate. properties: key: @@ -10789,7 +10973,8 @@ spec: type: object x-kubernetes-map-type: atomic availability: - description: Availability of the endpoint to connect to. + description: availability defines the availability of the endpoint + to connect to. enum: - Public - public @@ -10799,24 +10984,24 @@ spec: - internal type: string domainID: - description: DomainID + description: domainID defines The OpenStack domainID. minLength: 1 type: string domainName: description: |- - At most one of domainId and domainName must be provided if using username + domainName defines at most one of domainId and domainName that must be provided if using username with Identity V3. Otherwise, either are optional. minLength: 1 type: string identityEndpoint: description: |- - IdentityEndpoint specifies the HTTP endpoint that is required to work with + identityEndpoint defines the HTTP endpoint that is required to work with the Identity API of the appropriate version. pattern: ^http(s)?:\/\/.+$ type: string password: description: |- - Password for the Identity V2 and V3 APIs. Consult with your provider's + password defines the password for the Identity V2 and V3 APIs. Consult with your provider's control panel to discover your account's preferred method of authentication. properties: key: @@ -10842,35 +11027,37 @@ spec: x-kubernetes-map-type: atomic port: description: |- - The port to scrape metrics from. If using the public IP address, this must + port defines the port to scrape metrics from. If using the public IP address, this must instead be specified in the relabeling rule. format: int32 maximum: 65535 minimum: 0 type: integer projectID: - description: ' ProjectID' + description: projectID defines the OpenStack projectID. minLength: 1 type: string projectName: description: |- - The ProjectId and ProjectName fields are optional for the Identity V2 API. + projectName defines an optional field for the Identity V2 API. Some providers allow you to specify a ProjectName instead of the ProjectId. Some require both. Your provider's authentication policies will determine how these fields influence authentication. minLength: 1 type: string refreshInterval: - description: Refresh interval to re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: - description: The OpenStack Region. + description: region defines the OpenStack Region. minLength: 1 type: string role: description: |- - The OpenStack role of entities that should be discovered. + role defines the OpenStack role of entities that should be discovered. Note: The `LoadBalancer` role requires Prometheus >= v3.2.0. enum: @@ -10879,15 +11066,16 @@ spec: - LoadBalancer type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration applying + to the target HTTP endpoint. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10910,7 +11098,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10935,11 +11124,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10962,7 +11152,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10987,11 +11178,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -11016,7 +11208,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11027,7 +11219,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11037,16 +11229,17 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object userid: - description: UserID + description: userid defines the OpenStack userid. minLength: 1 type: string username: description: |- - Username is required if using Identity V2 API. Consult with your provider's + username defines the username required if using Identity V2 API. Consult with your provider's control panel to discover your account's username. In Identity V3, either userid or a combination of username and domainId or domainName are needed @@ -11058,7 +11251,7 @@ spec: type: object type: array ovhcloudSDConfigs: - description: OVHCloudSDConfigs defines a list of OVHcloud service + description: ovhcloudSDConfigs defines a list of OVHcloud service discovery configurations. items: description: |- @@ -11066,11 +11259,15 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ovhcloud_sd_config properties: applicationKey: - description: Access key to use. https://api.ovh.com. + description: |- + applicationKey defines the access key to use for OVHCloud API authentication. + This is obtained from the OVHCloud API credentials at https://api.ovh.com. minLength: 1 type: string applicationSecret: - description: SecretKeySelector selects a key of a Secret. + description: |- + applicationSecret defines the secret key for OVHCloud API authentication. + This contains the application secret obtained during OVHCloud API credential creation. properties: key: description: The key of the secret to select from. Must @@ -11094,7 +11291,9 @@ spec: type: object x-kubernetes-map-type: atomic consumerKey: - description: SecretKeySelector selects a key of a Secret. + description: |- + consumerKey defines the consumer key for OVHCloud API authentication. + This is the third component of OVHCloud's three-key authentication system. properties: key: description: The key of the secret to select from. Must @@ -11118,23 +11317,24 @@ spec: type: object x-kubernetes-map-type: atomic endpoint: - description: Custom endpoint to be used. + description: |- + endpoint defines a custom API endpoint to be used. + When not specified, defaults to the standard OVHCloud API endpoint for the region. minLength: 1 type: string refreshInterval: - description: Refresh interval to re-read the resources list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string service: - allOf: - - enum: - - VPS - - DedicatedServer - - enum: - - VPS - - DedicatedServer - description: Service of the targets to retrieve. Must be `VPS` - or `DedicatedServer`. + description: |- + service defines the service type of the targets to retrieve. + Must be either `VPS` or `DedicatedServer` to specify which OVHCloud resources to discover. + enum: + - VPS + - DedicatedServer type: string required: - applicationKey @@ -11148,7 +11348,7 @@ spec: items: type: string type: array - description: Optional HTTP URL parameters + description: params defines optional HTTP URL parameters type: object x-kubernetes-map-type: atomic proxyConnectHeader: @@ -11179,7 +11379,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11187,16 +11387,16 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string puppetDBSDConfigs: - description: PuppetDBSDConfigs defines a list of PuppetDB service + description: puppetDBSDConfigs defines a list of PuppetDB service discovery configurations. items: description: |- @@ -11205,12 +11405,12 @@ spec: properties: authorization: description: |- - Optional `authorization` HTTP header configuration. - Cannot be set at the same time as `basicAuth`, or `oauth2`. + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -11235,7 +11435,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -11244,12 +11444,12 @@ spec: type: object basicAuth: description: |- - Optional HTTP basic authentication information. + basicAuth defines information to use on every scrape request. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -11275,7 +11475,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -11301,21 +11501,21 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Configure whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether the HTTP requests should follow + description: followRedirects defines whether HTTP requests follow HTTP 3xx redirects. type: boolean includeParameters: description: |- - Whether to include the parameters as meta labels. + includeParameters defines whether to include the parameters as meta labels. Note: Enabling this exposes parameters in the Prometheus UI and API. Make sure that you don't have secrets exposed as parameters if you enable this. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -11323,17 +11523,17 @@ spec: type: string oauth2: description: |- - Optional OAuth2.0 configuration. - Cannot be set at the same time as `basicAuth`, or `authorization`. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11356,7 +11556,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11382,7 +11583,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -11410,12 +11611,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -11449,7 +11650,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11457,33 +11658,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11506,8 +11706,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -11532,12 +11732,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11560,8 +11760,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -11586,11 +11786,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -11615,7 +11816,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11626,7 +11827,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11636,12 +11837,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -11650,7 +11852,8 @@ spec: - tokenUrl type: object port: - description: Port to scrape the metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -11683,7 +11886,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11691,34 +11894,37 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string query: description: |- - Puppet Query Language (PQL) query. Only resources are supported. + query defines the Puppet Query Language (PQL) query. Only resources are supported. https://puppet.com/docs/puppetdb/latest/api/query/v4/pql.html minLength: 1 type: string refreshInterval: - description: Refresh interval to re-read the list of resources. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration to connect to the Puppet DB. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11741,7 +11947,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11766,11 +11973,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11793,7 +12001,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11818,11 +12027,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -11847,7 +12057,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11858,7 +12068,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11868,11 +12078,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the PuppetDB root query endpoint. + description: url defines the URL of the PuppetDB root query + endpoint. minLength: 1 pattern: ^http(s)?://.+$ type: string @@ -11883,7 +12095,7 @@ spec: type: array relabelings: description: |- - RelabelConfigs defines how to rewrite the target's labels before scraping. + relabelings defines how to rewrite the target's labels before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields. The original scrape job's name is available via the `__tmp_prometheus_job_name` label. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config @@ -11897,7 +12109,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -11929,40 +12141,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against which + the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated SourceLabels. + description: separator defines the string between concatenated + SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -11973,12 +12186,12 @@ spec: minItems: 1 type: array sampleLimit: - description: SampleLimit defines per-scrape limit on number of scraped + description: sampleLimit defines per-scrape limit on number of scraped samples that will be accepted. format: int64 type: integer scalewaySDConfigs: - description: ScalewaySDConfigs defines a list of Scaleway instances + description: scalewaySDConfigs defines a list of Scaleway instances and baremetal service discovery configurations. items: description: |- @@ -11986,41 +12199,43 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scaleway_sd_config properties: accessKey: - description: Access key to use. https://console.scaleway.com/project/credentials + description: accessKey defines the access key to use. https://console.scaleway.com/project/credentials minLength: 1 type: string apiURL: - description: API URL to use when doing the server listing requests. + description: apiURL defines the API URL to use when doing the + server listing requests. pattern: ^http(s)?://.+$ type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean nameFilter: - description: NameFilter specify a name filter (works as a LIKE) + description: nameFilter defines a name filter (works as a LIKE) to apply on the server listing request. minLength: 1 type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string port: - description: The port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 type: integer projectID: - description: Project ID of the targets. + description: projectID defines the Project ID of the targets. minLength: 1 type: string proxyConnectHeader: @@ -12051,7 +12266,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -12059,27 +12274,30 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the list of instances. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string role: - description: Service of the targets to retrieve. Must be `Instance` - or `Baremetal`. + description: role defines the service of the targets to retrieve. + Must be `Instance` or `Baremetal`. enum: - Instance - Baremetal type: string secretKey: - description: Secret key to use when listing targets. + description: secretKey defines the secret key to use when listing + targets. properties: key: description: The key of the secret to select from. Must @@ -12103,7 +12321,7 @@ spec: type: object x-kubernetes-map-type: atomic tagsFilter: - description: TagsFilter specify a tag filter (a server needs + description: tagsFilter defines a tag filter (a server needs to have all defined tags to be listed) to apply on the server listing request. items: @@ -12113,15 +12331,16 @@ spec: type: array x-kubernetes-list-type: set tlsConfig: - description: TLS configuration to use on every scrape request + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -12144,7 +12363,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -12169,11 +12389,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -12196,7 +12417,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -12221,11 +12443,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -12250,7 +12473,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -12261,7 +12484,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -12271,12 +12494,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object zone: - description: Zone is the availability zone of your targets (e.g. - fr-par-1). + description: zone defines the availability zone of your targets + (e.g. fr-par-1). minLength: 1 type: string required: @@ -12288,30 +12512,31 @@ spec: type: array scheme: description: |- - Configures the protocol scheme used for requests. + scheme defines the protocol scheme used for requests. If empty, Prometheus uses HTTP by default. enum: - HTTP - HTTPS type: string scrapeClass: - description: The scrape class to apply. + description: scrapeClass defines the scrape class to apply. minLength: 1 type: string scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. It requires Prometheus >= v2.45.0. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. type: boolean scrapeInterval: - description: ScrapeInterval is the interval between consecutive scrapes. + description: scrapeInterval defines the interval between consecutive + scrapes. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string scrapeProtocols: description: |- - The protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -12338,12 +12563,12 @@ spec: x-kubernetes-list-type: set scrapeTimeout: description: |- - ScrapeTimeout is the number of seconds to wait until a scrape request times out. + scrapeTimeout defines the number of seconds to wait until a scrape request times out. The value cannot be greater than the scrape interval otherwise the operator will reject the resource. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string staticConfigs: - description: StaticConfigs defines a list of static targets with a + description: staticConfigs defines a list of static targets with a common label set. items: description: |- @@ -12353,12 +12578,13 @@ spec: labels: additionalProperties: type: string - description: Labels assigned to all metrics scraped from the - targets. + description: labels defines labels assigned to all metrics scraped + from the targets. type: object x-kubernetes-map-type: atomic targets: - description: List of targets for this static configuration. + description: targets defines the list of targets for this static + configuration. items: description: |- Target represents a target for Prometheus to scrape @@ -12372,19 +12598,21 @@ spec: type: object type: array targetLimit: - description: TargetLimit defines a limit on the number of scraped + description: targetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer tlsConfig: - description: TLS configuration to use on every scrape request + description: tlsConfig defines the TLS configuration to use on every + scrape request properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when verifying + server certificates. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -12407,7 +12635,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -12432,10 +12661,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present when + doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -12458,7 +12689,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -12483,10 +12715,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must be @@ -12511,7 +12745,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -12522,7 +12756,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -12532,19 +12766,125 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for the + targets. type: string type: object trackTimestampsStaleness: description: |- - TrackTimestampsStaleness whether Prometheus tracks staleness of + trackTimestampsStaleness defines whether Prometheus tracks staleness of the metrics that have an explicit timestamp present in scraped data. Has no effect if `honorTimestamps` is false. It requires Prometheus >= v2.48.0. type: boolean type: object + status: + description: |- + status defines the status subresource. It is under active development and is updated only when the + "StatusForConfigurationResources" feature gate is enabled. + + Most recent observed status of the ScrapeConfig. Read-only. + More info: + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + bindings: + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. + items: + description: WorkloadBinding is a link between a configuration resource + and a workload resource. + properties: + conditions: + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. + items: + description: ConfigResourceCondition describes the status + of configuration resources linked to Prometheus, PrometheusAgent, + Alertmanager or ThanosRuler. + properties: + lastTransitionTime: + description: lastTransitionTime defines the time of the + last update to the current status property. + format: date-time + type: string + message: + description: message defines the human-readable message + indicating details for the condition's last transition. + type: string + observedGeneration: + description: |- + observedGeneration defines the .metadata.generation that the + condition was set based upon. For instance, if `.metadata.generation` is + currently 12, but the `.status.conditions[].observedGeneration` is 9, the + condition is out of date with respect to the current state of the object. + format: int64 + type: integer + reason: + description: reason for the condition's last transition. + type: string + status: + description: status of the condition. + minLength: 1 + type: string + type: + description: |- + type of the condition being reported. + Currently, only "Accepted" is supported. + enum: + - Accepted + minLength: 1 + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + group: + description: group defines the group of the referenced resource. + enum: + - monitoring.coreos.com + type: string + name: + description: name defines the name of the referenced object. + minLength: 1 + type: string + namespace: + description: namespace defines the namespace of the referenced + object. + minLength: 1 + type: string + resource: + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). + enum: + - prometheuses + - prometheusagents + - thanosrulers + - alertmanagers + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map + type: object required: - spec type: object served: true storage: true + subresources: + status: {} diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_servicemonitors.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_servicemonitors.yaml index 5913993c581..f1457ee9271 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_servicemonitors.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_servicemonitors.yaml @@ -50,19 +50,19 @@ spec: type: object spec: description: |- - Specification of desired Service selection for target discovery by + spec defines the specification of desired Service selection for target discovery by Prometheus. properties: attachMetadata: description: |- - `attachMetadata` defines additional metadata which is added to the + attachMetadata defines additional metadata which is added to the discovered targets. It requires Prometheus >= v2.37.0. properties: node: description: |- - When set to true, Prometheus attaches node metadata to the discovered + node when set to true, Prometheus attaches node metadata to the discovered targets. The Prometheus service account must have the `list` and `watch` @@ -71,7 +71,7 @@ spec: type: object bodySizeLimit: description: |- - When defined, bodySizeLimit specifies a job level limit on the size + bodySizeLimit when defined, bodySizeLimit specifies a job level limit on the size of uncompressed response body that will be accepted by Prometheus. It requires Prometheus >= v2.28.0. @@ -79,12 +79,12 @@ spec: type: string convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native histogram with custom buckets. + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.0.0. type: boolean endpoints: description: |- - List of endpoints part of this ServiceMonitor. + endpoints defines the list of endpoints part of this ServiceMonitor. Defines how to scrape metrics from Kubernetes [Endpoints](https://kubernetes.io/docs/concepts/services-networking/service/#endpoints) objects. In most cases, an Endpoints object is backed by a Kubernetes [Service](https://kubernetes.io/docs/concepts/services-networking/service/) object with the same name and labels. items: @@ -94,14 +94,14 @@ spec: properties: authorization: description: |- - `authorization` configures the Authorization header credentials to use when + authorization configures the Authorization header credentials to use when scraping the target. Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -126,7 +126,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -135,14 +135,14 @@ spec: type: object basicAuth: description: |- - `basicAuth` configures the Basic Authentication credentials to use when + basicAuth defines the Basic Authentication credentials to use when scraping the target. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -168,7 +168,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -195,13 +195,13 @@ spec: type: object bearerTokenFile: description: |- - File to read bearer token for scraping the target. + bearerTokenFile defines the file to read bearer token for scraping the target. Deprecated: use `authorization` instead. type: string bearerTokenSecret: description: |- - `bearerTokenSecret` specifies a key of a Secret containing the bearer + bearerTokenSecret defines a key of a Secret containing the bearer token for scraping targets. The secret needs to be in the same namespace as the ServiceMonitor object and readable by the Prometheus Operator. @@ -229,12 +229,12 @@ spec: type: object x-kubernetes-map-type: atomic enableHttp2: - description: '`enableHttp2` can be used to disable HTTP2 when - scraping the target.' + description: enableHttp2 can be used to disable HTTP2 when scraping + the target. type: boolean filterRunning: description: |- - When true, the pods which are not running (e.g. either in Failed or + filterRunning when true, the pods which are not running (e.g. either in Failed or Succeeded state) are dropped during the target discovery. If unset, the filtering is enabled. @@ -243,29 +243,29 @@ spec: type: boolean followRedirects: description: |- - `followRedirects` defines whether the scrape requests should follow HTTP + followRedirects defines whether the scrape requests should follow HTTP 3xx redirects. type: boolean honorLabels: description: |- - When true, `honorLabels` preserves the metric's labels when they collide + honorLabels defines when true the metric's labels when they collide with the target's labels. type: boolean honorTimestamps: description: |- - `honorTimestamps` controls whether Prometheus preserves the timestamps + honorTimestamps defines whether Prometheus preserves the timestamps when exposed by the target. type: boolean interval: description: |- - Interval at which Prometheus scrapes the metrics from the target. + interval at which Prometheus scrapes the metrics from the target. If empty, Prometheus uses the global scrape interval. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string metricRelabelings: description: |- - `metricRelabelings` configures the relabeling rules to apply to the + metricRelabelings defines the relabeling rules to apply to the samples before ingestion. items: description: |- @@ -277,7 +277,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -309,41 +309,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -354,7 +354,7 @@ spec: type: array noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -362,7 +362,7 @@ spec: type: string oauth2: description: |- - `oauth2` configures the OAuth2 settings to use when scraping the target. + oauth2 defines the OAuth2 settings to use when scraping the target. It requires Prometheus >= 2.27.0. @@ -370,12 +370,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -398,7 +398,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -424,7 +425,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -452,12 +453,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -491,7 +492,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -499,33 +500,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -548,8 +548,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -574,12 +574,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -602,8 +602,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -628,11 +628,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -657,7 +658,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -668,7 +669,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -678,12 +679,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -700,13 +702,13 @@ spec: type: object path: description: |- - HTTP path from which to scrape for metrics. + path defines the HTTP path from which to scrape for metrics. If empty, Prometheus uses the default value (e.g. `/metrics`). type: string port: description: |- - Name of the Service port which this endpoint refers to. + port defines the name of the Service port which this endpoint refers to. It takes precedence over `targetPort`. type: string @@ -738,7 +740,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -746,17 +748,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string relabelings: description: |- - `relabelings` configures the relabeling rules to apply the target's + relabelings defines the relabeling rules to apply the target's metadata labels. The Operator automatically adds relabelings for a few standard Kubernetes fields. @@ -774,7 +776,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -806,41 +808,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -851,7 +853,7 @@ spec: type: array scheme: description: |- - HTTP scheme to use for scraping. + scheme defines the HTTP scheme to use for scraping. `http` and `https` are the expected values unless you rewrite the `__scheme__` label via relabeling. @@ -863,7 +865,7 @@ spec: type: string scrapeTimeout: description: |- - Timeout after which Prometheus considers the scrape to be failed. + scrapeTimeout defines the timeout after which Prometheus considers the scrape to be failed. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used. @@ -875,19 +877,20 @@ spec: - type: integer - type: string description: |- - Name or number of the target port of the `Pod` object behind the + targetPort defines the name or number of the target port of the `Pod` object behind the Service. The port must be specified with the container's port property. x-kubernetes-int-or-string: true tlsConfig: - description: TLS configuration to use when scraping the target. + description: tlsConfig defines the TLS configuration to use + when scraping the target. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -910,7 +913,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -935,15 +939,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -966,7 +971,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -991,19 +997,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1028,7 +1035,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1039,7 +1046,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1049,12 +1056,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object trackTimestampsStaleness: description: |- - `trackTimestampsStaleness` defines whether Prometheus tracks staleness of + trackTimestampsStaleness defines whether Prometheus tracks staleness of the metrics that have an explicit timestamp present in scraped data. Has no effect if `honorTimestamps` is false. @@ -1064,7 +1072,7 @@ spec: type: array fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It requires Prometheus >= v3.0.0. enum: @@ -1076,7 +1084,7 @@ spec: type: string jobLabel: description: |- - `jobLabel` selects the label from the associated Kubernetes `Service` + jobLabel selects the label from the associated Kubernetes `Service` object which will be used as the `job` label for all metrics. For example if `jobLabel` is set to `foo` and the Kubernetes `Service` @@ -1089,7 +1097,7 @@ spec: type: string keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -1097,44 +1105,45 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines the per-scrape limit on number of labels that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer namespaceSelector: description: |- - `namespaceSelector` defines in which namespace(s) Prometheus should discover the services. + namespaceSelector defines in which namespace(s) Prometheus should discover the services. By default, the services are discovered in the same namespace as the `ServiceMonitor` object but it is possible to select pods across different/all namespaces. properties: any: description: |- - Boolean describing whether all namespaces are selected in contrast to a + any defines the boolean describing whether all namespaces are selected in contrast to a list restricting them. type: boolean matchNames: - description: List of namespace names to select from. + description: matchNames defines the list of namespace names to + select from. items: type: string type: array type: object nativeHistogramBucketLimit: description: |- - If there are more than this many buckets in a native histogram, + nativeHistogramBucketLimit defines ff there are more than this many buckets in a native histogram, buckets will be merged to stay within the limit. It requires Prometheus >= v2.45.0. format: int64 @@ -1144,38 +1153,38 @@ spec: - type: integer - type: string description: |- - If the growth factor of one bucket to the next is smaller than this, + nativeHistogramMinBucketFactor defines if the growth factor of one bucket to the next is smaller than this, buckets will be merged to increase the factor sufficiently. It requires Prometheus >= v2.50.0. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true podTargetLabels: description: |- - `podTargetLabels` defines the labels which are transferred from the + podTargetLabels defines the labels which are transferred from the associated Kubernetes `Pod` object onto the ingested metrics. items: type: string type: array sampleLimit: description: |- - `sampleLimit` defines a per-scrape limit on the number of scraped samples + sampleLimit defines a per-scrape limit on the number of scraped samples that will be accepted. format: int64 type: integer scrapeClass: - description: The scrape class to apply. + description: scrapeClass defines the scrape class to apply. minLength: 1 type: string scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. It requires Prometheus >= v2.45.0. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. type: boolean scrapeProtocols: description: |- - `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -1200,8 +1209,8 @@ spec: type: array x-kubernetes-list-type: set selector: - description: Label selector to select the Kubernetes `Endpoints` objects - to scrape metrics from. + description: selector defines the label selector to select the Kubernetes + `Endpoints` objects to scrape metrics from. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. @@ -1248,7 +1257,7 @@ spec: x-kubernetes-map-type: atomic selectorMechanism: description: |- - Mechanism used to select the endpoints to scrape. + selectorMechanism defines the mechanism used to select the endpoints to scrape. By default, the selection process relies on relabel configurations to filter the discovered targets. Alternatively, you can opt in for role selectors, which may offer better efficiency in large clusters. Which strategy is best for your use case needs to be carefully evaluated. @@ -1258,16 +1267,27 @@ spec: - RelabelConfig - RoleSelector type: string + serviceDiscoveryRole: + description: |- + serviceDiscoveryRole defines the service discovery role used to discover targets. + + If set, the value should be either "Endpoints" or "EndpointSlice". + Otherwise it defaults to the value defined in the + Prometheus/PrometheusAgent resource. + enum: + - Endpoints + - EndpointSlice + type: string targetLabels: description: |- - `targetLabels` defines the labels which are transferred from the + targetLabels defines the labels which are transferred from the associated Kubernetes `Service` object onto the ingested metrics. items: type: string type: array targetLimit: description: |- - `targetLimit` defines a limit on the number of scraped targets that will + targetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer @@ -1277,7 +1297,7 @@ spec: type: object status: description: |- - This Status subresource is under active development and is updated only when the + status defines the status subresource. It is under active development and is updated only when the "StatusForConfigurationResources" feature gate is enabled. Most recent observed status of the ServiceMonitor. Read-only. @@ -1285,47 +1305,48 @@ spec: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: bindings: - description: The list of workload resources (Prometheus or PrometheusAgent) - which select the configuration resource. + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. items: description: WorkloadBinding is a link between a configuration resource and a workload resource. properties: conditions: - description: The current state of the configuration resource - when bound to the referenced Prometheus object. + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. items: description: ConfigResourceCondition describes the status of configuration resources linked to Prometheus, PrometheusAgent, - Alertmanager, or ThanosRuler. + Alertmanager or ThanosRuler. properties: lastTransitionTime: - description: LastTransitionTime is the time of the last - update to the current status property. + description: lastTransitionTime defines the time of the + last update to the current status property. format: date-time type: string message: - description: Human-readable message indicating details - for the condition's last transition. + description: message defines the human-readable message + indicating details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the object. format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: description: |- - Type of the condition being reported. + type of the condition being reported. Currently, only "Accepted" is supported. enum: - Accepted @@ -1341,24 +1362,27 @@ spec: - type x-kubernetes-list-type: map group: - description: The group of the referenced resource. + description: group defines the group of the referenced resource. enum: - monitoring.coreos.com type: string name: - description: The name of the referenced object. + description: name defines the name of the referenced object. minLength: 1 type: string namespace: - description: The namespace of the referenced object. + description: namespace defines the namespace of the referenced + object. minLength: 1 type: string resource: - description: The type of resource being referenced (e.g. Prometheus - or PrometheusAgent). + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). enum: - prometheuses - prometheusagents + - thanosrulers + - alertmanagers type: string required: - group @@ -1367,6 +1391,12 @@ spec: - resource type: object type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map type: object required: - spec diff --git a/example/seed-crds/10-crd-monitoring.coreos.com_thanosrulers.yaml b/example/seed-crds/10-crd-monitoring.coreos.com_thanosrulers.yaml index 2285730da9d..6cb7205c60f 100644 --- a/example/seed-crds/10-crd-monitoring.coreos.com_thanosrulers.yaml +++ b/example/seed-crds/10-crd-monitoring.coreos.com_thanosrulers.yaml @@ -74,12 +74,12 @@ spec: type: object spec: description: |- - Specification of the desired behavior of the ThanosRuler cluster. More info: + spec defines the specification of the desired behavior of the ThanosRuler cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the ThanosRuler container. + additionalArgs defines how to add additional arguments for the ThanosRuler container. It is intended for e.g. activating hidden flags which are not supported by the dedicated configuration options yet. The arguments are passed as-is to the ThanosRuler container which may cause issues if they are invalid or not supported @@ -91,19 +91,20 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for name-only - arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. Can + be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name type: object type: array affinity: - description: If specified, the pod's scheduling constraints. + description: affinity defines when specified, the pod's scheduling + constraints. properties: nodeAffinity: description: Describes node affinity scheduling rules for the @@ -1016,7 +1017,7 @@ spec: type: object alertDropLabels: description: |- - Configures the label names which should be dropped in Thanos Ruler + alertDropLabels defines the label names which should be dropped in Thanos Ruler alerts. The replica label `thanos_ruler_replica` will always be dropped from the alerts. @@ -1025,13 +1026,13 @@ spec: type: array alertQueryUrl: description: |- - The external Query URL the Thanos Ruler will set in the 'Source' field + alertQueryUrl defines how Thanos Ruler will set in the 'Source' field of all alerts. Maps to the '--alert.query-url' CLI arg. type: string alertRelabelConfigFile: description: |- - Configures the path to the alert relabeling configuration file. + alertRelabelConfigFile defines the path to the alert relabeling configuration file. Alert relabel configuration must have the form as specified in the official Prometheus documentation: @@ -1043,7 +1044,7 @@ spec: type: string alertRelabelConfigs: description: |- - Configures alert relabeling in Thanos Ruler. + alertRelabelConfigs defines the alert relabeling in Thanos Ruler. Alert relabel configuration must have the form as specified in the official Prometheus documentation: @@ -1075,7 +1076,7 @@ spec: x-kubernetes-map-type: atomic alertmanagersConfig: description: |- - Configures the list of Alertmanager endpoints to send alerts to. + alertmanagersConfig defines the list of Alertmanager endpoints to send alerts to. The configuration format is defined at https://thanos.io/tip/components/rule.md/#alertmanager. @@ -1107,7 +1108,7 @@ spec: x-kubernetes-map-type: atomic alertmanagersUrl: description: |- - Configures the list of Alertmanager endpoints to send alerts to. + alertmanagersUrl defines the list of Alertmanager endpoints to send alerts to. For Thanos >= v0.10.0, it is recommended to use `alertmanagersConfig` instead. @@ -1117,7 +1118,7 @@ spec: type: array containers: description: |- - Containers allows injecting additional containers or modifying operator generated + containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a ThanosRuler pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a @@ -2644,11 +2645,12 @@ spec: type: object type: array dnsConfig: - description: Defines the DNS configuration for the pods. + description: dnsConfig defines Defines the DNS configuration for the + pods. properties: nameservers: description: |- - A list of DNS name server IP addresses. + nameservers defines the list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. items: minLength: 1 @@ -2657,7 +2659,7 @@ spec: x-kubernetes-list-type: set options: description: |- - A list of DNS resolver options. + options defines the list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Resolution options given in Options will override those that appear in the base DNSPolicy. @@ -2666,11 +2668,11 @@ spec: of a pod. properties: name: - description: Name is required and must be unique. + description: name is required and must be unique. minLength: 1 type: string value: - description: Value is optional. + description: value is optional. type: string required: - name @@ -2681,7 +2683,7 @@ spec: x-kubernetes-list-type: map searches: description: |- - A list of DNS search domains for host-name lookup. + searches defines the list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. items: minLength: 1 @@ -2690,7 +2692,7 @@ spec: x-kubernetes-list-type: set type: object dnsPolicy: - description: Defines the DNS policy for the pods. + description: dnsPolicy defines the DNS policy for the pods. enum: - ClusterFirstWithHostNet - ClusterFirst @@ -2699,7 +2701,7 @@ spec: type: string enableFeatures: description: |- - Enable access to Thanos Ruler feature flags. By default, no features are enabled. + enableFeatures defines how to setup Thanos Ruler feature flags. By default, no features are enabled. Enabling features which are disabled by default is entirely outside the scope of what the maintainers will support and by doing so, you accept @@ -2714,23 +2716,24 @@ spec: type: array x-kubernetes-list-type: set enableServiceLinks: - description: Indicates whether information about services should be - injected into pod's environment variables + description: enableServiceLinks defines whether information about + services should be injected into pod's environment variables type: boolean enforcedNamespaceLabel: description: |- - EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert + enforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created. type: string evaluationInterval: default: 15s - description: Interval between consecutive evaluations. + description: evaluationInterval defines the interval between consecutive + evaluations. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string excludedFromEnforcement: description: |- - List of references to PrometheusRule objects + excludedFromEnforcement defines the list of references to PrometheusRule objects to be excluded from enforcing a namespace label of origin. Applies only if enforcedNamespaceLabel set to true. items: @@ -2739,23 +2742,23 @@ spec: properties: group: default: monitoring.coreos.com - description: Group of the referent. When not specified, it defaults + description: group of the referent. When not specified, it defaults to `monitoring.coreos.com` enum: - monitoring.coreos.com type: string name: - description: Name of the referent. When not set, all resources + description: name of the referent. When not set, all resources in the namespace are matched. type: string namespace: description: |- - Namespace of the referent. + namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ minLength: 1 type: string resource: - description: Resource of the referent. + description: resource of the referent. enum: - prometheusrules - servicemonitors @@ -2770,23 +2773,24 @@ spec: type: array externalPrefix: description: |- - The external URL the Thanos Ruler instances will be available under. This is + externalPrefix defines the Thanos Ruler instances will be available under. This is necessary to generate correct URLs. This is necessary if Thanos Ruler is not served from root of a DNS name. type: string grpcServerTlsConfig: description: |- - GRPCServerTLSConfig configures the gRPC server from which Thanos Querier reads + grpcServerTlsConfig defines the gRPC server from which Thanos Querier reads recorded rule data. Note: Currently only the CAFile, CertFile, and KeyFile fields are supported. Maps to the '--grpc-server-tls-*' CLI args. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when verifying + server certificates. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -2809,7 +2813,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2834,14 +2839,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container to - use for the targets. + description: caFile defines the path to the CA cert in the Prometheus + container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present when + doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -2864,7 +2871,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2889,18 +2897,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus container - for the targets. + description: certFile defines the path to the client cert file + in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus container - for the targets. + description: keyFile defines the path to the client key file in + the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must be @@ -2925,7 +2935,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2936,7 +2946,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2946,23 +2956,24 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for the + targets. type: string type: object hostAliases: - description: Pods' hostAliases configuration + description: hostAliases defines pods' hostAliases configuration items: description: |- HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. + description: hostnames defines hostnames for the above IP address. items: type: string type: array ip: - description: IP address of the host file entry. + description: ip defines the IP address of the host file entry. type: string required: - hostnames @@ -2974,7 +2985,7 @@ spec: x-kubernetes-list-type: map hostUsers: description: |- - HostUsers supports the user space in Kubernetes. + hostUsers supports the user space in Kubernetes. More info: https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/ @@ -2982,11 +2993,11 @@ spec: Starting Kubernetes 1.33, the feature is enabled by default. type: boolean image: - description: Thanos container image URL. + description: image defines Thanos container image URL. type: string imagePullPolicy: description: |- - Image pull policy for the 'thanos', 'init-config-reloader' and 'config-reloader' containers. + imagePullPolicy defines for the 'thanos', 'init-config-reloader' and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. enum: - "" @@ -2996,7 +3007,7 @@ spec: type: string imagePullSecrets: description: |- - An optional list of references to secrets in the same namespace + imagePullSecrets defines an optional list of references to secrets in the same namespace to use for pulling thanos images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod items: @@ -3018,7 +3029,7 @@ spec: type: array initContainers: description: |- - InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. + initContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the ThanosRuler configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ @@ -4548,25 +4559,25 @@ spec: additionalProperties: type: string description: |- - Configures the external label pairs of the ThanosRuler resource. + labels defines the external label pairs of the ThanosRuler resource. A default replica label `thanos_ruler_replica` will be always added as a label with the value of the pod's name. type: object listenLocal: description: |- - ListenLocal makes the Thanos ruler listen on loopback, so that it + listenLocal defines the Thanos ruler listen on loopback, so that it does not bind against the Pod IP. type: boolean logFormat: - description: Log format for ThanosRuler to be configured with. + description: logFormat for ThanosRuler to be configured with. enum: - "" - logfmt - json type: string logLevel: - description: Log level for ThanosRuler to be configured with. + description: logLevel for ThanosRuler to be configured with. enum: - "" - debug @@ -4576,7 +4587,7 @@ spec: type: string minReadySeconds: description: |- - Minimum number of seconds for which a newly created pod should be ready + minReadySeconds defines the minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. If unset, pods will be considered available as soon as they are ready. @@ -4586,13 +4597,12 @@ spec: nodeSelector: additionalProperties: type: string - description: Define which Nodes the Pods are scheduled on. + description: nodeSelector defines which Nodes the Pods are scheduled + on. type: object objectStorageConfig: description: |- - Configures object storage. - - The configuration format is defined at https://thanos.io/tip/thanos/storage.md/#configuring-access-to-object-storage + objectStorageConfig defines the configuration format is defined at https://thanos.io/tip/thanos/storage.md/#configuring-access-to-object-storage The operator performs no validation of the configuration. @@ -4620,7 +4630,7 @@ spec: x-kubernetes-map-type: atomic objectStorageConfigFile: description: |- - Configures the path of the object storage configuration file. + objectStorageConfigFile defines the path of the object storage configuration file. The configuration format is defined at https://thanos.io/tip/thanos/storage.md/#configuring-access-to-object-storage @@ -4630,12 +4640,12 @@ spec: type: string paused: description: |- - When a ThanosRuler deployment is paused, no actions except for deletion + paused defines when a ThanosRuler deployment is paused, no actions except for deletion will be performed on the underlying objects. type: boolean podMetadata: description: |- - PodMetadata configures labels and annotations which are propagated to the ThanosRuler pods. + podMetadata defines labels and annotations which are propagated to the ThanosRuler pods. The following items are reserved and cannot be overridden: * "app.kubernetes.io/name" label, set to "thanos-ruler". @@ -4648,7 +4658,7 @@ spec: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -4657,14 +4667,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -4675,15 +4685,16 @@ spec: portName: default: web description: |- - Port name used for the pods and governing service. + portName defines the port name used for the pods and governing service. Defaults to `web`. type: string priorityClassName: - description: Priority class assigned to the Pods + description: priorityClassName defines the priority class assigned + to the Pods type: string prometheusRulesExcludedFromEnforce: description: |- - PrometheusRulesExcludedFromEnforce - list of Prometheus rules to be excluded from enforcing + prometheusRulesExcludedFromEnforce defines a list of Prometheus rules to be excluded from enforcing of adding namespace labels. Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair Deprecated: use excludedFromEnforcement instead. @@ -4694,10 +4705,12 @@ spec: namespace label for alerts and metrics. properties: ruleName: - description: Name of the excluded PrometheusRule object. + description: ruleName defines the name of the excluded PrometheusRule + object. type: string ruleNamespace: - description: Namespace of the excluded PrometheusRule object. + description: ruleNamespace defines the namespace of the excluded + PrometheusRule object. type: string required: - ruleName @@ -4706,7 +4719,7 @@ spec: type: array queryConfig: description: |- - Configures the list of Thanos Query endpoints from which to query metrics. + queryConfig defines the list of Thanos Query endpoints from which to query metrics. The configuration format is defined at https://thanos.io/tip/components/rule.md/#query-api @@ -4738,7 +4751,7 @@ spec: x-kubernetes-map-type: atomic queryEndpoints: description: |- - Configures the list of Thanos Query endpoints from which to query metrics. + queryEndpoints defines the list of Thanos Query endpoints from which to query metrics. For Thanos >= v0.11.0, it is recommended to use `queryConfig` instead. @@ -4748,7 +4761,7 @@ spec: type: array remoteWrite: description: |- - Defines the list of remote write configurations. + remoteWrite defines the list of remote write configurations. When the list isn't empty, the ruler is configured with stateless mode. @@ -4760,15 +4773,15 @@ spec: properties: authorization: description: |- - Authorization section for the URL. + authorization section for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. Cannot be set at the same time as `sigv4`, `basicAuth`, `oauth2`, or `azureAd`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -4792,12 +4805,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -4806,15 +4819,15 @@ spec: type: object azureAd: description: |- - AzureAD for the URL. + azureAd for the URL. It requires Prometheus >= v2.45.0 or Thanos >= v0.31.0. Cannot be set at the same time as `authorization`, `basicAuth`, `oauth2`, or `sigv4`. properties: cloud: - description: The Azure Cloud. Options are 'AzurePublic', - 'AzureChina', or 'AzureGovernment'. + description: cloud defines the Azure Cloud. Options are + 'AzurePublic', 'AzureChina', or 'AzureGovernment'. enum: - AzureChina - AzureGovernment @@ -4822,32 +4835,33 @@ spec: type: string managedIdentity: description: |- - ManagedIdentity defines the Azure User-assigned Managed identity. + managedIdentity defines the Azure User-assigned Managed identity. Cannot be set at the same time as `oauth` or `sdk`. properties: clientId: - description: The client id + description: clientId defines defines the Azure User-assigned + Managed identity. type: string required: - clientId type: object oauth: description: |- - OAuth defines the oauth config that is being used to authenticate. + oauth defines the oauth config that is being used to authenticate. Cannot be set at the same time as `managedIdentity` or `sdk`. It requires Prometheus >= v2.48.0 or Thanos >= v0.31.0. properties: clientId: - description: '`clientID` is the clientId of the Azure + description: clientId defines the clientId of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 type: string clientSecret: - description: '`clientSecret` specifies a key of a Secret + description: clientSecret specifies a key of a Secret containing the client secret of the Azure Active Directory - application that is being used to authenticate.' + application that is being used to authenticate. properties: key: description: The key of the secret to select from. Must @@ -4871,9 +4885,9 @@ spec: type: object x-kubernetes-map-type: atomic tenantId: - description: '`tenantId` is the tenant ID of the Azure + description: tenantId is the tenant ID of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 pattern: ^[0-9a-zA-Z-.]+$ type: string @@ -4884,29 +4898,29 @@ spec: type: object sdk: description: |- - SDK defines the Azure SDK config that is being used to authenticate. + sdk defines the Azure SDK config that is being used to authenticate. See https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication Cannot be set at the same time as `oauth` or `managedIdentity`. It requires Prometheus >= v2.52.0 or Thanos >= v0.36.0. properties: tenantId: - description: '`tenantId` is the tenant ID of the azure + description: tenantId defines the tenant ID of the azure active directory application that is being used to - authenticate.' + authenticate. pattern: ^[0-9a-zA-Z-.]+$ type: string type: object type: object basicAuth: description: |- - BasicAuth configuration for the URL. + basicAuth configuration for the URL. Cannot be set at the same time as `sigv4`, `authorization`, `oauth2`, or `azureAd`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -4932,7 +4946,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -4959,23 +4973,22 @@ spec: type: object bearerToken: description: |- + bearerToken is deprecated: this will be removed in a future release. *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File from which to read bearer token for the URL. + bearerTokenFile defines the file from which to read bearer token for the URL. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. type: boolean @@ -4983,14 +4996,14 @@ spec: additionalProperties: type: string description: |- - Custom HTTP headers to be sent along with each remote write request. + headers defines the custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. It requires Prometheus >= v2.25.0 or Thanos >= v0.24.0. type: object messageVersion: description: |- - The Remote Write message's version to use when writing to the endpoint. + messageVersion defines the Remote Write message's version to use when writing to the endpoint. `Version1.0` corresponds to the `prometheus.WriteRequest` protobuf message introduced in Remote Write 1.0. `Version2.0` corresponds to the `io.prometheus.write.v2.Request` protobuf message introduced in Remote Write 2.0. @@ -5007,37 +5020,37 @@ spec: - V2.0 type: string metadataConfig: - description: MetadataConfig configures the sending of series - metadata to the remote storage. + description: metadataConfig defines how to send a series metadata + to the remote storage. properties: maxSamplesPerSend: description: |- - MaxSamplesPerSend is the maximum number of metadata samples per send. + maxSamplesPerSend defines the maximum number of metadata samples per send. It requires Prometheus >= v2.29.0. format: int32 minimum: -1 type: integer send: - description: Defines whether metric metadata is sent to - the remote storage or not. + description: send defines whether metric metadata is sent + to the remote storage or not. type: boolean sendInterval: - description: Defines how frequently metric metadata is sent - to the remote storage. + description: sendInterval defines how frequently metric + metadata is sent to the remote storage. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object name: description: |- - The name of the remote write queue, it must be unique if specified. The + name of the remote write queue, it must be unique if specified. The name is used in metrics and logging in order to differentiate queues. It requires Prometheus >= v2.15.0 or Thanos >= 0.24.0. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5045,7 +5058,7 @@ spec: type: string oauth2: description: |- - OAuth2 configuration for the URL. + oauth2 configuration for the URL. It requires Prometheus >= v2.27.0 or Thanos >= v0.24.0. @@ -5053,12 +5066,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5081,7 +5094,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5107,7 +5121,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5135,12 +5149,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5174,7 +5188,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5182,33 +5196,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5231,8 +5244,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5257,12 +5270,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5285,8 +5298,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5311,11 +5324,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5340,7 +5354,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5351,7 +5365,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5361,12 +5375,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -5402,7 +5417,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5410,89 +5425,88 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string queueConfig: - description: QueueConfig allows tuning of the remote write queue + description: queueConfig allows tuning of the remote write queue parameters. properties: batchSendDeadline: - description: BatchSendDeadline is the maximum time a sample - will wait in buffer. + description: batchSendDeadline defines the maximum time + a sample will wait in buffer. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string capacity: description: |- - Capacity is the number of samples to buffer per shard before we start + capacity defines the number of samples to buffer per shard before we start dropping them. type: integer maxBackoff: - description: MaxBackoff is the maximum retry delay. + description: maxBackoff defines the maximum retry delay. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string maxRetries: - description: MaxRetries is the maximum number of times to - retry a batch on recoverable errors. + description: maxRetries defines the maximum number of times + to retry a batch on recoverable errors. type: integer maxSamplesPerSend: - description: MaxSamplesPerSend is the maximum number of - samples per send. + description: maxSamplesPerSend defines the maximum number + of samples per send. type: integer maxShards: - description: MaxShards is the maximum number of shards, + description: maxShards defines the maximum number of shards, i.e. amount of concurrency. type: integer minBackoff: - description: MinBackoff is the initial retry delay. Gets - doubled for every retry. + description: minBackoff defines the initial retry delay. + Gets doubled for every retry. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string minShards: - description: MinShards is the minimum number of shards, + description: minShards defines the minimum number of shards, i.e. amount of concurrency. type: integer retryOnRateLimit: description: |- - Retry upon receiving a 429 status code from the remote-write storage. + retryOnRateLimit defines the retry upon receiving a 429 status code from the remote-write storage. This is an *experimental feature*, it may change in any upcoming release in a breaking way. type: boolean sampleAgeLimit: description: |- - SampleAgeLimit drops samples older than the limit. + sampleAgeLimit drops samples older than the limit. It requires Prometheus >= v2.50.0 or Thanos >= v0.32.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object remoteTimeout: - description: Timeout for requests to the remote write endpoint. + description: remoteTimeout defines the timeout for requests + to the remote write endpoint. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string roundRobinDNS: - description: |- - When enabled: - - The remote-write mechanism will resolve the hostname via DNS. - - It will randomly select one of the resolved IP addresses and connect to it. - - When disabled (default behavior): - - The Go standard library will handle hostname resolution. - - It will attempt connections to each resolved IP address sequentially. - - Note: The connection timeout applies to the entire resolution and connection process. - If disabled, the timeout is distributed across all connection attempts. - - It requires Prometheus >= v3.1.0 or Thanos >= v0.38.0. + description: "roundRobinDNS controls the DNS resolution behavior + for remote-write connections.\nWhen enabled:\n - The remote-write + mechanism will resolve the hostname via DNS.\n - It will + randomly select one of the resolved IP addresses and connect + to it.\n\nWhen disabled (default behavior):\n - The Go standard + library will handle hostname resolution.\n - It will attempt + connections to each resolved IP address sequentially.\n\nNote: + The connection timeout applies to the entire resolution and + connection process.\n\n\tIf disabled, the timeout is distributed + across all connection attempts.\n\nIt requires Prometheus + >= v3.1.0 or Thanos >= v0.38.0." type: boolean sendExemplars: description: |- - Enables sending of exemplars over remote write. Note that + sendExemplars enables sending of exemplars over remote write. Note that exemplar-storage itself must be enabled using the `spec.enableFeatures` option for exemplars to be scraped in the first place. @@ -5500,14 +5514,14 @@ spec: type: boolean sendNativeHistograms: description: |- - Enables sending of native histograms, also known as sparse histograms + sendNativeHistograms enables sending of native histograms, also known as sparse histograms over remote write. It requires Prometheus >= v2.40.0 or Thanos >= v0.30.0. type: boolean sigv4: description: |- - Sigv4 allows to configures AWS's Signature Verification 4 for the URL. + sigv4 defines the AWS's Signature Verification 4 for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. @@ -5515,7 +5529,7 @@ spec: properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -5540,18 +5554,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used to authenticate. + description: profile defines the named AWS profile used + to authenticate. type: string region: - description: Region is the AWS region. If blank, the region - from the default credentials chain used. + description: region defines the AWS region. If blank, the + region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used to authenticate. + description: roleArn defines the named AWS profile used + to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -5575,17 +5591,22 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object tlsConfig: - description: TLS Config to use for the URL. + description: tlsConfig to use for the URL. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5608,7 +5629,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5633,15 +5655,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5664,7 +5687,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5689,19 +5713,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5726,7 +5751,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5737,7 +5762,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5747,15 +5772,18 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the endpoint to send samples to. + description: url defines the URL of the endpoint to send samples + to. minLength: 1 type: string writeRelabelConfigs: - description: The list of remote write relabel configurations. + description: writeRelabelConfigs defines the list of remote + write relabel configurations. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -5766,7 +5794,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -5798,41 +5826,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -5846,17 +5874,18 @@ spec: type: object type: array replicas: - description: Number of thanos ruler instances to deploy. + description: replicas defines the number of thanos ruler instances + to deploy. format: int32 type: integer resendDelay: - description: Minimum amount of time to wait before resending an alert - to Alertmanager. + description: resendDelay defines the minimum amount of time to wait + before resending an alert to Alertmanager. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string resources: description: |- - Resources defines the resource requirements for single Pods. + resources defines the resource requirements for single Pods. If not provided, no requests/limits will be set properties: claims: @@ -5918,7 +5947,7 @@ spec: retention: default: 24h description: |- - Time duration ThanosRuler shall retain data for. Default is '24h', and + retention defines the time duration ThanosRuler shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years). @@ -5927,26 +5956,26 @@ spec: pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string routePrefix: - description: The route prefix ThanosRuler registers HTTP handlers - for. This allows thanos UI to be served on a sub-path. + description: routePrefix defines the route prefix ThanosRuler registers + HTTP handlers for. This allows thanos UI to be served on a sub-path. type: string ruleConcurrentEval: description: |- - How many rules can be evaluated concurrently. + ruleConcurrentEval defines how many rules can be evaluated concurrently. It requires Thanos >= v0.37.0. format: int32 minimum: 1 type: integer ruleGracePeriod: description: |- - Minimum duration between alert and restored "for" state. + ruleGracePeriod defines the minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. It requires Thanos >= v0.30.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string ruleNamespaceSelector: description: |- - Namespaces to be selected for Rules discovery. If unspecified, only + ruleNamespaceSelector defines the namespaces to be selected for Rules discovery. If unspecified, only the same namespace as the ThanosRuler object is in is used. properties: matchExpressions: @@ -5994,19 +6023,19 @@ spec: x-kubernetes-map-type: atomic ruleOutageTolerance: description: |- - Max time to tolerate prometheus outage for restoring "for" state of alert. + ruleOutageTolerance defines the max time to tolerate prometheus outage for restoring "for" state of alert. It requires Thanos >= v0.30.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string ruleQueryOffset: description: |- - The default rule group's query offset duration to use. + ruleQueryOffset defines the default rule group's query offset duration to use. It requires Thanos >= v0.38.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string ruleSelector: description: |- - PrometheusRule objects to be selected for rule evaluation. An empty + ruleSelector defines the PrometheusRule objects to be selected for rule evaluation. An empty label selector matches all objects. A null label selector matches no objects. properties: @@ -6055,7 +6084,7 @@ spec: x-kubernetes-map-type: atomic securityContext: description: |- - SecurityContext holds pod-level security attributes and common container settings. + securityContext defines the pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: appArmorProfile: @@ -6289,12 +6318,12 @@ spec: type: object serviceAccountName: description: |- - ServiceAccountName is the name of the ServiceAccount to use to run the + serviceAccountName defines the name of the ServiceAccount to use to run the Thanos Ruler Pods. type: string serviceName: description: |- - The name of the service name used by the underlying StatefulSet(s) as the governing service. + serviceName defines the name of the service name used by the underlying StatefulSet(s) as the governing service. If defined, the Service must be created before the ThanosRuler resource in the same namespace and it must define a selector that matches the pod labels. If empty, the operator will create and manage a headless service named `thanos-ruler-operated` for ThanosRuler resources. When deploying multiple ThanosRuler resources in the same namespace, it is recommended to specify a different value for each. @@ -6302,15 +6331,16 @@ spec: minLength: 1 type: string storage: - description: Storage spec to specify how storage shall be used. + description: storage defines the specification of how storage shall + be used. properties: disableMountSubPath: - description: 'Deprecated: subPath usage will be removed in a future - release.' + description: 'disableMountSubPath deprecated: subPath usage will + be removed in a future release.' type: boolean emptyDir: description: |- - EmptyDirVolumeSource to be used by the StatefulSet. + emptyDir to be used by the StatefulSet. If specified, it takes precedence over `ephemeral` and `volumeClaimTemplate`. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir properties: @@ -6337,7 +6367,7 @@ spec: type: object ephemeral: description: |- - EphemeralVolumeSource to be used by the StatefulSet. + ephemeral to be used by the StatefulSet. This is a beta field in k8s 1.21 and GA in 1.15. For lower versions, starting with k8s 1.19, it requires enabling the GenericEphemeralVolume feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes @@ -6582,7 +6612,7 @@ spec: type: object volumeClaimTemplate: description: |- - Defines the PVC spec to be used by the Prometheus StatefulSets. + volumeClaimTemplate defines the PVC spec to be used by the Prometheus StatefulSets. The easiest way to use a volume that cannot be automatically provisioned is to use a label selector alongside manually created PersistentVolumes. properties: @@ -6602,14 +6632,14 @@ spec: More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: - description: EmbeddedMetadata contains metadata relevant to - an EmbeddedResource. + description: metadata defines EmbeddedMetadata contains metadata + relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -6618,14 +6648,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -6635,7 +6665,7 @@ spec: type: object spec: description: |- - Defines the desired characteristics of a volume requested by a pod author. + spec defines the specification of the characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: accessModes: @@ -6831,7 +6861,7 @@ spec: type: string type: object status: - description: 'Deprecated: this field is never set.' + description: 'status is deprecated: this field is never set.' properties: accessModes: description: |- @@ -7016,7 +7046,7 @@ spec: type: object terminationGracePeriodSeconds: description: |- - Optional duration in seconds the pod needs to terminate gracefully. + terminationGracePeriodSeconds defines the optional duration in seconds the pod needs to terminate gracefully. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down) which may lead to data corruption. @@ -7025,7 +7055,7 @@ spec: minimum: 0 type: integer tolerations: - description: If specified, the pod's tolerations. + description: tolerations defines when specified, the pod's tolerations. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -7064,7 +7094,8 @@ spec: type: object type: array topologySpreadConstraints: - description: If specified, the pod's topology spread constraints. + description: topologySpreadConstraints defines the pod's topology + spread constraints. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. @@ -7239,7 +7270,7 @@ spec: type: array tracingConfig: description: |- - Configures tracing. + tracingConfig defines the tracing configuration. The configuration format is defined at https://thanos.io/tip/thanos/tracing.md/#configuration @@ -7272,7 +7303,7 @@ spec: x-kubernetes-map-type: atomic tracingConfigFile: description: |- - Configures the path of the tracing configuration file. + tracingConfigFile defines the path of the tracing configuration file. The configuration format is defined at https://thanos.io/tip/thanos/tracing.md/#configuration @@ -7284,11 +7315,11 @@ spec: This field takes precedence over `tracingConfig`. type: string version: - description: Version of Thanos to be deployed. + description: version of Thanos to be deployed. type: string volumeMounts: description: |- - VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. + volumeMounts defines how the configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the ruler container, that are generated as a result of StorageSpec objects. items: @@ -7355,7 +7386,7 @@ spec: type: array volumes: description: |- - Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will + volumes defines how configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: description: Volume represents a named volume in a pod that may @@ -9240,22 +9271,24 @@ spec: type: object type: array web: - description: Defines the configuration of the ThanosRuler web server. + description: web defines the configuration of the ThanosRuler web + server. properties: httpConfig: - description: Defines HTTP parameters for web server. + description: httpConfig defines HTTP parameters for web server. properties: headers: - description: List of headers that can be added to HTTP responses. + description: headers defines a list of headers that can be + added to HTTP responses. properties: contentSecurityPolicy: description: |- - Set the Content-Security-Policy header to HTTP responses. + contentSecurityPolicy defines the Content-Security-Policy header to HTTP responses. Unset if blank. type: string strictTransportSecurity: description: |- - Set the Strict-Transport-Security header to HTTP responses. + strictTransportSecurity defines the Strict-Transport-Security header to HTTP responses. Unset if blank. Please make sure that you use this with care as this header might force browsers to load Prometheus and the other applications hosted on the same @@ -9264,7 +9297,7 @@ spec: type: string xContentTypeOptions: description: |- - Set the X-Content-Type-Options header to HTTP responses. + xContentTypeOptions defines the X-Content-Type-Options header to HTTP responses. Unset if blank. Accepted value is nosniff. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options enum: @@ -9273,7 +9306,7 @@ spec: type: string xFrameOptions: description: |- - Set the X-Frame-Options header to HTTP responses. + xFrameOptions defines the X-Frame-Options header to HTTP responses. Unset if blank. Accepted values are deny and sameorigin. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options enum: @@ -9283,32 +9316,32 @@ spec: type: string xXSSProtection: description: |- - Set the X-XSS-Protection header to all responses. + xXSSProtection defines the X-XSS-Protection header to all responses. Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection type: string type: object http2: description: |- - Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. + http2 enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. When TLSConfig is not configured, HTTP/2 will be disabled. Whenever the value of the field changes, a rolling update will be triggered. type: boolean type: object tlsConfig: - description: Defines the TLS parameters for HTTPS. + description: tlsConfig defines the TLS parameters for HTTPS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9331,7 +9364,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9357,7 +9391,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -9365,7 +9399,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -9375,14 +9409,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9405,7 +9439,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9431,21 +9466,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -9455,7 +9490,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -9463,7 +9498,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -9491,14 +9526,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -9509,18 +9546,19 @@ spec: type: object status: description: |- - Most recent observed status of the ThanosRuler cluster. Read-only. + status defines the most recent observed status of the ThanosRuler cluster. Read-only. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this ThanosRuler deployment. format: int32 type: integer conditions: - description: The current state of the ThanosRuler object. + description: conditions defines the current state of the ThanosRuler + object. items: description: |- Condition represents the state of the resources associated with the @@ -9532,12 +9570,12 @@ spec: format: date-time type: string message: - description: Human-readable message indicating details for the - condition's last transition. + description: message defines human-readable message indicating + details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the @@ -9545,14 +9583,14 @@ spec: format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: - description: Type of the condition being reported. + description: type of the condition being reported. minLength: 1 type: string required: @@ -9566,32 +9604,26 @@ spec: x-kubernetes-list-type: map paused: description: |- - Represents whether any actions on the underlying managed objects are + paused defines whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: |- - Total number of non-terminated pods targeted by this ThanosRuler deployment + replicas defines the total number of non-terminated pods targeted by this ThanosRuler deployment (their labels match the selector). format: int32 type: integer unavailableReplicas: - description: Total number of unavailable pods targeted by this ThanosRuler - deployment. + description: unavailableReplicas defines the total number of unavailable + pods targeted by this ThanosRuler deployment. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this ThanosRuler deployment + updatedReplicas defines the total number of non-terminated pods targeted by this ThanosRuler deployment that have the desired version spec. format: int32 type: integer - required: - - availableReplicas - - paused - - replicas - - unavailableReplicas - - updatedReplicas type: object required: - spec diff --git a/go.mod b/go.mod index a7ddcf13b4c..1b8e79e6c60 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/pelletier/go-toml v1.9.5 github.com/perses/perses-operator v0.2.0 - github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.85.0 + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.86.2 github.com/prometheus/blackbox_exporter v0.27.0 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/common v0.67.2 diff --git a/go.sum b/go.sum index ce1f18c37da..348dd24bcbf 100644 --- a/go.sum +++ b/go.sum @@ -623,8 +623,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.85.0 h1:oY+F5FZFmCjCyzkHWPjVQpzvnvEB/0FP+iyzDUUlqFc= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.85.0/go.mod h1:VB7wtBmDT6W2RJHzsvPZlBId+EnmeQA0d33fFTXvraM= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.86.2 h1:VRXUgbGmpmjZgFYiUnTwlC+JjfCUs5KKFsorJhI1ZKQ= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.86.2/go.mod h1:nPk0OteXBkbT0CRCa2oZQL1jRLW6RJ2fuIijHypeJdk= github.com/prometheus/blackbox_exporter v0.27.0 h1:LBya2SsKYc6rR/lICZXCt8fS6JWcB7LWrY6kI9fJmgI= github.com/prometheus/blackbox_exporter v0.27.0/go.mod h1:X1tfplLxckV+1vPT0JF5q2wYCg9k4chUuQBWg7y+OEA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index c5beb248513..4cc482ebc64 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -210,7 +210,7 @@ images: - name: prometheus-operator sourceRepository: github.com/prometheus-operator/prometheus-operator repository: quay.io/prometheus-operator/prometheus-operator - tag: v0.85.0 + tag: v0.86.2 labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -261,7 +261,7 @@ images: - name: configmap-reloader sourceRepository: github.com/prometheus-operator/prometheus-operator repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: v0.85.0 + tag: v0.86.2 labels: - name: gardener.cloud/cve-categorisation value: diff --git a/pkg/component/networking/istio/test_charts/istiod_servicemonitor.yaml b/pkg/component/networking/istio/test_charts/istiod_servicemonitor.yaml index 546eb495d85..23a168c2e04 100644 --- a/pkg/component/networking/istio/test_charts/istiod_servicemonitor.yaml +++ b/pkg/component/networking/istio/test_charts/istiod_servicemonitor.yaml @@ -18,4 +18,3 @@ spec: matchLabels: app: istiod istio: pilot -status: {} diff --git a/pkg/component/observability/monitoring/prometheus/seed/podmonitors.go b/pkg/component/observability/monitoring/prometheus/seed/podmonitors.go index 7af7c59da43..e7c05c054d2 100644 --- a/pkg/component/observability/monitoring/prometheus/seed/podmonitors.go +++ b/pkg/component/observability/monitoring/prometheus/seed/podmonitors.go @@ -67,8 +67,8 @@ func CentralPodMonitors() []*monitoringv1.PodMonitor { // {Key: "prometheus.io/port", Operator: metav1.LabelSelectorOpExists}, // }}, PodMetricsEndpoints: []monitoringv1.PodMetricsEndpoint{{ - Scheme: "https", - TLSConfig: &monitoringv1.SafeTLSConfig{InsecureSkipVerify: ptr.To(true)}, + Scheme: "https", + HTTPConfig: monitoringv1.HTTPConfig{TLSConfig: &monitoringv1.SafeTLSConfig{InsecureSkipVerify: ptr.To(true)}}, RelabelConfigs: []monitoringv1.RelabelConfig{ // TODO: These annotations should actually be labels so that PodMonitorSpec.Selector can be used // instead of manually crafting this relabel config. diff --git a/pkg/component/observability/monitoring/prometheus/seed/podmonitors_test.go b/pkg/component/observability/monitoring/prometheus/seed/podmonitors_test.go index c842031e8c7..1d2b9fea0b7 100644 --- a/pkg/component/observability/monitoring/prometheus/seed/podmonitors_test.go +++ b/pkg/component/observability/monitoring/prometheus/seed/podmonitors_test.go @@ -62,8 +62,8 @@ var _ = Describe("PodMonitors", func() { }, Spec: monitoringv1.PodMonitorSpec{ PodMetricsEndpoints: []monitoringv1.PodMetricsEndpoint{{ - Scheme: "https", - TLSConfig: &monitoringv1.SafeTLSConfig{InsecureSkipVerify: ptr.To(true)}, + Scheme: "https", + HTTPConfig: monitoringv1.HTTPConfig{TLSConfig: &monitoringv1.SafeTLSConfig{InsecureSkipVerify: ptr.To(true)}}, RelabelConfigs: []monitoringv1.RelabelConfig{ { SourceLabels: []monitoringv1.LabelName{ diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagerconfigs.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagerconfigs.yaml index 88429306f60..cdeeec7f519 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagerconfigs.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagerconfigs.yaml @@ -43,17 +43,11 @@ spec: metadata: type: object spec: - description: |- - AlertmanagerConfigSpec is a specification of the desired behavior of the - Alertmanager configuration. - By default, the Alertmanager configuration only applies to alerts for which - the `namespace` label is equal to the namespace of the AlertmanagerConfig - resource (see the `.spec.alertmanagerConfigMatcherStrategy` field of the - Alertmanager CRD). + description: spec defines the specification of AlertmanagerConfigSpec properties: inhibitRules: description: |- - List of inhibition rules. The rules will only apply to alerts matching + inhibitRules defines the list of inhibition rules. The rules will only apply to alerts matching the resource's namespace. items: description: |- @@ -63,23 +57,24 @@ spec: properties: equal: description: |- - Labels that must have an equal value in the source and target alert for - the inhibition to take effect. + equal defines labels that must have an equal value in the source and target alert + for the inhibition to take effect. This ensures related alerts are properly grouped. items: type: string type: array sourceMatch: description: |- - Matchers for which one or more alerts have to exist for the inhibition - to take effect. The operator enforces that the alert matches the - resource's namespace. + sourceMatch defines matchers for which one or more alerts have to exist for the inhibition + to take effect. The operator enforces that the alert matches the resource's namespace. + These are the "trigger" alerts that cause other alerts to be inhibited. items: description: Matcher defines how to match on alert's labels. properties: matchType: description: |- - Match operation available with AlertManager >= v0.22.0 and - takes precedence over Regex (deprecated) if non-empty. + matchType defines the match operation available with AlertManager >= v0.22.0. + Takes precedence over Regex (deprecated) if non-empty. + Valid values: "=" (equality), "!=" (inequality), "=~" (regex match), "!~" (regex non-match). enum: - '!=' - = @@ -87,16 +82,20 @@ spec: - '!~' type: string name: - description: Label to match. + description: |- + name defines the label to match. + This specifies which alert label should be evaluated. minLength: 1 type: string regex: description: |- - Whether to match on equality (false) or regular-expression (true). + regex defines whether to match on equality (false) or regular-expression (true). Deprecated: for AlertManager >= v0.22.0, `matchType` should be used instead. type: boolean value: - description: Label value to match. + description: |- + value defines the label value to match. + This is the expected value for the specified label. type: string required: - name @@ -104,15 +103,17 @@ spec: type: array targetMatch: description: |- - Matchers that have to be fulfilled in the alerts to be muted. The - operator enforces that the alert matches the resource's namespace. + targetMatch defines matchers that have to be fulfilled in the alerts to be muted. + The operator enforces that the alert matches the resource's namespace. + When these conditions are met, matching alerts will be inhibited (silenced). items: description: Matcher defines how to match on alert's labels. properties: matchType: description: |- - Match operation available with AlertManager >= v0.22.0 and - takes precedence over Regex (deprecated) if non-empty. + matchType defines the match operation available with AlertManager >= v0.22.0. + Takes precedence over Regex (deprecated) if non-empty. + Valid values: "=" (equality), "!=" (inequality), "=~" (regex match), "!~" (regex non-match). enum: - '!=' - = @@ -120,16 +121,20 @@ spec: - '!~' type: string name: - description: Label to match. + description: |- + name defines the label to match. + This specifies which alert label should be evaluated. minLength: 1 type: string regex: description: |- - Whether to match on equality (false) or regular-expression (true). + regex defines whether to match on equality (false) or regular-expression (true). Deprecated: for AlertManager >= v0.22.0, `matchType` should be used instead. type: boolean value: - description: Label value to match. + description: |- + value defines the label value to match. + This is the expected value for the specified label. type: string required: - name @@ -138,40 +143,40 @@ spec: type: object type: array muteTimeIntervals: - description: List of MuteTimeInterval specifying when the routes should - be muted. + description: muteTimeIntervals defines the list of MuteTimeInterval + specifying when the routes should be muted. items: description: MuteTimeInterval specifies the periods in time when notifications will be muted properties: name: - description: Name of the time interval + description: name of the time interval type: string timeIntervals: - description: TimeIntervals is a list of TimeInterval + description: timeIntervals defines a list of TimeInterval items: description: TimeInterval describes intervals of time properties: daysOfMonth: - description: DaysOfMonth is a list of DayOfMonthRange + description: daysOfMonth defines a list of DayOfMonthRange items: description: DayOfMonthRange is an inclusive range of days of the month beginning at 1 properties: end: - description: End of the inclusive range + description: end of the inclusive range maximum: 31 minimum: -31 type: integer start: - description: Start of the inclusive range + description: start of the inclusive range maximum: 31 minimum: -31 type: integer type: object type: array months: - description: Months is a list of MonthRange + description: months defines a list of MonthRange items: description: |- MonthRange is an inclusive range of months of the year beginning in January @@ -180,24 +185,25 @@ spec: type: string type: array times: - description: Times is a list of TimeRange + description: times defines a list of TimeRange items: description: TimeRange defines a start and end time in 24hr format properties: endTime: - description: EndTime is the end time in 24hr format. + description: endTime defines the end time in 24hr + format. pattern: ^((([01][0-9])|(2[0-3])):[0-5][0-9])$|(^24:00$) type: string startTime: - description: StartTime is the start time in 24hr - format. + description: startTime defines the start time in + 24hr format. pattern: ^((([01][0-9])|(2[0-3])):[0-5][0-9])$|(^24:00$) type: string type: object type: array weekdays: - description: Weekdays is a list of WeekdayRange + description: weekdays defines a list of WeekdayRange items: description: |- WeekdayRange is an inclusive range of days of the week beginning on Sunday @@ -206,7 +212,7 @@ spec: type: string type: array years: - description: Years is a list of YearRange + description: years defines a list of YearRange items: description: YearRange is an inclusive range of years pattern: ^2\d{3}(?::2\d{3}|$) @@ -219,12 +225,12 @@ spec: type: object type: array receivers: - description: List of receivers. + description: receivers defines the list of receivers. items: description: Receiver defines one or more notification integrations. properties: discordConfigs: - description: List of Discord configurations. + description: discordConfigs defines the list of Slack configurations. items: description: |- DiscordConfig configures notifications via Discord. @@ -232,7 +238,7 @@ spec: properties: apiURL: description: |- - The secret's key that contains the Discord webhook URL. + apiURL defines the secret's key that contains the Discord webhook URL. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -258,25 +264,27 @@ spec: type: object x-kubernetes-map-type: atomic avatarURL: - description: The avatar url of the message sender. + description: avatarURL defines the avatar url of the message + sender. pattern: ^https?://.+$ type: string content: - description: The template of the content's body. + description: content defines the template of the content's + body. minLength: 1 type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -301,7 +309,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -310,12 +318,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -341,7 +349,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -368,7 +376,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -394,30 +402,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -440,8 +453,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -468,7 +481,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -496,12 +509,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -537,7 +550,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -545,33 +558,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -594,8 +607,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -621,12 +634,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -649,8 +662,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -676,11 +689,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -706,7 +720,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -717,7 +731,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -727,13 +741,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -770,7 +784,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -778,31 +792,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -825,8 +840,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -852,12 +867,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -880,8 +895,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -907,11 +922,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -936,7 +952,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -947,7 +963,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -957,22 +973,26 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: The template of the message's body. + description: message defines the template of the message's + body. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean title: - description: The template of the message's title. + description: title defines the template of the message's + title. type: string username: - description: The username of the message sender. + description: username defines the username of the message + sender. minLength: 1 type: string required: @@ -980,16 +1000,18 @@ spec: type: object type: array emailConfigs: - description: List of Email configurations. + description: emailConfigs defines the list of Email configurations. items: description: EmailConfig configures notifications via Email. properties: authIdentity: - description: The identity to use for authentication. + description: |- + authIdentity defines the identity to use for SMTP authentication. + This is typically used with PLAIN authentication mechanism. type: string authPassword: description: |- - The secret's key that contains the password to use for authentication. + authPassword defines the secret's key that contains the password to use for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -1016,7 +1038,8 @@ spec: x-kubernetes-map-type: atomic authSecret: description: |- - The secret's key that contains the CRAM-MD5 secret. + authSecret defines the secret's key that contains the CRAM-MD5 secret. + This is used for CRAM-MD5 authentication mechanism. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -1042,24 +1065,32 @@ spec: type: object x-kubernetes-map-type: atomic authUsername: - description: The username to use for authentication. + description: |- + authUsername defines the username to use for SMTP authentication. + This is used for SMTP AUTH when the server requires authentication. type: string from: - description: The sender address. + description: |- + from defines the sender address for email notifications. + This appears as the "From" field in the email header. type: string headers: description: |- - Further headers email header key/value pairs. Overrides any headers - previously set by the notification implementation. + headers defines additional email header key/value pairs. + These override any headers previously set by the notification implementation. items: description: KeyValue defines a (key, value) tuple. properties: key: - description: Key of the tuple. + description: |- + key defines the key of the tuple. + This is the identifier or name part of the key-value pair. minLength: 1 type: string value: - description: Value of the tuple. + description: |- + value defines the value of the tuple. + This is the data or content associated with the key. type: string required: - key @@ -1067,36 +1098,46 @@ spec: type: object type: array hello: - description: The hostname to identify to the SMTP server. + description: |- + hello defines the hostname to identify to the SMTP server. + This is used in the SMTP HELO/EHLO command during the connection handshake. type: string html: - description: The HTML body of the email notification. + description: |- + html defines the HTML body of the email notification. + This allows for rich formatting in the email content. type: string requireTLS: description: |- - The SMTP TLS requirement. + requireTLS defines the SMTP TLS requirement. Note that Go does not support unencrypted connections to remote SMTP endpoints. type: boolean sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean smarthost: - description: The SMTP host and port through which emails - are sent. E.g. example.com:25 + description: |- + smarthost defines the SMTP host and port through which emails are sent. + Format should be "hostname:port", e.g. "smtp.example.com:587". type: string text: - description: The text body of the email notification. + description: |- + text defines the plain text body of the email notification. + This provides a fallback for email clients that don't support HTML. type: string tlsConfig: - description: TLS configuration + description: |- + tlsConfig defines the TLS configuration for SMTP connections. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1119,8 +1160,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1145,12 +1186,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1173,8 +1214,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1199,11 +1240,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file - for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1228,7 +1270,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1239,7 +1281,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1249,17 +1291,20 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object to: - description: The email address to send notifications to. + description: |- + to defines the email address to send notifications to. + This is the recipient address for alert notifications. type: string type: object type: array msteamsConfigs: description: |- - List of MSTeams configurations. + msteamsConfigs defines the list of MSTeams configurations. It requires Alertmanager >= 0.26.0. items: description: |- @@ -1267,17 +1312,18 @@ spec: It requires Alertmanager >= 0.26.0. properties: httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for Teams webhook requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -1302,7 +1348,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1311,12 +1357,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1342,7 +1388,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1369,7 +1415,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -1395,30 +1441,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1441,8 +1492,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1469,7 +1520,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -1497,12 +1548,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1538,7 +1589,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1546,33 +1597,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1595,8 +1646,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -1622,12 +1673,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1650,8 +1701,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -1677,11 +1728,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -1707,7 +1759,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1718,7 +1770,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1728,13 +1780,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -1771,7 +1823,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1779,31 +1831,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1826,8 +1879,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1853,12 +1906,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1881,8 +1934,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1908,11 +1961,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -1937,7 +1991,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1948,7 +2002,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1958,27 +2012,35 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean summary: description: |- - Message summary template. + summary defines the message summary template for Teams notifications. + This provides a brief overview that appears in Teams notification previews. It requires Alertmanager >= 0.27.0. type: string text: - description: Message body template. + description: |- + text defines the message body template for Teams notifications. + This contains the detailed content of the Teams message. type: string title: - description: Message title template. + description: |- + title defines the message title template for Teams notifications. + This appears as the main heading of the Teams message card. type: string webhookUrl: - description: MSTeams webhook URL. + description: |- + webhookUrl defines the MSTeams webhook URL for sending notifications. + This is the incoming webhook URL configured in your Teams channel. properties: key: description: The key of the secret to select from. Must @@ -2007,26 +2069,27 @@ spec: type: array msteamsv2Configs: description: |- - List of MSTeamsV2 configurations. + msteamsv2Configs defines the list of MSTeamsV2 configurations. It requires Alertmanager >= 0.28.0. items: description: |- - MSTeamsV2Config configures notifications via Microsoft Teams using the new message format with adaptive cards as required by flows + MSTeamsV2Config configures notifications via Microsoft Teams using the new message format with adaptive cards as required by flows. See https://prometheus.io/docs/alerting/latest/configuration/#msteamsv2_config It requires Alertmanager >= 0.28.0. properties: httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for Teams webhook requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -2051,7 +2114,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -2060,12 +2123,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -2091,7 +2154,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -2118,7 +2181,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -2144,30 +2207,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2190,8 +2258,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -2218,7 +2286,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -2246,12 +2314,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -2287,7 +2355,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2295,33 +2363,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2344,8 +2412,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -2371,12 +2439,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2399,8 +2467,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -2426,11 +2494,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -2456,7 +2525,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2467,7 +2536,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2477,13 +2546,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -2520,7 +2589,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2528,31 +2597,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2575,8 +2645,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -2602,12 +2672,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2630,8 +2700,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -2657,11 +2727,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -2686,7 +2757,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2697,7 +2768,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2707,24 +2778,31 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean text: - description: Message body template. + description: |- + text defines the message body template for adaptive card notifications. + This contains the detailed content displayed in the Teams adaptive card format. minLength: 1 type: string title: - description: Message title template. + description: |- + title defines the message title template for adaptive card notifications. + This appears as the main heading in the Teams adaptive card. minLength: 1 type: string webhookURL: - description: MSTeams incoming webhook URL. + description: |- + webhookURL defines the MSTeams incoming webhook URL for adaptive card notifications. + This webhook must support the newer adaptive cards format required by Teams flows. properties: key: description: The key of the secret to select from. Must @@ -2750,24 +2828,25 @@ spec: type: object type: array name: - description: Name of the receiver. Must be unique across all - items from the list. + description: name defines the name of the receiver. Must be + unique across all items from the list. minLength: 1 type: string opsgenieConfigs: - description: List of OpsGenie configurations. + description: opsgenieConfigs defines the list of OpsGenie configurations. items: description: |- OpsGenieConfig configures notifications via OpsGenie. See https://prometheus.io/docs/alerting/latest/configuration/#opsgenie_config properties: actions: - description: Comma separated list of actions that will - be available for the alert. + description: |- + actions defines a comma separated list of actions that will be available for the alert. + These appear as action buttons in the OpsGenie interface. type: string apiKey: description: |- - The secret's key that contains the OpsGenie API key. + apiKey defines the secret's key that contains the OpsGenie API key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -2793,23 +2872,32 @@ spec: type: object x-kubernetes-map-type: atomic apiURL: - description: The URL to send OpsGenie API requests to. + description: |- + apiURL defines the URL to send OpsGenie API requests to. + When not specified, defaults to the standard OpsGenie API endpoint. type: string description: - description: Description of the incident. + description: |- + description defines the detailed description of the incident. + This provides additional context beyond the message field. type: string details: - description: A set of arbitrary key/value pairs that provide - further detail about the incident. + description: |- + details defines a set of arbitrary key/value pairs that provide further detail about the incident. + These appear as additional fields in the OpsGenie alert. items: description: KeyValue defines a (key, value) tuple. properties: key: - description: Key of the tuple. + description: |- + key defines the key of the tuple. + This is the identifier or name part of the key-value pair. minLength: 1 type: string value: - description: Value of the tuple. + description: |- + value defines the value of the tuple. + This is the data or content associated with the key. type: string required: - key @@ -2817,21 +2905,23 @@ spec: type: object type: array entity: - description: Optional field that can be used to specify - which domain alert is related to. + description: |- + entity defines an optional field that can be used to specify which domain alert is related to. + This helps group related alerts together in OpsGenie. type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for OpsGenie API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -2856,7 +2946,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -2865,12 +2955,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -2896,7 +2986,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -2923,7 +3013,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -2949,30 +3039,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -2995,8 +3090,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -3023,7 +3118,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -3051,12 +3146,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3092,7 +3187,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3100,33 +3195,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3149,8 +3244,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -3176,12 +3271,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3204,8 +3299,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -3231,11 +3326,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -3261,7 +3357,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -3272,7 +3368,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3282,13 +3378,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -3325,7 +3421,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3333,31 +3429,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3380,8 +3477,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -3407,12 +3504,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3435,8 +3532,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -3462,11 +3559,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -3491,7 +3589,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -3502,7 +3600,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3512,97 +3610,129 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: Alert text limited to 130 characters. + description: |- + message defines the alert text limited to 130 characters. + This appears as the main alert title in OpsGenie. type: string note: - description: Additional alert note. + description: |- + note defines an additional alert note. + This provides supplementary information about the alert. type: string priority: - description: Priority level of alert. Possible values - are P1, P2, P3, P4, and P5. + description: |- + priority defines the priority level of alert. + Possible values are P1, P2, P3, P4, and P5, where P1 is highest priority. type: string responders: - description: List of responders responsible for notifications. + description: |- + responders defines the list of responders responsible for notifications. + These determine who gets notified when the alert is created. items: description: |- OpsGenieConfigResponder defines a responder to an incident. One of `id`, `name` or `username` has to be defined. properties: id: - description: ID of the responder. + description: |- + id defines the unique identifier of the responder. + This corresponds to the responder's ID within OpsGenie. type: string name: - description: Name of the responder. + description: |- + name defines the display name of the responder. + This is used when the responder is identified by name rather than ID. type: string type: - description: Type of responder. + description: |- + type defines the type of responder. + Valid values include "user", "team", "schedule", and "escalation". + This determines how OpsGenie interprets the other identifier fields. + enum: + - team + - teams + - user + - escalation + - schedule minLength: 1 type: string username: - description: Username of the responder. + description: |- + username defines the username of the responder. + This is typically used for user-type responders when identifying by username. type: string required: - type type: object type: array sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean source: - description: Backlink to the sender of the notification. + description: |- + source defines the backlink to the sender of the notification. + This helps identify where the alert originated from. type: string tags: - description: Comma separated list of tags attached to - the notifications. + description: |- + tags defines a comma separated list of tags attached to the notifications. + These help categorize and filter alerts within OpsGenie. type: string updateAlerts: description: |- - Whether to update message and description of the alert in OpsGenie if it already exists + updateAlerts defines Whether to update message and description of the alert in OpsGenie if it already exists By default, the alert is never updated in OpsGenie, the new message only appears in activity log. type: boolean type: object type: array pagerdutyConfigs: - description: List of PagerDuty configurations. + description: pagerdutyConfigs defines the List of PagerDuty + configurations. items: description: |- PagerDutyConfig configures notifications via PagerDuty. See https://prometheus.io/docs/alerting/latest/configuration/#pagerduty_config properties: class: - description: The class/type of the event. + description: class defines the class/type of the event. type: string client: - description: Client identification. + description: client defines the client identification. type: string clientURL: - description: Backlink to the sender of notification. + description: clientURL defines the backlink to the sender + of notification. type: string component: - description: The part or component of the affected system - that is broken. + description: component defines the part or component of + the affected system that is broken. type: string description: - description: Description of the incident. + description: description of the incident. type: string details: - description: Arbitrary key/value pairs that provide further - detail about the incident. + description: details defines the arbitrary key/value pairs + that provide further detail about the incident. items: description: KeyValue defines a (key, value) tuple. properties: key: - description: Key of the tuple. + description: |- + key defines the key of the tuple. + This is the identifier or name part of the key-value pair. minLength: 1 type: string value: - description: Value of the tuple. + description: |- + value defines the value of the tuple. + This is the data or content associated with the key. type: string required: - key @@ -3610,20 +3740,20 @@ spec: type: object type: array group: - description: A cluster or grouping of sources. + description: group defines a cluster or grouping of sources. type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -3648,7 +3778,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -3657,12 +3787,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -3688,7 +3818,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -3715,7 +3845,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -3741,30 +3871,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3787,8 +3922,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -3815,7 +3950,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -3843,12 +3978,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3884,7 +4019,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3892,33 +4027,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3941,8 +4076,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -3968,12 +4103,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -3996,8 +4131,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -4023,11 +4158,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -4053,7 +4189,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4064,7 +4200,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4074,13 +4210,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -4117,7 +4253,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4125,31 +4261,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4172,8 +4309,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -4199,12 +4336,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4227,8 +4364,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -4254,11 +4391,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -4283,7 +4421,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4294,7 +4432,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4304,51 +4442,55 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object pagerDutyImageConfigs: - description: A list of image details to attach that provide - further detail about an incident. + description: pagerDutyImageConfigs defines a list of image + details to attach that provide further detail about + an incident. items: description: PagerDutyImageConfig attaches images to an incident properties: alt: - description: Alt is the optional alternative text + description: alt is the optional alternative text for the image. type: string href: - description: Optional URL; makes the image a clickable - link. + description: href defines the optional URL; makes + the image a clickable link. type: string src: - description: Src of the image being attached to + description: src of the image being attached to the incident type: string type: object type: array pagerDutyLinkConfigs: - description: A list of link details to attach that provide - further detail about an incident. + description: pagerDutyLinkConfigs defines a list of link + details to attach that provide further detail about + an incident. items: description: PagerDutyLinkConfig attaches text links to an incident properties: alt: - description: Text that describes the purpose of - the link, and can be used as the link's text. + description: alt defines the text that describes + the purpose of the link, and can be used as the + link's text. type: string href: - description: Href is the URL of the link to be attached + description: href defines the URL of the link to + be attached type: string type: object type: array routingKey: description: |- - The secret's key that contains the PagerDuty integration key (when using + routingKey defines the secret's key that contains the PagerDuty integration key (when using Events API v2). Either this field or `serviceKey` needs to be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -4375,11 +4517,12 @@ spec: type: object x-kubernetes-map-type: atomic sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean serviceKey: description: |- - The secret's key that contains the PagerDuty service key (when using + serviceKey defines the secret's key that contains the PagerDuty service key (when using integration type "Prometheus"). Either this field or `routingKey` needs to be defined. The secret needs to be in the same namespace as the AlertmanagerConfig @@ -4407,49 +4550,53 @@ spec: type: object x-kubernetes-map-type: atomic severity: - description: Severity of the incident. + description: severity of the incident. type: string source: - description: Unique location of the affected system. + description: source defines the unique location of the + affected system. type: string url: - description: The URL to send requests to. + description: url defines the URL to send requests to. type: string type: object type: array pushoverConfigs: - description: List of Pushover configurations. + description: pushoverConfigs defines the list of Pushover configurations. items: description: |- PushoverConfig configures notifications via Pushover. See https://prometheus.io/docs/alerting/latest/configuration/#pushover_config properties: device: - description: The name of a device to send the notification - to + description: |- + device defines the name of a specific device to send the notification to. + If not specified, the notification is sent to all user's devices. type: string expire: description: |- - How long your notification will continue to be retried for, unless the user - acknowledges the notification. + expire defines how long your notification will continue to be retried for, + unless the user acknowledges the notification. Only applies to priority 2 notifications. pattern: ^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$ type: string html: - description: Whether notification message is HTML or plain - text. + description: |- + html defines whether notification message is HTML or plain text. + When true, the message can include HTML formatting tags. type: boolean httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for Pushover API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -4474,7 +4621,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -4483,12 +4630,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -4514,7 +4661,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -4541,7 +4688,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -4567,30 +4714,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4613,8 +4765,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -4641,7 +4793,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -4669,12 +4821,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -4710,7 +4862,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4718,33 +4870,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4767,8 +4919,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -4794,12 +4946,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4822,8 +4974,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -4849,11 +5001,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -4879,7 +5032,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4890,7 +5043,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4900,13 +5053,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -4943,7 +5096,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4951,31 +5104,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -4998,8 +5152,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5025,12 +5179,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5053,8 +5207,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5080,11 +5234,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -5109,7 +5264,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5120,7 +5275,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5130,37 +5285,45 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: Notification message. + description: |- + message defines the notification message content. + This is the main body text of the Pushover notification. type: string priority: - description: Priority, see https://pushover.net/api#priority + description: |- + priority defines the notification priority level. + See https://pushover.net/api#priority for valid values and behavior. type: string retry: description: |- - How often the Pushover servers will send the same notification to the user. - Must be at least 30 seconds. + retry defines how often the Pushover servers will send the same notification to the user. + Must be at least 30 seconds. Only applies to priority 2 notifications. pattern: ^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$ type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean sound: - description: The name of one of the sounds supported by - device clients to override the user's default sound - choice + description: |- + sound defines the name of one of the sounds supported by device clients. + This overrides the user's default sound choice for this notification. type: string title: - description: Notification title. + description: |- + title defines the notification title displayed in the Pushover message. + This appears as the bold header text in the notification. type: string token: description: |- - The secret's key that contains the registered application's API token, see https://pushover.net/apps. + token defines the secret's key that contains the registered application's API token. + See https://pushover.net/apps for application registration. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. Either `token` or `tokenFile` is required. @@ -5188,25 +5351,30 @@ spec: x-kubernetes-map-type: atomic tokenFile: description: |- - The token file that contains the registered application's API token, see https://pushover.net/apps. + tokenFile defines the token file that contains the registered application's API token. + See https://pushover.net/apps for application registration. Either `token` or `tokenFile` is required. It requires Alertmanager >= v0.26.0. type: string ttl: - description: The time to live definition for the alert - notification + description: |- + ttl defines the time to live for the alert notification. + This determines how long the notification remains active before expiring. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string url: - description: A supplementary URL shown alongside the message. + description: |- + url defines a supplementary URL shown alongside the message. + This creates a clickable link within the Pushover notification. type: string urlTitle: - description: A title for supplementary URL, otherwise - just the URL is shown + description: |- + urlTitle defines a title for the supplementary URL. + If not specified, the raw URL is shown instead. type: string userKey: description: |- - The secret's key that contains the recipient user's user key. + userKey defines the secret's key that contains the recipient user's user key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. Either `userKey` or `userKeyFile` is required. @@ -5234,7 +5402,7 @@ spec: x-kubernetes-map-type: atomic userKeyFile: description: |- - The user key file that contains the recipient user's user key. + userKeyFile defines the user key file that contains the recipient user's user key. Either `userKey` or `userKeyFile` is required. It requires Alertmanager >= v0.26.0. type: string @@ -5242,7 +5410,7 @@ spec: type: array rocketchatConfigs: description: |- - List of RocketChat configurations. + rocketchatConfigs defines the list of RocketChat configurations. It requires Alertmanager >= 0.28.0. items: description: |- @@ -5250,22 +5418,29 @@ spec: It requires Alertmanager >= 0.28.0. properties: actions: - description: Actions to include in the message. + description: |- + actions defines interactive actions to include in the message. + These appear as buttons that users can click to trigger responses. items: description: RocketChatActionConfig defines actions for RocketChat messages. properties: msg: - description: The message to send when the button - is clicked. + description: |- + msg defines the message to send when the button is clicked. + This allows the button to post a predefined message to the channel. minLength: 1 type: string text: - description: The button text. + description: |- + text defines the button text displayed to users. + This is the label that appears on the interactive button. minLength: 1 type: string url: - description: The URL the button links to. + description: |- + url defines the URL the button links to when clicked. + This creates a clickable button that opens the specified URL. pattern: ^https?://.+$ type: string type: object @@ -5273,57 +5448,69 @@ spec: type: array apiURL: description: |- - The API URL for RocketChat. + apiURL defines the API URL for RocketChat. Defaults to https://open.rocket.chat/ if not specified. pattern: ^https?://.+$ type: string channel: - description: The channel to send alerts to. + description: |- + channel defines the channel to send alerts to. + This can be a channel name (e.g., "#alerts") or a direct message recipient. minLength: 1 type: string color: - description: The message color. + description: |- + color defines the message color displayed in RocketChat. + This appears as a colored bar alongside the message. minLength: 1 type: string emoji: - description: If provided, the avatar will be displayed - as an emoji. + description: |- + emoji defines the emoji to be displayed as an avatar. + If provided, this emoji will be used instead of the default avatar or iconURL. minLength: 1 type: string fields: - description: Additional fields for the message. + description: |- + fields defines additional fields for the message attachment. + These appear as structured key-value pairs within the message. items: description: RocketChatFieldConfig defines additional fields for RocketChat messages. properties: short: - description: Whether this field should be a short - field. + description: |- + short defines whether this field should be a short field. + When true, the field may be displayed inline with other short fields to save space. type: boolean title: - description: The title of this field. + description: |- + title defines the title of this field. + This appears as bold text labeling the field content. minLength: 1 type: string value: - description: The value of this field, displayed - underneath the title value. + description: |- + value defines the value of this field, displayed underneath the title. + This contains the actual data or content for the field. minLength: 1 type: string type: object minItems: 1 type: array httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for RocketChat API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -5348,7 +5535,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -5357,12 +5544,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -5388,7 +5575,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -5415,7 +5602,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -5441,30 +5628,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5487,8 +5679,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5515,7 +5707,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5543,12 +5735,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5584,7 +5776,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5592,33 +5784,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5641,8 +5833,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -5668,12 +5860,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5696,8 +5888,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -5723,11 +5915,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -5753,7 +5946,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5764,7 +5957,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5774,13 +5967,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -5817,7 +6010,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5825,31 +6018,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5872,8 +6066,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5899,12 +6093,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -5927,8 +6121,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -5954,11 +6148,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -5983,7 +6178,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5994,7 +6189,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6004,47 +6199,65 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object iconURL: - description: Icon URL for the message. + description: |- + iconURL defines the icon URL for the message avatar. + This displays a custom image as the message sender's avatar. pattern: ^https?://.+$ type: string imageURL: - description: Image URL for the message. + description: |- + imageURL defines the image URL to display within the message. + This embeds an image directly in the message attachment. pattern: ^https?://.+$ type: string linkNames: - description: Whether to enable link names. + description: |- + linkNames defines whether to enable automatic linking of usernames and channels. + When true, @username and #channel references become clickable links. type: boolean sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean shortFields: - description: Whether to use short fields. + description: |- + shortFields defines whether to use short fields in the message layout. + When true, fields may be displayed side by side to save space. type: boolean text: - description: The message text to send, it is optional - because of attachments. + description: |- + text defines the message text to send. + This is optional because attachments can be used instead of or alongside text. minLength: 1 type: string thumbURL: - description: Thumbnail URL for the message. + description: |- + thumbURL defines the thumbnail URL for the message. + This displays a small thumbnail image alongside the message content. pattern: ^https?://.+$ type: string title: - description: The message title. + description: |- + title defines the message title displayed prominently in the message. + This appears as bold text at the top of the message attachment. minLength: 1 type: string titleLink: - description: The title link for the message. + description: |- + titleLink defines the URL that the title will link to when clicked. + This makes the message title clickable in the RocketChat interface. minLength: 1 type: string token: - description: The sender token. + description: |- + token defines the sender token for RocketChat authentication. + This is the personal access token or bot token used to authenticate API requests. properties: key: description: The key of the secret to select from. Must @@ -6068,7 +6281,9 @@ spec: type: object x-kubernetes-map-type: atomic tokenID: - description: The sender token ID. + description: |- + tokenID defines the sender token ID for RocketChat authentication. + This is the user ID associated with the token used for API requests. properties: key: description: The key of the secret to select from. Must @@ -6097,15 +6312,15 @@ spec: type: object type: array slackConfigs: - description: List of Slack configurations. + description: slackConfigs defines the list of Slack configurations. items: description: |- SlackConfig configures notifications via Slack. See https://prometheus.io/docs/alerting/latest/configuration/#slack_config properties: actions: - description: A list of Slack actions that are sent with - each notification. + description: actions defines a list of Slack actions that + are sent with each notification. items: description: |- SlackAction configures a single Slack action that is sent with each @@ -6115,37 +6330,64 @@ spec: properties: confirm: description: |- - SlackConfirmationField protect users from destructive actions or - particularly distinguished decisions by asking them to confirm their button - click one more time. - See https://api.slack.com/docs/interactive-message-field-guide#confirmation_fields - for more information. + confirm defines an optional confirmation dialog that appears before the action is executed. + When set, users must confirm their intent before the action proceeds. properties: dismissText: + description: |- + dismissText defines the label for the cancel button in the dialog. + When not specified, defaults to "Cancel". This button cancels the action. type: string okText: + description: |- + okText defines the label for the confirmation button in the dialog. + When not specified, defaults to "Okay". This button proceeds with the action. type: string text: + description: |- + text defines the main message displayed in the confirmation dialog. + This should be a clear question or statement asking the user to confirm their action. minLength: 1 type: string title: + description: |- + title defines the title text displayed at the top of the confirmation dialog. + When not specified, a default title will be used. type: string required: - text type: object name: + description: |- + name defines a unique identifier for the action within the message. + This value is sent back to your application when the action is triggered. type: string style: + description: |- + style defines the visual appearance of the action element. + Valid values include "default", "primary" (green), and "danger" (red). type: string text: + description: |- + text defines the user-visible label displayed on the action element. + For buttons, this is the button text. For select menus, this is the placeholder text. minLength: 1 type: string type: + description: |- + type defines the type of interactive component. + Common values include "button" for clickable buttons and "select" for dropdown menus. minLength: 1 type: string url: + description: |- + url defines the URL to open when the action is triggered. + Only applicable for button-type actions. When set, clicking the button opens this URL. type: string value: + description: |- + value defines the payload sent when the action is triggered. + This data is included in the callback sent to your application. type: string required: - text @@ -6154,7 +6396,7 @@ spec: type: array apiURL: description: |- - The secret's key that contains the Slack webhook URL. + apiURL defines the secret's key that contains the Slack webhook URL. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -6180,18 +6422,25 @@ spec: type: object x-kubernetes-map-type: atomic callbackId: + description: callbackId defines an identifier for the + message used in interactive components. type: string channel: - description: The channel or user to send notifications - to. + description: channel defines the channel or user to send + notifications to. type: string color: + description: |- + color defines the color of the left border of the Slack message attachment. + Can be a hex color code (e.g., "#ff0000") or a predefined color name. type: string fallback: + description: fallback defines a plain-text summary of + the attachment for clients that don't support attachments. type: string fields: - description: A list of Slack fields that are sent with - each notification. + description: fields defines a list of Slack fields that + are sent with each notification. items: description: |- SlackField configures a single Slack field that is sent with each notification. @@ -6200,11 +6449,21 @@ spec: See https://api.slack.com/docs/message-attachments#fields for more information. properties: short: + description: |- + short determines whether this field can be displayed alongside other short fields. + When true, Slack may display this field side by side with other short fields. + When false or not specified, the field takes the full width of the message. type: boolean title: + description: |- + title defines the label or header text displayed for this field. + This appears as bold text above the field value in the Slack message. minLength: 1 type: string value: + description: |- + value defines the content or data displayed for this field. + This appears below the title and can contain plain text or Slack markdown. minLength: 1 type: string required: @@ -6213,19 +6472,21 @@ spec: type: object type: array footer: + description: footer defines small text displayed at the + bottom of the message attachment. type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -6250,7 +6511,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6259,12 +6520,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -6290,7 +6551,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -6317,7 +6578,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -6343,30 +6604,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6389,8 +6655,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -6417,7 +6683,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -6445,12 +6711,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6486,7 +6752,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6494,33 +6760,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6543,8 +6809,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -6570,12 +6836,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6598,8 +6864,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -6625,11 +6891,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -6655,7 +6922,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6666,7 +6933,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6676,13 +6943,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -6719,7 +6986,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6727,31 +6994,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6774,8 +7042,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -6801,12 +7069,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -6829,8 +7097,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -6856,11 +7124,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -6885,7 +7154,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6896,7 +7165,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6906,44 +7175,72 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object iconEmoji: + description: iconEmoji defines the emoji to use as the + bot's avatar (e.g., ":ghost:"). type: string iconURL: + description: iconURL defines the URL to an image to use + as the bot's avatar. type: string imageURL: + description: imageURL defines the URL to an image file + that will be displayed inside the message attachment. type: string linkNames: + description: |- + linkNames enables automatic linking of channel names and usernames in the message. + When true, @channel and @username will be converted to clickable links. type: boolean mrkdwnIn: + description: |- + mrkdwnIn defines which fields should be parsed as Slack markdown. + Valid values include "pretext", "text", and "fields". items: type: string type: array pretext: + description: pretext defines optional text that appears + above the message attachment block. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean shortFields: + description: |- + shortFields determines whether fields are displayed in a compact format. + When true, fields are shown side by side when possible. type: boolean text: + description: text defines the main text content of the + Slack message attachment. type: string thumbURL: + description: |- + thumbURL defines the URL to an image file that will be displayed as a thumbnail + on the right side of the message attachment. type: string title: + description: title defines the title text displayed in + the Slack message attachment. type: string titleLink: + description: titleLink defines the URL that the title + will link to when clicked. type: string username: + description: username defines the slack bot user name. type: string type: object type: array snsConfigs: - description: List of SNS configurations + description: snsConfigs defines the list of SNS configurations items: description: |- SNSConfig configures notifications via AWS SNS. @@ -6951,26 +7248,29 @@ spec: properties: apiURL: description: |- - The SNS API URL i.e. https://sns.us-east-2.amazonaws.com. + apiURL defines the SNS API URL, e.g. https://sns.us-east-2.amazonaws.com. If not specified, the SNS API URL from the SNS SDK will be used. type: string attributes: additionalProperties: type: string - description: SNS message attributes. + description: |- + attributes defines SNS message attributes as key-value pairs. + These provide additional metadata that can be used for message filtering and routing. type: object httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for SNS API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -6995,7 +7295,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -7004,12 +7304,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -7035,7 +7335,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -7062,7 +7362,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -7088,30 +7388,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7134,8 +7439,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -7162,7 +7467,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -7190,12 +7495,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7231,7 +7536,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7239,33 +7544,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7288,8 +7593,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -7315,12 +7620,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7343,8 +7648,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -7370,11 +7675,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -7400,7 +7706,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7411,7 +7717,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7421,13 +7727,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -7464,7 +7770,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7472,31 +7778,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7519,8 +7826,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -7546,12 +7853,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7574,8 +7881,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -7601,11 +7908,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -7630,7 +7938,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7641,7 +7949,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7651,29 +7959,33 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: The message content of the SNS notification. + description: |- + message defines the message content of the SNS notification. + This is the actual notification text that will be sent to subscribers. type: string phoneNumber: description: |- - Phone number if message is delivered via SMS in E.164 format. + phoneNumber defines the phone number if message is delivered via SMS in E.164 format. If you don't specify this value, you must specify a value for the TopicARN or TargetARN. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean sigv4: - description: Configures AWS's Signature Verification 4 - signing process to sign requests. + description: |- + sigv4 configures AWS's Signature Verification 4 signing process to sign requests. + This includes AWS credentials and region configuration for authentication. properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -7698,20 +8010,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used - to authenticate. + description: profile defines the named AWS profile + used to authenticate. type: string region: - description: Region is the AWS region. If blank, the - region from the default credentials chain used. + description: region defines the AWS region. If blank, + the region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used - to authenticate. + description: roleArn defines the named AWS profile + used to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -7735,25 +8047,31 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object subject: - description: Subject line when the message is delivered - to email endpoints. + description: |- + subject defines the subject line when the message is delivered to email endpoints. + This field is only used when sending to email subscribers of an SNS topic. type: string targetARN: description: |- - The mobile platform endpoint ARN if message is delivered via mobile notifications. - If you don't specify this value, you must specify a value for the topic_arn or PhoneNumber. + targetARN defines the mobile platform endpoint ARN if message is delivered via mobile notifications. + If you don't specify this value, you must specify a value for the TopicARN or PhoneNumber. type: string topicARN: description: |- - SNS topic ARN, i.e. arn:aws:sns:us-east-2:698519295917:My-Topic + topicARN defines the SNS topic ARN, e.g. arn:aws:sns:us-east-2:698519295917:My-Topic. If you don't specify this value, you must specify a value for the PhoneNumber or TargetARN. type: string type: object type: array telegramConfigs: - description: List of Telegram configurations. + description: telegramConfigs defines the list of Telegram configurations. items: description: |- TelegramConfig configures notifications via Telegram. @@ -7761,15 +8079,14 @@ spec: properties: apiURL: description: |- - The Telegram API URL i.e. https://api.telegram.org. - If not specified, default API URL will be used. + apiURL defines the Telegram API URL, e.g. https://api.telegram.org. + If not specified, the default Telegram API URL will be used. type: string botToken: description: |- - Telegram bot token. It is mutually exclusive with `botTokenFile`. + botToken defines the Telegram bot token. It is mutually exclusive with `botTokenFile`. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. - Either `botToken` or `botTokenFile` is required. properties: key: @@ -7795,30 +8112,35 @@ spec: x-kubernetes-map-type: atomic botTokenFile: description: |- - File to read the Telegram bot token from. It is mutually exclusive with `botToken`. + botTokenFile defines the file to read the Telegram bot token from. + It is mutually exclusive with `botToken`. Either `botToken` or `botTokenFile` is required. - It requires Alertmanager >= v0.26.0. type: string chatID: - description: The Telegram chat ID. + description: |- + chatID defines the Telegram chat ID where messages will be sent. + This can be a user ID, group ID, or channel ID (with @ prefix for public channels). format: int64 type: integer disableNotifications: - description: Disable telegram notifications + description: |- + disableNotifications controls whether Telegram notifications are sent silently. + When true, users will receive the message without notification sounds. type: boolean httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for Telegram API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -7843,7 +8165,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -7852,12 +8174,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -7883,7 +8205,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -7910,7 +8232,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -7936,30 +8258,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -7982,8 +8309,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -8010,7 +8337,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -8038,12 +8365,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -8079,7 +8406,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8087,33 +8414,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8136,8 +8463,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -8163,12 +8490,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8191,8 +8518,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -8218,11 +8545,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -8248,7 +8576,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8259,7 +8587,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8269,13 +8597,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -8312,7 +8640,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8320,31 +8648,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8367,8 +8696,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -8394,12 +8723,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8422,8 +8751,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -8449,11 +8778,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -8478,7 +8808,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8489,7 +8819,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8499,36 +8829,44 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: Message template + description: |- + message defines the message template for the Telegram notification. + This is the content that will be sent to the specified chat. type: string messageThreadID: description: |- - The Telegram Group Topic ID. + messageThreadID defines the Telegram Group Topic ID for threaded messages. + This allows sending messages to specific topics within Telegram groups. It requires Alertmanager >= 0.26.0. format: int64 type: integer parseMode: - description: Parse mode for telegram message + description: |- + parseMode defines the parse mode for telegram message formatting. + Valid values are "MarkdownV2", "Markdown", and "HTML". + This determines how text formatting is interpreted in the message. enum: - MarkdownV2 - Markdown - HTML type: string sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean required: - chatID type: object type: array victoropsConfigs: - description: List of VictorOps configurations. + description: victoropsConfigs defines the list of VictorOps + configurations. items: description: |- VictorOpsConfig configures notifications via VictorOps. @@ -8536,7 +8874,7 @@ spec: properties: apiKey: description: |- - The secret's key that contains the API key to use when talking to the VictorOps API. + apiKey defines the secret's key that contains the API key to use when talking to the VictorOps API. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -8562,19 +8900,27 @@ spec: type: object x-kubernetes-map-type: atomic apiUrl: - description: The VictorOps API URL. + description: |- + apiUrl defines the VictorOps API URL. + When not specified, defaults to the standard VictorOps API endpoint. type: string customFields: - description: Additional custom fields for notification. + description: |- + customFields defines additional custom fields for notification. + These provide extra metadata that will be included with the VictorOps incident. items: description: KeyValue defines a (key, value) tuple. properties: key: - description: Key of the tuple. + description: |- + key defines the key of the tuple. + This is the identifier or name part of the key-value pair. minLength: 1 type: string value: - description: Value of the tuple. + description: |- + value defines the value of the tuple. + This is the data or content associated with the key. type: string required: - key @@ -8582,20 +8928,23 @@ spec: type: object type: array entityDisplayName: - description: Contains summary of the alerted problem. + description: |- + entityDisplayName contains a summary of the alerted problem. + This appears as the main title or identifier for the incident. type: string httpConfig: - description: The HTTP client's configuration. + description: httpConfig defines the HTTP client's configuration + for VictorOps API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -8620,7 +8969,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -8629,12 +8978,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -8660,7 +9009,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -8687,7 +9036,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -8713,30 +9062,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8759,8 +9113,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -8787,7 +9141,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -8815,12 +9169,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -8856,7 +9210,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8864,33 +9218,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8913,8 +9267,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -8940,12 +9294,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -8968,8 +9322,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -8995,11 +9349,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -9025,7 +9380,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9036,7 +9391,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9046,13 +9401,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -9089,7 +9444,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9097,31 +9452,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9144,8 +9500,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9171,12 +9527,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9199,8 +9555,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9226,11 +9582,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -9255,7 +9612,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9266,7 +9623,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9276,58 +9633,61 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object messageType: - description: Describes the behavior of the alert (CRITICAL, - WARNING, INFO). + description: |- + messageType describes the behavior of the alert. + Valid values are "CRITICAL", "WARNING", and "INFO". type: string monitoringTool: - description: The monitoring tool the state message is - from. + description: |- + monitoringTool defines the monitoring tool the state message is from. + This helps identify the source system that generated the alert. type: string routingKey: - description: A key used to map the alert to a team. + description: |- + routingKey defines a key used to map the alert to a team. + This determines which VictorOps team will receive the alert notification. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean stateMessage: - description: Contains long explanation of the alerted - problem. + description: |- + stateMessage contains a long explanation of the alerted problem. + This provides detailed context about the incident. type: string type: object type: array webexConfigs: - description: List of Webex configurations. + description: webexConfigs defines the list of Webex configurations. items: description: |- WebexConfig configures notification via Cisco Webex See https://prometheus.io/docs/alerting/latest/configuration/#webex_config properties: apiURL: - description: |- - The Webex Teams API URL i.e. https://webexapis.com/v1/messages - Provide if different from the default API URL. + description: apiURL defines the Webex Teams API URL i.e. + https://webexapis.com/v1/messages pattern: ^https?://.+$ type: string httpConfig: - description: |- - The HTTP client's configuration. - You must supply the bot token via the `httpConfig.authorization` field. + description: httpConfig defines the HTTP client's configuration. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -9352,7 +9712,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -9361,12 +9721,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -9392,7 +9752,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -9419,7 +9779,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -9445,30 +9805,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9491,8 +9856,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9519,7 +9884,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -9547,12 +9912,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -9588,7 +9953,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9596,33 +9961,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9645,8 +10010,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -9672,12 +10037,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9700,8 +10065,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -9727,11 +10092,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -9757,7 +10123,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9768,7 +10134,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9778,13 +10144,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -9821,7 +10187,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9829,31 +10195,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9876,8 +10243,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9903,12 +10270,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -9931,8 +10298,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -9958,11 +10325,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -9987,7 +10355,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9998,7 +10366,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10008,45 +10376,47 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: Message template + description: message defines the message template type: string roomID: - description: ID of the Webex Teams room where to send - the messages. + description: roomID defines the ID of the Webex Teams + room where to send the messages. minLength: 1 type: string sendResolved: - description: Whether to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean required: - roomID type: object type: array webhookConfigs: - description: List of webhook configurations. + description: webhookConfigs defines the List of webhook configurations. items: description: |- WebhookConfig configures notifications via a generic receiver supporting the webhook payload. See https://prometheus.io/docs/alerting/latest/configuration/#webhook_config properties: httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for webhook requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -10071,7 +10441,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -10080,12 +10450,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -10111,7 +10481,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -10138,7 +10508,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -10164,30 +10534,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10210,8 +10585,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -10238,7 +10613,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -10266,12 +10641,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -10307,7 +10682,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10315,33 +10690,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10364,8 +10739,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -10391,12 +10766,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10419,8 +10794,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -10446,11 +10821,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -10476,7 +10852,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10487,7 +10863,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10497,13 +10873,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -10540,7 +10916,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10548,31 +10924,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10595,8 +10972,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -10622,12 +10999,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -10650,8 +11027,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -10677,11 +11054,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -10706,7 +11084,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10717,7 +11095,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10727,37 +11105,38 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object maxAlerts: - description: Maximum number of alerts to be sent per webhook - message. When 0, all alerts are included. + description: |- + maxAlerts defines the maximum number of alerts to be sent per webhook message. + When 0, all alerts are included in the webhook payload. format: int32 minimum: 0 type: integer sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean timeout: description: |- - The maximum time to wait for a webhook request to complete, before failing the - request and allowing it to be retried. + timeout defines the maximum time to wait for a webhook request to complete, + before failing the request and allowing it to be retried. It requires Alertmanager >= v0.28.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string url: description: |- - The URL to send HTTP POST requests to. `urlSecret` takes precedence over - `url`. One of `urlSecret` and `url` should be defined. + url defines the URL to send HTTP POST requests to. + urlSecret takes precedence over url. One of urlSecret and url should be defined. type: string urlSecret: description: |- - The secret's key that contains the webhook URL to send HTTP requests to. - `urlSecret` takes precedence over `url`. One of `urlSecret` and `url` - should be defined. + urlSecret defines the secret's key that contains the webhook URL to send HTTP requests to. + urlSecret takes precedence over url. One of urlSecret and url should be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -10785,17 +11164,20 @@ spec: type: object type: array wechatConfigs: - description: List of WeChat configurations. + description: wechatConfigs defines the list of WeChat configurations. items: description: |- WeChatConfig configures notifications via WeChat. See https://prometheus.io/docs/alerting/latest/configuration/#wechat_config properties: agentID: + description: |- + agentID defines the application agent ID within WeChat Work. + This identifies which WeChat Work application will send the notifications. type: string apiSecret: description: |- - The secret's key that contains the WeChat API key. + apiSecret defines the secret's key that contains the WeChat API key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: @@ -10821,23 +11203,28 @@ spec: type: object x-kubernetes-map-type: atomic apiURL: - description: The WeChat API URL. + description: |- + apiURL defines the WeChat API URL. + When not specified, defaults to the standard WeChat Work API endpoint. type: string corpID: - description: The corp id for authentication. + description: |- + corpID defines the corp id for authentication. + This is the unique identifier for your WeChat Work organization. type: string httpConfig: - description: HTTP client configuration. + description: httpConfig defines the HTTP client configuration + for WeChat API requests. properties: authorization: description: |- - Authorization header configuration for the client. + authorization defines the authorization header configuration for the client. This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. properties: credentials: - description: Selects a key of a Secret in the - namespace that contains the credentials for - authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials + for authentication. properties: key: description: The key of the secret to select @@ -10862,7 +11249,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -10871,12 +11258,12 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. + basicAuth defines the basic authentication credentials for the client. This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -10902,7 +11289,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -10929,7 +11316,7 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client + bearerTokenSecret defines the secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. @@ -10955,30 +11342,35 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the - client should follow HTTP 3xx redirects. + description: |- + followRedirects specifies whether the client should follow HTTP 3xx redirects. + When true, the client will automatically follow redirect responses. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch - a token for the targets. + description: |- + oauth2 defines the OAuth2 client credentials used to fetch a token for the targets. + This enables OAuth2 authentication flow for HTTP requests. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11001,8 +11393,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -11029,7 +11421,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -11057,12 +11449,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -11098,7 +11490,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11106,33 +11498,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy - server to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes - used for the token request.' + description: scopes defines the OAuth2 scopes + used for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when - verifying server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11155,8 +11547,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -11182,12 +11574,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present - when doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data - to use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11210,8 +11602,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to - use for the targets. + description: secret defines the Secret + containing data to use for the targets. properties: key: description: The key of the secret @@ -11237,11 +11629,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how + to disable target certificate validation. type: boolean keySecret: - description: Secret containing the client - key file for the targets. + description: keySecret defines the Secret + containing the client key file for the targets. properties: key: description: The key of the secret to @@ -11267,7 +11660,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11278,7 +11671,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11288,13 +11681,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for - the targets. + description: serverName is used to verify + the hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to - fetch the token from.' + description: tokenUrl defines the URL to fetch + the token from. minLength: 1 type: string required: @@ -11331,7 +11724,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11339,31 +11732,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyURL: description: |- - Optional proxy URL. - + proxyURL defines an optional proxy URL for HTTP requests. If defined, this field takes precedence over `proxyUrl`. type: string proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: |- + tlsConfig defines the TLS configuration for the client. + This includes settings for certificates, CA validation, and TLS protocol options. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11386,8 +11780,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -11413,12 +11807,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -11441,8 +11835,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -11468,11 +11862,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -11497,7 +11892,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11508,7 +11903,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11518,25 +11913,39 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object type: object message: - description: API request data as defined by the WeChat - API. + description: |- + message defines the API request data as defined by the WeChat API. + This contains the actual notification content to be sent. type: string messageType: + description: |- + messageType defines the type of message to send. + Valid values include "text", "markdown", and other WeChat Work supported message types. type: string sendResolved: - description: Whether or not to notify about resolved alerts. + description: sendResolved defines whether or not to notify + about resolved alerts. type: boolean toParty: + description: |- + toParty defines the target department(s) to receive the notification. + Can be a single department ID or multiple department IDs separated by '|'. type: string toTag: + description: |- + toTag defines the target tag(s) to receive the notification. + Can be a single tag ID or multiple tag IDs separated by '|'. type: string toUser: + description: |- + toUser defines the target user(s) to receive the notification. + Can be a single user ID or multiple user IDs separated by '|'. type: string type: object type: array @@ -11546,25 +11955,25 @@ spec: type: array route: description: |- - The Alertmanager route definition for alerts matching the resource's + route defines the Alertmanager route definition for alerts matching the resource's namespace. If present, it will be added to the generated Alertmanager configuration as a first-level route. properties: activeTimeIntervals: - description: ActiveTimeIntervals is a list of MuteTimeInterval + description: activeTimeIntervals is a list of MuteTimeInterval names when this route should be active. items: type: string type: array continue: description: |- - Boolean indicating whether an alert should continue matching subsequent + continue defines the boolean indicating whether an alert should continue matching subsequent sibling nodes. It will always be overridden to true for the first-level route by the Prometheus operator. type: boolean groupBy: description: |- - List of labels to group by. + groupBy defines the list of labels to group by. Labels must not be repeated (unique list). Special label "..." (aggregate by all possible labels), if provided, must be the only element in the list. items: @@ -11572,19 +11981,19 @@ spec: type: array groupInterval: description: |- - How long to wait before sending an updated notification. + groupInterval defines how long to wait before sending an updated notification. Must match the regular expression`^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$` Example: "5m" type: string groupWait: description: |- - How long to wait before sending the initial notification. + groupWait defines how long to wait before sending the initial notification. Must match the regular expression`^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$` Example: "30s" type: string matchers: description: |- - List of matchers that the alert's labels should match. For the first + matchers defines the list of matchers that the alert's labels should match. For the first level route, the operator removes any existing equality and regexp matcher on the `namespace` label and adds a `namespace: ` matcher. @@ -11593,8 +12002,9 @@ spec: properties: matchType: description: |- - Match operation available with AlertManager >= v0.22.0 and - takes precedence over Regex (deprecated) if non-empty. + matchType defines the match operation available with AlertManager >= v0.22.0. + Takes precedence over Regex (deprecated) if non-empty. + Valid values: "=" (equality), "!=" (inequality), "=~" (regex match), "!~" (regex non-match). enum: - '!=' - = @@ -11602,47 +12012,44 @@ spec: - '!~' type: string name: - description: Label to match. + description: |- + name defines the label to match. + This specifies which alert label should be evaluated. minLength: 1 type: string regex: description: |- - Whether to match on equality (false) or regular-expression (true). + regex defines whether to match on equality (false) or regular-expression (true). Deprecated: for AlertManager >= v0.22.0, `matchType` should be used instead. type: boolean value: - description: Label value to match. + description: |- + value defines the label value to match. + This is the expected value for the specified label. type: string required: - name type: object type: array muteTimeIntervals: - description: |- - Note: this comment applies to the field definition above but appears - below otherwise it gets included in the generated manifest. - CRD schema doesn't support self-referential types for now (see - https://github.com/kubernetes/kubernetes/issues/62872). We have to use - an alternative type to circumvent the limitation. The downside is that - the Kube API can't validate the data beyond the fact that it is a valid - JSON representation. - MuteTimeIntervals is a list of MuteTimeInterval names that will mute this route when matched, + description: muteTimeIntervals is a list of MuteTimeInterval names + that will mute this route when matched, items: type: string type: array receiver: description: |- - Name of the receiver for this route. If not empty, it should be listed in + receiver defines the name of the receiver for this route. If not empty, it should be listed in the `receivers` field. type: string repeatInterval: description: |- - How long to wait before repeating the last notification. + repeatInterval defines how long to wait before repeating the last notification. Must match the regular expression`^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$` Example: "4h" type: string routes: - description: Child routes. + description: routes defines the child routes. items: x-kubernetes-preserve-unknown-fields: true type: array diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagers.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagers.yaml index 96fad4d0d5d..e57360088a4 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagers.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_alertmanagers.yaml @@ -74,12 +74,12 @@ spec: type: object spec: description: |- - Specification of the desired behavior of the Alertmanager cluster. More info: + spec defines the specification of the desired behavior of the Alertmanager cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the 'Alertmanager' container. + additionalArgs allows setting additional arguments for the 'Alertmanager' container. It is intended for e.g. activating hidden flags which are not supported by the dedicated configuration options yet. The arguments are passed as-is to the Alertmanager container which may cause issues if they are invalid or not supported @@ -88,25 +88,25 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for name-only - arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. Can + be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name type: object type: array additionalPeers: - description: AdditionalPeers allows injecting a set of additional + description: additionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. items: type: string type: array affinity: - description: If specified, the pod's scheduling constraints. + description: affinity defines the pod's scheduling constraints. properties: nodeAffinity: description: Describes node affinity scheduling rules for the @@ -1019,13 +1019,13 @@ spec: type: object alertmanagerConfigMatcherStrategy: description: |- - AlertmanagerConfigMatcherStrategy defines how AlertmanagerConfig objects + alertmanagerConfigMatcherStrategy defines how AlertmanagerConfig objects process incoming alerts. properties: type: default: OnNamespace description: |- - AlertmanagerConfigMatcherStrategyType defines the strategy used by + type defines the strategy used by AlertmanagerConfig objects to match alerts in the routes and inhibition rules. @@ -1038,7 +1038,7 @@ spec: type: object alertmanagerConfigNamespaceSelector: description: |- - Namespaces to be selected for AlertmanagerConfig discovery. If nil, only + alertmanagerConfigNamespaceSelector defines the namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. properties: matchExpressions: @@ -1085,8 +1085,8 @@ spec: type: object x-kubernetes-map-type: atomic alertmanagerConfigSelector: - description: AlertmanagerConfigs to be selected for to merge and configure - Alertmanager with. + description: alertmanagerConfigSelector defines the selector to be + used for to merge and configure Alertmanager with. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. @@ -1133,7 +1133,7 @@ spec: x-kubernetes-map-type: atomic alertmanagerConfiguration: description: |- - alertmanagerConfiguration specifies the configuration of Alertmanager. + alertmanagerConfiguration defines the configuration of Alertmanager. If defined, it takes precedence over the `configSecret` field. @@ -1141,20 +1141,23 @@ spec: in a breaking way. properties: global: - description: Defines the global parameters of the Alertmanager + description: global defines the global parameters of the Alertmanager configuration. properties: httpConfig: - description: HTTP client configuration. + description: httpConfig defines the default HTTP configuration. properties: authorization: description: |- - Authorization header configuration for the client. - This is mutually exclusive with BasicAuth and is only available starting from Alertmanager v0.22+. + authorization configures the Authorization header credentials used by + the client. + + Cannot be set at the same time as `basicAuth`, `bearerTokenSecret` or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret + in the namespace that contains the credentials for + authentication. properties: key: description: The key of the secret to select from. Must @@ -1179,7 +1182,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1188,12 +1191,14 @@ spec: type: object basicAuth: description: |- - BasicAuth for the client. - This is mutually exclusive with Authorization. If both are defined, BasicAuth takes precedence. + basicAuth defines the Basic Authentication credentials used by the + client. + + Cannot be set at the same time as `authorization`, `bearerTokenSecret` or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1219,7 +1224,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1246,10 +1251,14 @@ spec: type: object bearerTokenSecret: description: |- - The secret's key that contains the bearer token to be used by the client - for authentication. - The secret needs to be in the same namespace as the Alertmanager - object and accessible by the Prometheus Operator. + bearerTokenSecret defines a key of a Secret containing the bearer token + used by the client for authentication. The secret needs to be in the + same namespace as the custom resource and readable by the Prometheus + Operator. + + Cannot be set at the same time as `authorization`, `basicAuth` or `oauth2`. + + Deprecated: use `authorization` instead. properties: key: description: The key of the secret to select from. Must @@ -1272,30 +1281,38 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableHttp2: + description: enableHttp2 can be used to disable HTTP2. + type: boolean followRedirects: - description: FollowRedirects specifies whether the client - should follow HTTP 3xx redirects. + description: |- + followRedirects defines whether the client should follow HTTP 3xx + redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: OAuth2 client credentials used to fetch a - token for the targets. + description: |- + oauth2 defines the OAuth2 settings used by the client. + + It requires Prometheus >= 2.27.0. + + Cannot be set at the same time as `authorization`, `basicAuth` or `bearerTokenSecret`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1318,8 +1335,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1345,7 +1362,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -1373,12 +1390,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1413,7 +1430,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1421,33 +1438,33 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server + to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used - for the token request.' + description: scopes defines the OAuth2 scopes used + for the token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1470,8 +1487,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1497,12 +1514,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when - doing client-authentication. + description: cert defines the Client certificate + to present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to - use for the targets. + description: configMap defines the ConfigMap + containing data to use for the targets. properties: key: description: The key to select. @@ -1525,8 +1542,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use - for the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to @@ -1552,11 +1569,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to + disable target certificate validation. type: boolean keySecret: - description: Secret containing the client key - file for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select @@ -1581,7 +1599,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1592,7 +1610,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1602,13 +1620,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the - targets. + description: serverName is used to verify the + hostname for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch - the token from.' + description: tokenUrl defines the URL to fetch the + token from. minLength: 1 type: string required: @@ -1645,7 +1663,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1653,25 +1671,26 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server - to use.' + description: proxyUrl defines the HTTP proxy server to + use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS configuration for the client. + description: tlsConfig defines the TLS configuration used + by the client. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1694,8 +1713,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1720,12 +1739,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1748,8 +1767,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1774,11 +1793,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file - for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1803,7 +1823,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1814,7 +1834,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1824,23 +1844,25 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object type: object jira: - description: The default configuration for Jira. + description: jira defines the default configuration for Jira. properties: apiURL: description: |- - The default Jira API URL. + apiURL defines the default Jira API URL. It requires Alertmanager >= v0.28.0. pattern: ^(http|https)://.+$ type: string type: object opsGenieApiKey: - description: The default OpsGenie API Key. + description: opsGenieApiKey defines the default OpsGenie API + Key. properties: key: description: The key of the secret to select from. Must @@ -1864,7 +1886,8 @@ spec: type: object x-kubernetes-map-type: atomic opsGenieApiUrl: - description: The default OpsGenie API URL. + description: opsGenieApiUrl defines the default OpsGenie API + URL. properties: key: description: The key of the secret to select from. Must @@ -1888,28 +1911,30 @@ spec: type: object x-kubernetes-map-type: atomic pagerdutyUrl: - description: The default Pagerduty URL. + description: pagerdutyUrl defines the default Pagerduty URL. + pattern: ^(http|https)://.+$ type: string resolveTimeout: description: |- - ResolveTimeout is the default value used by alertmanager if the alert does + resolveTimeout defines the default value used by alertmanager if the alert does not include EndsAt, after this time passes it can declare the alert as resolved if it has not been updated. This has no impact on alerts from Prometheus, as they always include EndsAt. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string rocketChat: - description: The default configuration for Rocket Chat. + description: rocketChat defines the default configuration + for Rocket Chat. properties: apiURL: description: |- - The default Rocket Chat API URL. + apiURL defines the default Rocket Chat API URL. It requires Alertmanager >= v0.28.0. pattern: ^(http|https)://.+$ type: string token: description: |- - The default Rocket Chat token. + token defines the default Rocket Chat token. It requires Alertmanager >= v0.28.0. properties: @@ -1936,7 +1961,7 @@ spec: x-kubernetes-map-type: atomic tokenID: description: |- - The default Rocket Chat Token ID. + tokenID defines the default Rocket Chat Token ID. It requires Alertmanager >= v0.28.0. properties: @@ -1963,7 +1988,7 @@ spec: x-kubernetes-map-type: atomic type: object slackApiUrl: - description: The default Slack API URL. + description: slackApiUrl defines the default Slack API URL. properties: key: description: The key of the secret to select from. Must @@ -1987,13 +2012,14 @@ spec: type: object x-kubernetes-map-type: atomic smtp: - description: Configures global SMTP parameters. + description: smtp defines global SMTP parameters. properties: authIdentity: - description: SMTP Auth using PLAIN + description: authIdentity represents SMTP Auth using PLAIN type: string authPassword: - description: SMTP Auth using LOGIN and PLAIN. + description: authPassword represents SMTP Auth using LOGIN + and PLAIN. properties: key: description: The key of the secret to select from. Must @@ -2017,7 +2043,7 @@ spec: type: object x-kubernetes-map-type: atomic authSecret: - description: SMTP Auth using CRAM-MD5. + description: authSecret represents SMTP Auth using CRAM-MD5. properties: key: description: The key of the secret to select from. Must @@ -2041,34 +2067,35 @@ spec: type: object x-kubernetes-map-type: atomic authUsername: - description: SMTP Auth using CRAM-MD5, LOGIN and PLAIN. - If empty, Alertmanager doesn't authenticate to the SMTP - server. + description: authUsername represents SMTP Auth using CRAM-MD5, + LOGIN and PLAIN. If empty, Alertmanager doesn't authenticate + to the SMTP server. type: string from: - description: The default SMTP From header field. + description: from defines the default SMTP From header + field. type: string hello: - description: The default hostname to identify to the SMTP - server. + description: hello defines the default hostname to identify + to the SMTP server. type: string requireTLS: description: |- - The default SMTP TLS requirement. + requireTLS defines the default SMTP TLS requirement. Note that Go does not support unencrypted connections to remote SMTP endpoints. type: boolean smartHost: - description: The default SMTP smarthost used for sending - emails. + description: smartHost defines the default SMTP smarthost + used for sending emails. properties: host: - description: Defines the host's address, it can be - a DNS name or a literal IP address. + description: host defines the host's address, it can + be a DNS name or a literal IP address. minLength: 1 type: string port: - description: Defines the host's port, it can be a - literal port number or a port name. + description: port defines the host's port, it can + be a literal port number or a port name. minLength: 1 type: string required: @@ -2076,15 +2103,16 @@ spec: - port type: object tlsConfig: - description: The default TLS configuration for SMTP receivers + description: tlsConfig defines the default TLS configuration + for SMTP receivers properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority + used when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2107,8 +2135,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -2133,12 +2161,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use - for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2161,8 +2189,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for - the targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -2187,11 +2215,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file - for the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2216,7 +2245,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2227,7 +2256,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2237,26 +2266,29 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object type: object telegram: - description: The default Telegram config + description: telegram defines the default Telegram config properties: apiURL: description: |- - The default Telegram API URL. + apiURL defines he default Telegram API URL. It requires Alertmanager >= v0.24.0. pattern: ^(http|https)://.+$ type: string type: object victorops: - description: The default configuration for VictorOps. + description: victorops defines the default configuration for + VictorOps. properties: apiKey: - description: The default VictorOps API Key. + description: apiKey defines the default VictorOps API + Key. properties: key: description: The key of the secret to select from. Must @@ -2280,30 +2312,33 @@ spec: type: object x-kubernetes-map-type: atomic apiURL: - description: The default VictorOps API URL. + description: apiURL defines the default VictorOps API + URL. pattern: ^(http|https)://.+$ type: string type: object webex: - description: The default configuration for Jira. + description: webex defines the default configuration for Jira. properties: apiURL: description: |- - The default Webex API URL. + apiURL defines the is the default Webex API URL. It requires Alertmanager >= v0.25.0. pattern: ^(http|https)://.+$ type: string type: object wechat: - description: The default WeChat Config + description: wechat defines the default WeChat Config properties: apiCorpID: - description: The default WeChat API Corporate ID. + description: apiCorpID defines the default WeChat API + Corporate ID. minLength: 1 type: string apiSecret: - description: The default WeChat API Secret. + description: apiSecret defines the default WeChat API + Secret. properties: key: description: The key of the secret to select from. Must @@ -2328,7 +2363,7 @@ spec: x-kubernetes-map-type: atomic apiURL: description: |- - The default WeChat API URL. + apiURL defines he default WeChat API URL. The default value is "https://qyapi.weixin.qq.com/cgi-bin/" pattern: ^(http|https)://.+$ type: string @@ -2336,19 +2371,20 @@ spec: type: object name: description: |- - The name of the AlertmanagerConfig resource which is used to generate the Alertmanager configuration. + name defines the name of the AlertmanagerConfig custom resource which is used to generate the Alertmanager configuration. It must be defined in the same namespace as the Alertmanager object. The operator will not enforce a `namespace` label for routes and inhibition rules. minLength: 1 type: string templates: - description: Custom notification templates. + description: templates defines the custom notification templates. items: description: SecretOrConfigMap allows to specify data as a Secret or ConfigMap. Fields are mutually exclusive. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2371,7 +2407,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2399,53 +2436,56 @@ spec: type: object automountServiceAccountToken: description: |- - AutomountServiceAccountToken indicates whether a service account token should be automatically mounted in the pod. + automountServiceAccountToken defines whether a service account token should be automatically mounted in the pod. If the service account has `automountServiceAccountToken: true`, set the field to `false` to opt out of automounting API credentials. type: boolean baseImage: description: |- - Base image that is used to deploy pods, without tag. + baseImage that is used to deploy pods, without tag. Deprecated: use 'image' instead. type: string clusterAdvertiseAddress: description: |- - ClusterAdvertiseAddress is the explicit address to advertise in cluster. + clusterAdvertiseAddress defines the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 type: string clusterGossipInterval: - description: Interval between gossip attempts. + description: clusterGossipInterval defines the interval between gossip + attempts. pattern: ^(0|(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string clusterLabel: description: |- - Defines the identifier that uniquely identifies the Alertmanager cluster. + clusterLabel defines the identifier that uniquely identifies the Alertmanager cluster. You should only set it when the Alertmanager cluster includes Alertmanager instances which are external to this Alertmanager resource. In practice, the addresses of the external instances are provided via the `.spec.additionalPeers` field. type: string clusterPeerTimeout: - description: Timeout for cluster peering. + description: clusterPeerTimeout defines the timeout for cluster peering. pattern: ^(0|(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string clusterPushpullInterval: - description: Interval between pushpull attempts. + description: clusterPushpullInterval defines the interval between + pushpull attempts. pattern: ^(0|(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string clusterTLS: description: |- - Configures the mutual TLS configuration for the Alertmanager cluster's gossip protocol. + clusterTLS defines the mutual TLS configuration for the Alertmanager cluster's gossip protocol. It requires Alertmanager >= 0.24.0. properties: client: - description: Client-side configuration for mutual TLS. + description: client defines the client-side configuration for + mutual TLS. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2468,7 +2508,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2493,11 +2534,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2520,7 +2562,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2545,11 +2588,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2574,7 +2618,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2585,7 +2629,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2595,23 +2639,25 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object server: - description: Server-side configuration for mutual TLS. + description: server defines the server-side configuration for + mutual TLS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2634,7 +2680,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2660,7 +2707,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -2668,7 +2715,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -2678,14 +2725,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2708,7 +2755,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2734,21 +2782,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -2758,7 +2806,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -2766,7 +2814,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -2794,14 +2842,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -2814,7 +2864,7 @@ spec: type: object configMaps: description: |- - ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager + configMaps defines a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. Each ConfigMap is added to the StatefulSet definition as a volume named `configmap-`. The ConfigMaps are mounted into `/etc/alertmanager/configmaps/` in the 'alertmanager' container. @@ -2823,7 +2873,7 @@ spec: type: array configSecret: description: |- - ConfigSecret is the name of a Kubernetes Secret in the same namespace as the + configSecret defines the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains the configuration for this Alertmanager instance. If empty, it defaults to `alertmanager-`. @@ -2838,7 +2888,7 @@ spec: type: string containers: description: |- - Containers allows injecting additional containers. This is meant to + containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge @@ -4366,11 +4416,11 @@ spec: type: object type: array dnsConfig: - description: Defines the DNS configuration for the pods. + description: dnsConfig defines the DNS configuration for the pods. properties: nameservers: description: |- - A list of DNS name server IP addresses. + nameservers defines the list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. items: minLength: 1 @@ -4379,7 +4429,7 @@ spec: x-kubernetes-list-type: set options: description: |- - A list of DNS resolver options. + options defines the list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Resolution options given in Options will override those that appear in the base DNSPolicy. @@ -4388,11 +4438,11 @@ spec: of a pod. properties: name: - description: Name is required and must be unique. + description: name is required and must be unique. minLength: 1 type: string value: - description: Value is optional. + description: value is optional. type: string required: - name @@ -4403,7 +4453,7 @@ spec: x-kubernetes-list-type: map searches: description: |- - A list of DNS search domains for host-name lookup. + searches defines the list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. items: minLength: 1 @@ -4412,7 +4462,7 @@ spec: x-kubernetes-list-type: set type: object dnsPolicy: - description: Defines the DNS policy for the pods. + description: dnsPolicy defines the DNS policy for the pods. enum: - ClusterFirstWithHostNet - ClusterFirst @@ -4421,7 +4471,7 @@ spec: type: string enableFeatures: description: |- - Enable access to Alertmanager feature flags. By default, no features are enabled. + enableFeatures defines the Alertmanager's feature flags. By default, no features are enabled. Enabling features which are disabled by default is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice. @@ -4431,34 +4481,34 @@ spec: type: string type: array enableServiceLinks: - description: Indicates whether information about services should be - injected into pod's environment variables + description: enableServiceLinks defines whether information about + services should be injected into pod's environment variables type: boolean externalUrl: description: |- - The external URL the Alertmanager instances will be available under. This is + externalUrl defines the URL used to access the Alertmanager web service. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. type: string forceEnableClusterMode: description: |- - ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + forceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. type: boolean hostAliases: - description: Pods' hostAliases configuration + description: hostAliases Pods configuration items: description: |- HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. + description: hostnames defines hostnames for the above IP address. items: type: string type: array ip: - description: IP address of the host file entry. + description: ip defines the IP address of the host file entry. type: string required: - hostnames @@ -4470,7 +4520,7 @@ spec: x-kubernetes-list-type: map hostUsers: description: |- - HostUsers supports the user space in Kubernetes. + hostUsers supports the user space in Kubernetes. More info: https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/ @@ -4479,14 +4529,14 @@ spec: type: boolean image: description: |- - Image if specified has precedence over baseImage, tag and sha + image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Alertmanager is being configured. type: string imagePullPolicy: description: |- - Image pull policy for the 'alertmanager', 'init-config-reloader' and 'config-reloader' containers. + imagePullPolicy for the 'alertmanager', 'init-config-reloader' and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. enum: - "" @@ -4496,7 +4546,7 @@ spec: type: string imagePullSecrets: description: |- - An optional list of references to secrets in the same namespace + imagePullSecrets An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ items: @@ -4518,7 +4568,7 @@ spec: type: array initContainers: description: |- - InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. + initContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the Alertmanager configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ InitContainers described here modify an operator @@ -6047,18 +6097,19 @@ spec: type: object type: array limits: - description: Defines the limits command line flags when starting Alertmanager. + description: limits defines the limits command line flags when starting + Alertmanager. properties: maxPerSilenceBytes: description: |- - The maximum size of an individual silence as stored on disk. This corresponds to the Alertmanager's + maxPerSilenceBytes defines the maximum size of an individual silence as stored on disk. This corresponds to the Alertmanager's `--silences.max-per-silence-bytes` flag. It requires Alertmanager >= v0.28.0. pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ type: string maxSilences: description: |- - The maximum number active and pending silences. This corresponds to the + maxSilences defines the maximum number active and pending silences. This corresponds to the Alertmanager's `--silences.max-silences` flag. It requires Alertmanager >= v0.28.0. format: int32 @@ -6067,19 +6118,19 @@ spec: type: object listenLocal: description: |- - ListenLocal makes the Alertmanager server listen on loopback, so that it + listenLocal defines the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. Note this is only for the Alertmanager UI, not the gossip communication. type: boolean logFormat: - description: Log format for Alertmanager to be configured with. + description: logFormat for Alertmanager to be configured with. enum: - "" - logfmt - json type: string logLevel: - description: Log level for Alertmanager to be configured with. + description: logLevel for Alertmanager to be configured with. enum: - "" - debug @@ -6089,7 +6140,7 @@ spec: type: string minReadySeconds: description: |- - Minimum number of seconds for which a newly created pod should be ready + minReadySeconds defines the minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. If unset, pods will be considered available as soon as they are ready. @@ -6099,16 +6150,17 @@ spec: nodeSelector: additionalProperties: type: string - description: Define which Nodes the Pods are scheduled on. + description: nodeSelector defines which Nodes the Pods are scheduled + on. type: object paused: description: |- - If set to true all actions on the underlying managed objects are not + paused if set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. type: boolean persistentVolumeClaimRetentionPolicy: description: |- - The field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. + persistentVolumeClaimRetentionPolicy controls if and how PVCs are deleted during the lifecycle of a StatefulSet. The default behavior is all PVCs are retained. This is an alpha field from kubernetes 1.23 until 1.26 and a beta field from 1.26. It requires enabling the StatefulSetAutoDeletePVC feature gate. @@ -6131,7 +6183,7 @@ spec: type: object podMetadata: description: |- - PodMetadata configures labels and annotations which are propagated to the Alertmanager pods. + podMetadata defines labels and annotations which are propagated to the Alertmanager pods. The following items are reserved and cannot be overridden: * "alertmanager" label, set to the name of the Alertmanager instance. @@ -6145,7 +6197,7 @@ spec: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -6154,14 +6206,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -6172,21 +6224,22 @@ spec: portName: default: web description: |- - Port name used for the pods and governing service. + portName defines the port's name for the pods and governing service. Defaults to `web`. type: string priorityClassName: - description: Priority class assigned to the Pods + description: priorityClassName assigned to the Pods type: string replicas: description: |- - Size is the expected size of the alertmanager cluster. The controller will + replicas defines the expected size of the alertmanager cluster. The controller will eventually make the size of the running cluster equal to the expected size. format: int32 type: integer resources: - description: Define resources requests and limits for single Pods. + description: resources defines the resource requests and limits of + the Pods. properties: claims: description: |- @@ -6247,20 +6300,20 @@ spec: retention: default: 120h description: |- - Time duration Alertmanager shall retain data for. Default is '120h', + retention defines the time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours). pattern: ^(0|(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string routePrefix: description: |- - The route prefix Alertmanager registers HTTP handlers for. This is useful, + routePrefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`. type: string secrets: description: |- - Secrets is a list of Secrets in the same namespace as the Alertmanager + secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. Each Secret is added to the StatefulSet definition as a volume named `secret-`. The Secrets are mounted into `/etc/alertmanager/secrets/` in the 'alertmanager' container. @@ -6269,7 +6322,7 @@ spec: type: array securityContext: description: |- - SecurityContext holds pod-level security attributes and common container settings. + securityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: appArmorProfile: @@ -6503,37 +6556,37 @@ spec: type: object serviceAccountName: description: |- - ServiceAccountName is the name of the ServiceAccount to use to run the + serviceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods. type: string serviceName: description: |- - The name of the service name used by the underlying StatefulSet(s) as the governing service. + serviceName defines the service name used by the underlying StatefulSet(s) as the governing service. If defined, the Service must be created before the Alertmanager resource in the same namespace and it must define a selector that matches the pod labels. - If empty, the operator will create and manage a headless service named `alertmanager-operated` for Alermanager resources. + If empty, the operator will create and manage a headless service named `alertmanager-operated` for Alertmanager resources. When deploying multiple Alertmanager resources in the same namespace, it is recommended to specify a different value for each. See https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id for more details. minLength: 1 type: string sha: description: |- - SHA of Alertmanager container image to be deployed. Defaults to the value of `version`. + sha of Alertmanager container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set. Deprecated: use 'image' instead. The image digest can be specified as part of the image URL. type: string storage: description: |- - Storage is the definition of how storage will be used by the Alertmanager + storage defines the definition of how storage will be used by the Alertmanager instances. properties: disableMountSubPath: - description: 'Deprecated: subPath usage will be removed in a future - release.' + description: 'disableMountSubPath deprecated: subPath usage will + be removed in a future release.' type: boolean emptyDir: description: |- - EmptyDirVolumeSource to be used by the StatefulSet. + emptyDir to be used by the StatefulSet. If specified, it takes precedence over `ephemeral` and `volumeClaimTemplate`. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir properties: @@ -6560,7 +6613,7 @@ spec: type: object ephemeral: description: |- - EphemeralVolumeSource to be used by the StatefulSet. + ephemeral to be used by the StatefulSet. This is a beta field in k8s 1.21 and GA in 1.15. For lower versions, starting with k8s 1.19, it requires enabling the GenericEphemeralVolume feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes @@ -6805,7 +6858,7 @@ spec: type: object volumeClaimTemplate: description: |- - Defines the PVC spec to be used by the Prometheus StatefulSets. + volumeClaimTemplate defines the PVC spec to be used by the Prometheus StatefulSets. The easiest way to use a volume that cannot be automatically provisioned is to use a label selector alongside manually created PersistentVolumes. properties: @@ -6825,14 +6878,14 @@ spec: More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: - description: EmbeddedMetadata contains metadata relevant to - an EmbeddedResource. + description: metadata defines EmbeddedMetadata contains metadata + relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -6841,14 +6894,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -6858,7 +6911,7 @@ spec: type: object spec: description: |- - Defines the desired characteristics of a volume requested by a pod author. + spec defines the specification of the characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: accessModes: @@ -7054,7 +7107,7 @@ spec: type: string type: object status: - description: 'Deprecated: this field is never set.' + description: 'status is deprecated: this field is never set.' properties: accessModes: description: |- @@ -7239,13 +7292,13 @@ spec: type: object tag: description: |- - Tag of Alertmanager container image to be deployed. Defaults to the value of `version`. + tag of Alertmanager container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set. Deprecated: use 'image' instead. The image tag can be specified as part of the image URL. type: string terminationGracePeriodSeconds: description: |- - Optional duration in seconds the pod needs to terminate gracefully. + terminationGracePeriodSeconds defines the Optional duration in seconds the pod needs to terminate gracefully. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down) which may lead to data corruption. @@ -7254,7 +7307,7 @@ spec: minimum: 0 type: integer tolerations: - description: If specified, the pod's tolerations. + description: tolerations defines the pod's tolerations. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -7293,7 +7346,8 @@ spec: type: object type: array topologySpreadConstraints: - description: If specified, the pod's topology spread constraints. + description: topologySpreadConstraints defines the Pod's topology + spread constraints. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. @@ -7467,11 +7521,11 @@ spec: type: object type: array version: - description: Version the cluster should be on. + description: version the cluster should be on. type: string volumeMounts: description: |- - VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. + volumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the alertmanager container, that are generated as a result of StorageSpec objects. items: @@ -7538,7 +7592,7 @@ spec: type: array volumes: description: |- - Volumes allows configuration of additional volumes on the output StatefulSet definition. + volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: @@ -9424,28 +9478,30 @@ spec: type: object type: array web: - description: Defines the web command line flags when starting Alertmanager. + description: web defines the web command line flags when starting + Alertmanager. properties: getConcurrency: description: |- - Maximum number of GET requests processed concurrently. This corresponds to the + getConcurrency defines the maximum number of GET requests processed concurrently. This corresponds to the Alertmanager's `--web.get-concurrency` flag. format: int32 type: integer httpConfig: - description: Defines HTTP parameters for web server. + description: httpConfig defines HTTP parameters for web server. properties: headers: - description: List of headers that can be added to HTTP responses. + description: headers defines a list of headers that can be + added to HTTP responses. properties: contentSecurityPolicy: description: |- - Set the Content-Security-Policy header to HTTP responses. + contentSecurityPolicy defines the Content-Security-Policy header to HTTP responses. Unset if blank. type: string strictTransportSecurity: description: |- - Set the Strict-Transport-Security header to HTTP responses. + strictTransportSecurity defines the Strict-Transport-Security header to HTTP responses. Unset if blank. Please make sure that you use this with care as this header might force browsers to load Prometheus and the other applications hosted on the same @@ -9454,7 +9510,7 @@ spec: type: string xContentTypeOptions: description: |- - Set the X-Content-Type-Options header to HTTP responses. + xContentTypeOptions defines the X-Content-Type-Options header to HTTP responses. Unset if blank. Accepted value is nosniff. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options enum: @@ -9463,7 +9519,7 @@ spec: type: string xFrameOptions: description: |- - Set the X-Frame-Options header to HTTP responses. + xFrameOptions defines the X-Frame-Options header to HTTP responses. Unset if blank. Accepted values are deny and sameorigin. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options enum: @@ -9473,38 +9529,38 @@ spec: type: string xXSSProtection: description: |- - Set the X-XSS-Protection header to all responses. + xXSSProtection defines the X-XSS-Protection header to all responses. Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection type: string type: object http2: description: |- - Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. + http2 enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. When TLSConfig is not configured, HTTP/2 will be disabled. Whenever the value of the field changes, a rolling update will be triggered. type: boolean type: object timeout: description: |- - Timeout for HTTP requests. This corresponds to the Alertmanager's + timeout for HTTP requests. This corresponds to the Alertmanager's `--web.timeout` flag. format: int32 type: integer tlsConfig: - description: Defines the TLS parameters for HTTPS. + description: tlsConfig defines the TLS parameters for HTTPS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9527,7 +9583,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9553,7 +9610,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -9561,7 +9618,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -9571,14 +9628,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9601,7 +9658,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9627,21 +9685,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -9651,7 +9709,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -9659,7 +9717,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -9687,14 +9745,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -9705,18 +9765,19 @@ spec: type: object status: description: |- - Most recent observed status of the Alertmanager cluster. Read-only. + status defines the most recent observed status of the Alertmanager cluster. Read-only. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this Alertmanager cluster. format: int32 type: integer conditions: - description: The current state of the Alertmanager object. + description: conditions defines the current state of the Alertmanager + object. items: description: |- Condition represents the state of the resources associated with the @@ -9728,12 +9789,12 @@ spec: format: date-time type: string message: - description: Human-readable message indicating details for the - condition's last transition. + description: message defines human-readable message indicating + details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the @@ -9741,14 +9802,14 @@ spec: format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: - description: Type of the condition being reported. + description: type of the condition being reported. minLength: 1 type: string required: @@ -9762,36 +9823,30 @@ spec: x-kubernetes-list-type: map paused: description: |- - Represents whether any actions on the underlying managed objects are + paused defines whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: |- - Total number of non-terminated pods targeted by this Alertmanager + replicas defines the total number of non-terminated pods targeted by this Alertmanager object (their labels match the selector). format: int32 type: integer selector: - description: The selector used to match the pods targeted by this - Alertmanager object. + description: selector used to match the pods targeted by this Alertmanager + object. type: string unavailableReplicas: - description: Total number of unavailable pods targeted by this Alertmanager - object. + description: unavailableReplicas defines the total number of unavailable + pods targeted by this Alertmanager object. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this Alertmanager + updatedReplicas defines the total number of non-terminated pods targeted by this Alertmanager object that have the desired version spec. format: int32 type: integer - required: - - availableReplicas - - paused - - replicas - - unavailableReplicas - - updatedReplicas type: object required: - spec diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_podmonitors.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_podmonitors.yaml index c888967e7c4..638e9a8c77c 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_podmonitors.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_podmonitors.yaml @@ -49,19 +49,19 @@ spec: metadata: type: object spec: - description: Specification of desired Pod selection for target discovery - by Prometheus. + description: spec defines the specification of desired Pod selection for + target discovery by Prometheus. properties: attachMetadata: description: |- - `attachMetadata` defines additional metadata which is added to the + attachMetadata defines additional metadata which is added to the discovered targets. It requires Prometheus >= v2.35.0. properties: node: description: |- - When set to true, Prometheus attaches node metadata to the discovered + node when set to true, Prometheus attaches node metadata to the discovered targets. The Prometheus service account must have the `list` and `watch` @@ -70,7 +70,7 @@ spec: type: object bodySizeLimit: description: |- - When defined, bodySizeLimit specifies a job level limit on the size + bodySizeLimit when defined specifies a job level limit on the size of uncompressed response body that will be accepted by Prometheus. It requires Prometheus >= v2.28.0. @@ -78,12 +78,12 @@ spec: type: string convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native histogram with custom buckets. + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.0.0. type: boolean fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It requires Prometheus >= v3.0.0. enum: @@ -95,7 +95,7 @@ spec: type: string jobLabel: description: |- - The label to use to retrieve the job name from. + jobLabel defines the label to use to retrieve the job name from. `jobLabel` selects the label from the associated Kubernetes `Pod` object which will be used as the `job` label for all metrics. @@ -108,7 +108,7 @@ spec: type: string keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -116,44 +116,45 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines the per-scrape limit on number of labels that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer namespaceSelector: description: |- - `namespaceSelector` defines in which namespace(s) Prometheus should discover the pods. + namespaceSelector defines in which namespace(s) Prometheus should discover the pods. By default, the pods are discovered in the same namespace as the `PodMonitor` object but it is possible to select pods across different/all namespaces. properties: any: description: |- - Boolean describing whether all namespaces are selected in contrast to a + any defines the boolean describing whether all namespaces are selected in contrast to a list restricting them. type: boolean matchNames: - description: List of namespace names to select from. + description: matchNames defines the list of namespace names to + select from. items: type: string type: array type: object nativeHistogramBucketLimit: description: |- - If there are more than this many buckets in a native histogram, + nativeHistogramBucketLimit defines ff there are more than this many buckets in a native histogram, buckets will be merged to stay within the limit. It requires Prometheus >= v2.45.0. format: int64 @@ -163,13 +164,14 @@ spec: - type: integer - type: string description: |- - If the growth factor of one bucket to the next is smaller than this, + nativeHistogramMinBucketFactor defines if the growth factor of one bucket to the next is smaller than this, buckets will be merged to increase the factor sufficiently. It requires Prometheus >= v2.50.0. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true podMetricsEndpoints: - description: Defines how to scrape metrics from the selected pods. + description: podMetricsEndpoints defines how to scrape metrics from + the selected pods. items: description: |- PodMetricsEndpoint defines an endpoint serving Prometheus metrics to be scraped by @@ -177,14 +179,14 @@ spec: properties: authorization: description: |- - `authorization` configures the Authorization header credentials to use when - scraping the target. + authorization configures the Authorization header credentials used by + the client. - Cannot be set at the same time as `basicAuth`, or `oauth2`. + Cannot be set at the same time as `basicAuth`, `bearerTokenSecret` or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -209,7 +211,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -218,14 +220,14 @@ spec: type: object basicAuth: description: |- - `basicAuth` configures the Basic Authentication credentials to use when - scraping the target. + basicAuth defines the Basic Authentication credentials used by the + client. - Cannot be set at the same time as `authorization`, or `oauth2`. + Cannot be set at the same time as `authorization`, `bearerTokenSecret` or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -251,7 +253,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -278,9 +280,12 @@ spec: type: object bearerTokenSecret: description: |- - `bearerTokenSecret` specifies a key of a Secret containing the bearer - token for scraping targets. The secret needs to be in the same namespace - as the PodMonitor object and readable by the Prometheus Operator. + bearerTokenSecret defines a key of a Secret containing the bearer token + used by the client for authentication. The secret needs to be in the + same namespace as the custom resource and readable by the Prometheus + Operator. + + Cannot be set at the same time as `authorization`, `basicAuth` or `oauth2`. Deprecated: use `authorization` instead. properties: @@ -306,12 +311,11 @@ spec: type: object x-kubernetes-map-type: atomic enableHttp2: - description: '`enableHttp2` can be used to disable HTTP2 when - scraping the target.' + description: enableHttp2 can be used to disable HTTP2. type: boolean filterRunning: description: |- - When true, the pods which are not running (e.g. either in Failed or + filterRunning when true, the pods which are not running (e.g. either in Failed or Succeeded state) are dropped during the target discovery. If unset, the filtering is enabled. @@ -320,29 +324,29 @@ spec: type: boolean followRedirects: description: |- - `followRedirects` defines whether the scrape requests should follow HTTP - 3xx redirects. + followRedirects defines whether the client should follow HTTP 3xx + redirects. type: boolean honorLabels: description: |- - When true, `honorLabels` preserves the metric's labels when they collide + honorLabels when true preserves the metric's labels when they collide with the target's labels. type: boolean honorTimestamps: description: |- - `honorTimestamps` controls whether Prometheus preserves the timestamps + honorTimestamps defines whether Prometheus preserves the timestamps when exposed by the target. type: boolean interval: description: |- - Interval at which Prometheus scrapes the metrics from the target. + interval at which Prometheus scrapes the metrics from the target. If empty, Prometheus uses the global scrape interval. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string metricRelabelings: description: |- - `metricRelabelings` configures the relabeling rules to apply to the + metricRelabelings defines the relabeling rules to apply to the samples before ingestion. items: description: |- @@ -354,7 +358,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -386,41 +390,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -431,7 +435,7 @@ spec: type: array noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -439,20 +443,20 @@ spec: type: string oauth2: description: |- - `oauth2` configures the OAuth2 settings to use when scraping the target. + oauth2 defines the OAuth2 settings used by the client. It requires Prometheus >= 2.27.0. - Cannot be set at the same time as `authorization`, or `basicAuth`. + Cannot be set at the same time as `authorization`, `basicAuth` or `bearerTokenSecret`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -475,7 +479,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -501,7 +506,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -529,12 +534,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -568,7 +573,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -576,33 +581,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -625,8 +629,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -651,12 +655,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -679,8 +683,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -705,11 +709,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -734,7 +739,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -745,7 +750,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -755,12 +760,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -773,22 +779,23 @@ spec: items: type: string type: array - description: '`params` define optional HTTP URL parameters.' + description: params define optional HTTP URL parameters. type: object path: description: |- - HTTP path from which to scrape for metrics. + path defines the HTTP path from which to scrape for metrics. If empty, Prometheus uses the default value (e.g. `/metrics`). type: string port: description: |- - The `Pod` port name which exposes the endpoint. + port defines the `Pod` port name which exposes the endpoint. It takes precedence over the `portNumber` and `targetPort` fields. type: string portNumber: - description: The `Pod` port number which exposes the endpoint. + description: portNumber defines the `Pod` port number which + exposes the endpoint. format: int32 maximum: 65535 minimum: 1 @@ -821,7 +828,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -829,17 +836,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string relabelings: description: |- - `relabelings` configures the relabeling rules to apply the target's + relabelings defines the relabeling rules to apply the target's metadata labels. The Operator automatically adds relabelings for a few standard Kubernetes fields. @@ -857,7 +864,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -889,41 +896,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -934,7 +941,7 @@ spec: type: array scheme: description: |- - HTTP scheme to use for scraping. + scheme defines the HTTP scheme to use for scraping. `http` and `https` are the expected values unless you rewrite the `__scheme__` label via relabeling. @@ -946,7 +953,7 @@ spec: type: string scrapeTimeout: description: |- - Timeout after which Prometheus considers the scrape to be failed. + scrapeTimeout defines the timeout after which Prometheus considers the scrape to be failed. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used. @@ -958,21 +965,22 @@ spec: - type: integer - type: string description: |- - Name or number of the target port of the `Pod` object behind the Service, the + targetPort defines the name or number of the target port of the `Pod` object behind the Service, the port must be specified with container port property. Deprecated: use 'port' or 'portNumber' instead. x-kubernetes-int-or-string: true tlsConfig: - description: TLS configuration to use when scraping the target. + description: tlsConfig defines the TLS configuration used by + the client. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -995,7 +1003,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1020,11 +1029,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1047,7 +1057,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1072,11 +1083,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1101,7 +1113,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1112,7 +1124,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1122,12 +1134,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object trackTimestampsStaleness: description: |- - `trackTimestampsStaleness` defines whether Prometheus tracks staleness of + trackTimestampsStaleness defines whether Prometheus tracks staleness of the metrics that have an explicit timestamp present in scraped data. Has no effect if `honorTimestamps` is false. @@ -1137,31 +1150,31 @@ spec: type: array podTargetLabels: description: |- - `podTargetLabels` defines the labels which are transferred from the + podTargetLabels defines the labels which are transferred from the associated Kubernetes `Pod` object onto the ingested metrics. items: type: string type: array sampleLimit: description: |- - `sampleLimit` defines a per-scrape limit on the number of scraped samples + sampleLimit defines a per-scrape limit on the number of scraped samples that will be accepted. format: int64 type: integer scrapeClass: - description: The scrape class to apply. + description: scrapeClass defines the scrape class to apply. minLength: 1 type: string scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. It requires Prometheus >= v2.45.0. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. type: boolean scrapeProtocols: description: |- - `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -1186,8 +1199,8 @@ spec: type: array x-kubernetes-list-type: set selector: - description: Label selector to select the Kubernetes `Pod` objects - to scrape metrics from. + description: selector defines the label selector to select the Kubernetes + `Pod` objects to scrape metrics from. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. @@ -1234,7 +1247,7 @@ spec: x-kubernetes-map-type: atomic selectorMechanism: description: |- - Mechanism used to select the endpoints to scrape. + selectorMechanism defines the mechanism used to select the endpoints to scrape. By default, the selection process relies on relabel configurations to filter the discovered targets. Alternatively, you can opt in for role selectors, which may offer better efficiency in large clusters. Which strategy is best for your use case needs to be carefully evaluated. @@ -1246,15 +1259,120 @@ spec: type: string targetLimit: description: |- - `targetLimit` defines a limit on the number of scraped targets that will + targetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer required: - selector type: object + status: + description: |- + status defines the status subresource. It is under active development and is updated only when the + "StatusForConfigurationResources" feature gate is enabled. + + Most recent observed status of the PodMonitor. Read-only. + More info: + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + bindings: + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. + items: + description: WorkloadBinding is a link between a configuration resource + and a workload resource. + properties: + conditions: + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. + items: + description: ConfigResourceCondition describes the status + of configuration resources linked to Prometheus, PrometheusAgent, + Alertmanager or ThanosRuler. + properties: + lastTransitionTime: + description: lastTransitionTime defines the time of the + last update to the current status property. + format: date-time + type: string + message: + description: message defines the human-readable message + indicating details for the condition's last transition. + type: string + observedGeneration: + description: |- + observedGeneration defines the .metadata.generation that the + condition was set based upon. For instance, if `.metadata.generation` is + currently 12, but the `.status.conditions[].observedGeneration` is 9, the + condition is out of date with respect to the current state of the object. + format: int64 + type: integer + reason: + description: reason for the condition's last transition. + type: string + status: + description: status of the condition. + minLength: 1 + type: string + type: + description: |- + type of the condition being reported. + Currently, only "Accepted" is supported. + enum: + - Accepted + minLength: 1 + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + group: + description: group defines the group of the referenced resource. + enum: + - monitoring.coreos.com + type: string + name: + description: name defines the name of the referenced object. + minLength: 1 + type: string + namespace: + description: namespace defines the namespace of the referenced + object. + minLength: 1 + type: string + resource: + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). + enum: + - prometheuses + - prometheusagents + - thanosrulers + - alertmanagers + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map + type: object required: - spec type: object served: true storage: true + subresources: + status: {} diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_probes.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_probes.yaml index 4ac47e50aa1..40424236be3 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_probes.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_probes.yaml @@ -48,15 +48,15 @@ spec: metadata: type: object spec: - description: Specification of desired Ingress selection for target discovery - by Prometheus. + description: spec defines the specification of desired Ingress selection + for target discovery by Prometheus. properties: authorization: - description: Authorization section for this endpoint + description: authorization section for this endpoint properties: credentials: - description: Selects a key of a Secret in the namespace that contains - the credentials for authentication. + description: credentials defines a key of a Secret in the namespace + that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must be @@ -81,7 +81,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -90,12 +90,12 @@ spec: type: object basicAuth: description: |- - BasicAuth allow an endpoint to authenticate over basic authentication. + basicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -121,7 +121,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -148,7 +148,7 @@ spec: type: object bearerTokenSecret: description: |- - Secret to mount to read bearer token for scraping targets. The secret + bearerTokenSecret defines the secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the probe and accessible by the Prometheus Operator. properties: @@ -174,12 +174,12 @@ spec: x-kubernetes-map-type: atomic convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native histogram with custom buckets. + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.0.0. type: boolean fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It requires Prometheus >= v3.0.0. enum: @@ -191,16 +191,16 @@ spec: type: string interval: description: |- - Interval at which targets are probed using the configured prober. + interval at which targets are probed using the configured prober. If not specified Prometheus' global scrape interval is used. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string jobName: - description: The job name assigned to scraped metrics by default. + description: jobName assigned to scraped metrics by default. type: string keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -208,24 +208,25 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines the per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer metricRelabelings: - description: MetricRelabelConfigs to apply to samples before ingestion. + description: metricRelabelings defines the RelabelConfig to apply + to samples before ingestion. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -236,7 +237,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -268,40 +269,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against which + the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated SourceLabels. + description: separator defines the string between concatenated + SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -312,13 +314,13 @@ spec: type: array module: description: |- - The module to use for probing specifying how to probe the target. + module to use for probing specifying how to probe the target. Example module configuring in the blackbox exporter: https://github.com/prometheus/blackbox_exporter/blob/master/example.yml type: string nativeHistogramBucketLimit: description: |- - If there are more than this many buckets in a native histogram, + nativeHistogramBucketLimit defines ff there are more than this many buckets in a native histogram, buckets will be merged to stay within the limit. It requires Prometheus >= v2.45.0. format: int64 @@ -328,22 +330,23 @@ spec: - type: integer - type: string description: |- - If the growth factor of one bucket to the next is smaller than this, + nativeHistogramMinBucketFactor defines if the growth factor of one bucket to the next is smaller than this, buckets will be merged to increase the factor sufficiently. It requires Prometheus >= v2.50.0. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true oauth2: - description: OAuth2 for the URL. Only valid in Prometheus versions + description: oauth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -366,7 +369,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -392,7 +396,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -420,12 +424,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -459,7 +463,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -467,32 +471,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for the - token request.' + description: scopes defines the OAuth2 scopes used for the token + request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -515,7 +519,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -540,11 +545,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -567,7 +573,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -592,11 +599,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -621,7 +629,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -632,7 +640,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -642,12 +650,12 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the token - from.' + description: tokenUrl defines the URL to fetch the token from. minLength: 1 type: string required: @@ -657,7 +665,7 @@ spec: type: object params: description: |- - The list of HTTP query parameters for the scrape. + params defines the list of HTTP query parameters for the scrape. Please note that the `.spec.module` field takes precedence over the `module` parameter from this list when both are defined. The module name must be added using Module under ProbeSpec. items: @@ -665,11 +673,11 @@ spec: for a Probe. properties: name: - description: The parameter name + description: name defines the parameter name minLength: 1 type: string values: - description: The parameter values + description: values defines the parameter values items: minLength: 1 type: string @@ -685,12 +693,12 @@ spec: x-kubernetes-list-type: map prober: description: |- - Specification for the prober to use for probing targets. + prober defines the specification for the prober to use for probing targets. The prober.URL parameter is required. Targets cannot be probed if left empty. properties: noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -699,7 +707,7 @@ spec: path: default: /probe description: |- - Path to collect metrics from. + path to collect metrics from. Defaults to `/probe`. type: string proxyConnectHeader: @@ -730,7 +738,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -738,17 +746,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scheme: description: |- - HTTP scheme to use for scraping. + scheme defines the HTTP scheme to use for scraping. `http` and `https` are the expected values unless you rewrite the `__scheme__` label via relabeling. If empty, Prometheus uses the default value `http`. enum: @@ -756,30 +764,30 @@ spec: - https type: string url: - description: Mandatory URL of the prober. + description: url defines the mandatory URL of the prober. type: string required: - url type: object sampleLimit: - description: SampleLimit defines per-scrape limit on number of scraped + description: sampleLimit defines per-scrape limit on number of scraped samples that will be accepted. format: int64 type: integer scrapeClass: - description: The scrape class to apply. + description: scrapeClass defines the scrape class to apply. minLength: 1 type: string scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. It requires Prometheus >= v2.45.0. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. type: boolean scrapeProtocols: description: |- - `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -805,18 +813,18 @@ spec: x-kubernetes-list-type: set scrapeTimeout: description: |- - Timeout for scraping metrics from the Prometheus exporter. + scrapeTimeout defines the timeout for scraping metrics from the Prometheus exporter. If not specified, the Prometheus global scrape timeout is used. The value cannot be greater than the scrape interval otherwise the operator will reject the resource. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string targetLimit: - description: TargetLimit defines a limit on the number of scraped + description: targetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer targets: - description: Targets defines a set of static or dynamically discovered + description: targets defines a set of static or dynamically discovered targets to probe. properties: ingress: @@ -826,22 +834,24 @@ spec: If `staticConfig` is also defined, `staticConfig` takes precedence. properties: namespaceSelector: - description: From which namespaces to select Ingress objects. + description: namespaceSelector defines from which namespaces + to select Ingress objects. properties: any: description: |- - Boolean describing whether all namespaces are selected in contrast to a + any defines the boolean describing whether all namespaces are selected in contrast to a list restricting them. type: boolean matchNames: - description: List of namespace names to select from. + description: matchNames defines the list of namespace + names to select from. items: type: string type: array type: object relabelingConfigs: description: |- - RelabelConfigs to apply to the label set of the target before it gets + relabelingConfigs to apply to the label set of the target before it gets scraped. The original ingress address is available via the `__tmp_prometheus_ingress_address` label. It can be used to customize the @@ -858,7 +868,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -890,41 +900,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -934,7 +944,7 @@ spec: type: object type: array selector: - description: Selector to select the Ingress objects. + description: selector to select the Ingress objects. properties: matchExpressions: description: matchExpressions is a list of label selector @@ -990,12 +1000,12 @@ spec: labels: additionalProperties: type: string - description: Labels assigned to all metrics scraped from the - targets. + description: labels defines all labels assigned to all metrics + scraped from the targets. type: object relabelingConfigs: description: |- - RelabelConfigs to apply to the label set of the targets before it gets + relabelingConfigs defines relabelings to be apply to the label set of the targets before it gets scraped. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config items: @@ -1008,7 +1018,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -1040,41 +1050,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -1084,21 +1094,23 @@ spec: type: object type: array static: - description: The list of hosts to probe. + description: static defines the list of hosts to probe. items: type: string type: array type: object type: object tlsConfig: - description: TLS configuration to use when scraping the endpoint. + description: tlsConfig defines the TLS configuration to use when scraping + the endpoint. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when verifying + server certificates. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -1121,7 +1133,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1146,10 +1159,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present when + doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -1172,7 +1187,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1197,10 +1213,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must be @@ -1225,7 +1243,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1236,7 +1254,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1246,12 +1264,118 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for the + targets. type: string type: object type: object + status: + description: |- + status defines the status subresource. It is under active development and is updated only when the + "StatusForConfigurationResources" feature gate is enabled. + + Most recent observed status of the Probe. Read-only. + More info: + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + bindings: + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. + items: + description: WorkloadBinding is a link between a configuration resource + and a workload resource. + properties: + conditions: + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. + items: + description: ConfigResourceCondition describes the status + of configuration resources linked to Prometheus, PrometheusAgent, + Alertmanager or ThanosRuler. + properties: + lastTransitionTime: + description: lastTransitionTime defines the time of the + last update to the current status property. + format: date-time + type: string + message: + description: message defines the human-readable message + indicating details for the condition's last transition. + type: string + observedGeneration: + description: |- + observedGeneration defines the .metadata.generation that the + condition was set based upon. For instance, if `.metadata.generation` is + currently 12, but the `.status.conditions[].observedGeneration` is 9, the + condition is out of date with respect to the current state of the object. + format: int64 + type: integer + reason: + description: reason for the condition's last transition. + type: string + status: + description: status of the condition. + minLength: 1 + type: string + type: + description: |- + type of the condition being reported. + Currently, only "Accepted" is supported. + enum: + - Accepted + minLength: 1 + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + group: + description: group defines the group of the referenced resource. + enum: + - monitoring.coreos.com + type: string + name: + description: name defines the name of the referenced object. + minLength: 1 + type: string + namespace: + description: namespace defines the namespace of the referenced + object. + minLength: 1 + type: string + resource: + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). + enum: + - prometheuses + - prometheusagents + - thanosrulers + - alertmanagers + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map + type: object required: - spec type: object served: true storage: true + subresources: + status: {} diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusagents.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusagents.yaml index 74e741011b3..993c388a774 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusagents.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusagents.yaml @@ -72,12 +72,12 @@ spec: type: object spec: description: |- - Specification of the desired behavior of the Prometheus agent. More info: + spec defines the specification of the desired behavior of the Prometheus agent. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the 'prometheus' container. + additionalArgs allows setting additional arguments for the 'prometheus' container. It is intended for e.g. activating hidden flags which are not supported by the dedicated configuration options yet. The arguments are passed as-is to the @@ -91,12 +91,12 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for name-only - arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. Can + be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name @@ -104,7 +104,7 @@ spec: type: array additionalScrapeConfigs: description: |- - AdditionalScrapeConfigs allows specifying a key of a Secret containing + additionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified @@ -137,7 +137,8 @@ spec: type: object x-kubernetes-map-type: atomic affinity: - description: Defines the Pods' affinity scheduling rules if specified. + description: affinity defines the Pods' affinity scheduling rules + if specified. properties: nodeAffinity: description: Describes node affinity scheduling rules for the @@ -1050,7 +1051,7 @@ spec: type: object apiserverConfig: description: |- - APIServerConfig allows specifying a host and auth methods to access the + apiserverConfig allows specifying a host and auth methods to access the Kuberntees API server. If null, Prometheus is assumed to run inside of the cluster: it will discover the API servers automatically and use the Pod's CA certificate @@ -1058,14 +1059,14 @@ spec: properties: authorization: description: |- - Authorization section for the API server. + authorization section for the API server. Cannot be set at the same time as `basicAuth`, `bearerToken`, or `bearerTokenFile`. properties: credentials: - description: Selects a key of a Secret in the namespace that - contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -1089,12 +1090,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a secret + from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1103,14 +1104,14 @@ spec: type: object basicAuth: description: |- - BasicAuth configuration for the API server. + basicAuth configuration for the API server. Cannot be set at the same time as `authorization`, `bearerToken`, or `bearerTokenFile`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1136,7 +1137,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1163,14 +1164,13 @@ spec: type: object bearerToken: description: |- - *Warning: this field shouldn't be used because the token value appears + bearerToken is deprecated: this will be removed in a future release. + *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File to read bearer token for accessing apiserver. + bearerTokenFile defines the file to read bearer token for accessing apiserver. Cannot be set at the same time as `basicAuth`, `authorization`, or `bearerToken`. @@ -1178,12 +1178,12 @@ spec: type: string host: description: |- - Kubernetes API address consisting of a hostname or IP address followed + host defines the Kubernetes API address consisting of a hostname or IP address followed by an optional port number. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1217,7 +1217,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1225,24 +1225,24 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS Config to use for the API server. + description: tlsConfig to use for the API server. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1265,7 +1265,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1290,15 +1291,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1321,7 +1323,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1346,19 +1349,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1383,7 +1387,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1394,7 +1398,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1404,7 +1408,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -1412,7 +1417,7 @@ spec: type: object arbitraryFSAccessThroughSMs: description: |- - When true, ServiceMonitor, PodMonitor and Probe object are forbidden to + arbitraryFSAccessThroughSMs when true, ServiceMonitor, PodMonitor and Probe object are forbidden to reference arbitrary files on the file system of the 'prometheus' container. When a ServiceMonitor's endpoint specifies a `bearerTokenFile` value @@ -1424,11 +1429,16 @@ spec: `spec.bearerTokenSecret` field. properties: deny: + description: |- + deny prevents service monitors from accessing arbitrary files on the file system. + When true, service monitors cannot use file-based configurations like BearerTokenFile + that could potentially access sensitive files. When false (default), such access is allowed. + Setting this to true enhances security by preventing potential credential theft attacks. type: boolean type: object automountServiceAccountToken: description: |- - AutomountServiceAccountToken indicates whether a service account token should be automatically mounted in the pod. + automountServiceAccountToken defines whether a service account token should be automatically mounted in the pod. If the field isn't set, the operator mounts the service account token by default. **Warning:** be aware that by default, Prometheus requires the service account token for Kubernetes service discovery. @@ -1436,7 +1446,7 @@ spec: type: boolean bodySizeLimit: description: |- - BodySizeLimit defines per-scrape on response body size. + bodySizeLimit defines per-scrape on response body size. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -1445,7 +1455,7 @@ spec: type: string configMaps: description: |- - ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus + configMaps defines a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. Each ConfigMap is added to the StatefulSet definition as a volume named `configmap-`. The ConfigMaps are mounted into /etc/prometheus/configmaps/ in the 'prometheus' container. @@ -1454,7 +1464,7 @@ spec: type: array containers: description: |- - Containers allows injecting additional containers or modifying operator + containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to the Pods or to change the behavior of an operator generated container. Containers described here modify an operator generated @@ -2990,17 +3000,17 @@ spec: type: array convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.4.0. type: boolean dnsConfig: - description: Defines the DNS configuration for the pods. + description: dnsConfig defines the DNS configuration for the pods. properties: nameservers: description: |- - A list of DNS name server IP addresses. + nameservers defines the list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. items: minLength: 1 @@ -3009,7 +3019,7 @@ spec: x-kubernetes-list-type: set options: description: |- - A list of DNS resolver options. + options defines the list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Resolution options given in Options will override those that appear in the base DNSPolicy. @@ -3018,11 +3028,11 @@ spec: of a pod. properties: name: - description: Name is required and must be unique. + description: name is required and must be unique. minLength: 1 type: string value: - description: Value is optional. + description: value is optional. type: string required: - name @@ -3033,7 +3043,7 @@ spec: x-kubernetes-list-type: map searches: description: |- - A list of DNS search domains for host-name lookup. + searches defines the list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. items: minLength: 1 @@ -3042,7 +3052,7 @@ spec: x-kubernetes-list-type: set type: object dnsPolicy: - description: Defines the DNS policy for the pods. + description: dnsPolicy defines the DNS policy for the pods. enum: - ClusterFirstWithHostNet - ClusterFirst @@ -3051,7 +3061,7 @@ spec: type: string enableFeatures: description: |- - Enable access to Prometheus feature flags. By default, no features are enabled. + enableFeatures enables access to Prometheus feature flags. By default, no features are enabled. Enabling features which are disabled by default is entirely outside the scope of what the maintainers will support and by doing so, you accept @@ -3065,7 +3075,7 @@ spec: x-kubernetes-list-type: set enableOTLPReceiver: description: |- - Enable Prometheus to be used as a receiver for the OTLP Metrics protocol. + enableOTLPReceiver defines the Prometheus to be used as a receiver for the OTLP Metrics protocol. Note that the OTLP receiver endpoint is automatically enabled if `.spec.otlpConfig` is defined. @@ -3073,7 +3083,7 @@ spec: type: boolean enableRemoteWriteReceiver: description: |- - Enable Prometheus to be used as a receiver for the Prometheus remote + enableRemoteWriteReceiver defines the Prometheus to be used as a receiver for the Prometheus remote write protocol. WARNING: This is not considered an efficient way of ingesting samples. @@ -3085,12 +3095,12 @@ spec: It requires Prometheus >= v2.33.0. type: boolean enableServiceLinks: - description: Indicates whether information about services should be - injected into pod's environment variables + description: enableServiceLinks defines whether information about + services should be injected into pod's environment variables type: boolean enforcedBodySizeLimit: description: |- - When defined, enforcedBodySizeLimit specifies a global limit on the size + enforcedBodySizeLimit when defined specifies a global limit on the size of uncompressed response body that will be accepted by Prometheus. Targets responding with a body larger than this many bytes will cause the scrape to fail. @@ -3106,7 +3116,7 @@ spec: type: string enforcedKeepDroppedTargets: description: |- - When defined, enforcedKeepDroppedTargets specifies a global limit on the number of targets + enforcedKeepDroppedTargets when defined specifies a global limit on the number of targets dropped by relabeling that will be kept in memory. The value overrides any `spec.keepDroppedTargets` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.keepDroppedTargets` is @@ -3123,7 +3133,7 @@ spec: type: integer enforcedLabelLimit: description: |- - When defined, enforcedLabelLimit specifies a global limit on the number + enforcedLabelLimit when defined specifies a global limit on the number of labels per sample. The value overrides any `spec.labelLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelLimit` is greater than zero and less than `spec.enforcedLabelLimit`. @@ -3139,7 +3149,7 @@ spec: type: integer enforcedLabelNameLengthLimit: description: |- - When defined, enforcedLabelNameLengthLimit specifies a global limit on the length + enforcedLabelNameLengthLimit when defined specifies a global limit on the length of labels name per sample. The value overrides any `spec.labelNameLengthLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelNameLengthLimit` is greater than zero and less than `spec.enforcedLabelNameLengthLimit`. @@ -3155,7 +3165,7 @@ spec: type: integer enforcedLabelValueLengthLimit: description: |- - When not null, enforcedLabelValueLengthLimit defines a global limit on the length + enforcedLabelValueLengthLimit when not null defines a global limit on the length of labels value per sample. The value overrides any `spec.labelValueLengthLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelValueLengthLimit` is greater than zero and less than `spec.enforcedLabelValueLengthLimit`. @@ -3171,7 +3181,7 @@ spec: type: integer enforcedNamespaceLabel: description: |- - When not empty, a label will be added to: + enforcedNamespaceLabel when not empty, a label will be added to: 1. All metrics scraped from `ServiceMonitor`, `PodMonitor`, `Probe` and `ScrapeConfig` objects. 2. All metrics generated from recording rules defined in `PrometheusRule` objects. @@ -3186,7 +3196,7 @@ spec: type: string enforcedSampleLimit: description: |- - When defined, enforcedSampleLimit specifies a global limit on the number + enforcedSampleLimit when defined specifies a global limit on the number of scraped samples that will be accepted. This overrides any `spec.sampleLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.sampleLimit` is greater than zero and less than @@ -3204,7 +3214,7 @@ spec: type: integer enforcedTargetLimit: description: |- - When defined, enforcedTargetLimit specifies a global limit on the number + enforcedTargetLimit when defined specifies a global limit on the number of scraped targets. The value overrides any `spec.targetLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.targetLimit` is greater than zero and less than `spec.enforcedTargetLimit`. @@ -3221,7 +3231,7 @@ spec: type: integer excludedFromEnforcement: description: |- - List of references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + excludedFromEnforcement defines the list of references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects to be excluded from enforcing a namespace label of origin. It is only applicable if `spec.enforcedNamespaceLabel` set to true. @@ -3231,23 +3241,23 @@ spec: properties: group: default: monitoring.coreos.com - description: Group of the referent. When not specified, it defaults + description: group of the referent. When not specified, it defaults to `monitoring.coreos.com` enum: - monitoring.coreos.com type: string name: - description: Name of the referent. When not set, all resources + description: name of the referent. When not set, all resources in the namespace are matched. type: string namespace: description: |- - Namespace of the referent. + namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ minLength: 1 type: string resource: - description: Resource of the referent. + description: resource of the referent. enum: - prometheusrules - servicemonitors @@ -3264,20 +3274,20 @@ spec: additionalProperties: type: string description: |- - The labels to add to any time series or alerts when communicating with + externalLabels defines the labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager). Labels defined by `spec.replicaExternalLabelName` and `spec.prometheusExternalLabelName` take precedence over this list. type: object externalUrl: description: |- - The external URL under which the Prometheus service is externally + externalUrl defines the external URL under which the Prometheus service is externally available. This is necessary to generate correct URLs (for instance if Prometheus is accessible behind an Ingress resource). type: string hostAliases: description: |- - Optional list of hosts and IPs that will be injected into the Pod's + hostAliases defines the optional list of hosts and IPs that will be injected into the Pod's hosts file if specified. items: description: |- @@ -3285,12 +3295,12 @@ spec: pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. + description: hostnames defines hostnames for the above IP address. items: type: string type: array ip: - description: IP address of the host file entry. + description: ip defines the IP address of the host file entry. type: string required: - hostnames @@ -3302,7 +3312,7 @@ spec: x-kubernetes-list-type: map hostNetwork: description: |- - Use the host's network namespace if true. + hostNetwork defines the host's network namespace if true. Make sure to understand the security implications if you want to enable it (https://kubernetes.io/docs/concepts/configuration/overview/ ). @@ -3313,7 +3323,7 @@ spec: type: boolean hostUsers: description: |- - HostUsers supports the user space in Kubernetes. + hostUsers supports the user space in Kubernetes. More info: https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/ @@ -3322,14 +3332,14 @@ spec: type: boolean ignoreNamespaceSelectors: description: |- - When true, `spec.namespaceSelector` from all PodMonitor, ServiceMonitor + ignoreNamespaceSelectors when true, `spec.namespaceSelector` from all PodMonitor, ServiceMonitor and Probe objects will be ignored. They will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object. type: boolean image: description: |- - Container image name for Prometheus. If specified, it takes precedence + image defines the container image name for Prometheus. If specified, it takes precedence over the `spec.baseImage`, `spec.tag` and `spec.sha` fields. Specifying `spec.version` is still necessary to ensure the Prometheus @@ -3341,7 +3351,7 @@ spec: type: string imagePullPolicy: description: |- - Image pull policy for the 'prometheus', 'init-config-reloader' and 'config-reloader' containers. + imagePullPolicy defines the image pull policy for the 'prometheus', 'init-config-reloader' and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. enum: - "" @@ -3351,7 +3361,7 @@ spec: type: string imagePullSecrets: description: |- - An optional list of references to Secrets in the same namespace + imagePullSecrets defines an optional list of references to Secrets in the same namespace to use for pulling images from registries. See http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod items: @@ -3373,7 +3383,7 @@ spec: type: array initContainers: description: |- - InitContainers allows injecting initContainers to the Pod definition. Those + initContainers allows injecting initContainers to the Pod definition. Those can be used to e.g. fetch secrets for injection into the Prometheus configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: @@ -4909,7 +4919,7 @@ spec: type: array keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -4920,7 +4930,7 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -4929,7 +4939,7 @@ spec: type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -4938,7 +4948,7 @@ spec: type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -4947,11 +4957,11 @@ spec: type: integer listenLocal: description: |- - When true, the Prometheus server listens on the loopback address + listenLocal when true, the Prometheus server listens on the loopback address instead of the Pod IP's address. type: boolean logFormat: - description: Log format for Log level for Prometheus and the config-reloader + description: logFormat for Log level for Prometheus and the config-reloader sidecar. enum: - "" @@ -4959,7 +4969,7 @@ spec: - json type: string logLevel: - description: Log level for Prometheus and the config-reloader sidecar. + description: logLevel for Prometheus and the config-reloader sidecar. enum: - "" - debug @@ -4969,14 +4979,14 @@ spec: type: string maximumStartupDurationSeconds: description: |- - Defines the maximum time that the `prometheus` container's startup probe will wait before being considered failed. The startup probe will return success after the WAL replay is complete. - If set, the value should be greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 minutes). + maximumStartupDurationSeconds defines the maximum time that the `prometheus` container's startup probe will wait before being considered failed. The startup probe will return success after the WAL replay is complete. + If set, the value should be greater than 60 (seconds). Otherwise it will be equal to 900 seconds (15 minutes). format: int32 minimum: 60 type: integer minReadySeconds: description: |- - Minimum number of seconds for which a newly created Pod should be ready + minReadySeconds defines the minimum number of seconds for which a newly created Pod should be ready without any of its container crashing for it to be considered available. If unset, pods will be considered available as soon as they are ready. @@ -4985,7 +4995,7 @@ spec: type: integer mode: description: |- - Mode defines how the Prometheus operator deploys the PrometheusAgent pod(s). + mode defines how the Prometheus operator deploys the PrometheusAgent pod(s). (Alpha) Using this field requires the `PrometheusAgentDaemonSet` feature gate to be enabled. enum: @@ -4994,7 +5004,7 @@ spec: type: string nameEscapingScheme: description: |- - Specifies the character escaping scheme that will be requested when scraping + nameEscapingScheme defines the character escaping scheme that will be requested when scraping for metric and label names that do not conform to the legacy Prometheus character set. @@ -5007,7 +5017,7 @@ spec: type: string nameValidationScheme: description: |- - Specifies the validation scheme for metric and label names. + nameValidationScheme defines the validation scheme for metric and label names. It requires Prometheus >= v2.55.0. enum: @@ -5017,21 +5027,21 @@ spec: nodeSelector: additionalProperties: type: string - description: Defines on which Nodes the Pods are scheduled. + description: nodeSelector defines on which Nodes the Pods are scheduled. type: object otlp: description: |- - Settings related to the OTLP receiver feature. + otlp defines the settings related to the OTLP receiver feature. It requires Prometheus >= v2.55.0. properties: convertHistogramsToNHCB: description: |- - Configures optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. + convertHistogramsToNHCB defines optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. It requires Prometheus >= v3.4.0. type: boolean ignoreResourceAttributes: description: |- - List of OpenTelemetry resource attributes to ignore when `promoteAllResourceAttributes` is true. + ignoreResourceAttributes defines the list of OpenTelemetry resource attributes to ignore when `promoteAllResourceAttributes` is true. It requires `promoteAllResourceAttributes` to be true. It requires Prometheus >= v3.5.0. @@ -5043,21 +5053,21 @@ spec: x-kubernetes-list-type: set keepIdentifyingResourceAttributes: description: |- - Enables adding `service.name`, `service.namespace` and `service.instance.id` + keepIdentifyingResourceAttributes enables adding `service.name`, `service.namespace` and `service.instance.id` resource attributes to the `target_info` metric, on top of converting them into the `instance` and `job` labels. It requires Prometheus >= v3.1.0. type: boolean promoteAllResourceAttributes: description: |- - Promote all resource attributes to metric labels except the ones defined in `ignoreResourceAttributes`. + promoteAllResourceAttributes promotes all resource attributes to metric labels except the ones defined in `ignoreResourceAttributes`. Cannot be true when `promoteResourceAttributes` is defined. It requires Prometheus >= v3.5.0. type: boolean promoteResourceAttributes: description: |- - List of OpenTelemetry Attributes that should be promoted to metric labels, defaults to none. + promoteResourceAttributes defines the list of OpenTelemetry Attributes that should be promoted to metric labels, defaults to none. Cannot be defined when `promoteAllResourceAttributes` is true. items: minLength: 1 @@ -5065,39 +5075,46 @@ spec: minItems: 1 type: array x-kubernetes-list-type: set + promoteScopeMetadata: + description: |- + promoteScopeMetadata controls whether to promote OpenTelemetry scope metadata (i.e. name, version, schema URL, and attributes) to metric labels. + As per the OpenTelemetry specification, the aforementioned scope metadata should be identifying, i.e. made into metric labels. + It requires Prometheus >= v3.6.0. + type: boolean translationStrategy: description: |- - Configures how the OTLP receiver endpoint translates the incoming metrics. + translationStrategy defines how the OTLP receiver endpoint translates the incoming metrics. It requires Prometheus >= v3.0.0. enum: - NoUTF8EscapingWithSuffixes - UnderscoreEscapingWithSuffixes - NoTranslation + - UnderscoreEscapingWithoutSuffixes type: string type: object overrideHonorLabels: description: |- - When true, Prometheus resolves label conflicts by renaming the labels in the scraped data + overrideHonorLabels when true, Prometheus resolves label conflicts by renaming the labels in the scraped data to “exported_” for all targets created from ServiceMonitor, PodMonitor and ScrapeConfig objects. Otherwise the HonorLabels field of the service or pod monitor applies. - In practice,`overrideHonorLaels:true` enforces `honorLabels:false` + In practice,`OverrideHonorLabels:true` enforces `honorLabels:false` for all ServiceMonitor, PodMonitor and ScrapeConfig objects. type: boolean overrideHonorTimestamps: description: |- - When true, Prometheus ignores the timestamps for all the targets created + overrideHonorTimestamps when true, Prometheus ignores the timestamps for all the targets created from service and pod monitors. Otherwise the HonorTimestamps field of the service or pod monitor applies. type: boolean paused: description: |- - When a Prometheus deployment is paused, no actions except for deletion + paused defines when a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects. type: boolean persistentVolumeClaimRetentionPolicy: description: |- - The field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. + persistentVolumeClaimRetentionPolicy defines the field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. The default behavior is all PVCs are retained. This is an alpha field from kubernetes 1.23 until 1.26 and a beta field from 1.26. It requires enabling the StatefulSetAutoDeletePVC feature gate. @@ -5120,7 +5137,7 @@ spec: type: object podMetadata: description: |- - PodMetadata configures labels and annotations which are propagated to the Prometheus pods. + podMetadata defines labels and annotations which are propagated to the Prometheus pods. The following items are reserved and cannot be overridden: * "prometheus" label, set to the name of the Prometheus object. @@ -5136,7 +5153,7 @@ spec: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -5145,14 +5162,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -5162,7 +5179,7 @@ spec: type: object podMonitorNamespaceSelector: description: |- - Namespaces to match for PodMonitors discovery. An empty label selector + podMonitorNamespaceSelector defines the namespaces to match for PodMonitors discovery. An empty label selector matches all namespaces. A null label selector (default value) matches the current namespace only. properties: @@ -5211,7 +5228,7 @@ spec: x-kubernetes-map-type: atomic podMonitorSelector: description: |- - PodMonitors to be selected for target discovery. An empty label selector + podMonitorSelector defines the podMonitors to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -5268,7 +5285,7 @@ spec: x-kubernetes-map-type: atomic podTargetLabels: description: |- - PodTargetLabels are appended to the `spec.podTargetLabels` field of all + podTargetLabels are appended to the `spec.podTargetLabels` field of all PodMonitor and ServiceMonitor objects. items: type: string @@ -5276,15 +5293,15 @@ spec: portName: default: web description: |- - Port name used for the pods and governing service. + portName used for the pods and governing service. Default: "web" type: string priorityClassName: - description: Priority class assigned to the Pods. + description: priorityClassName assigned to the Pods. type: string probeNamespaceSelector: description: |- - Namespaces to match for Probe discovery. An empty label + probeNamespaceSelector defines the namespaces to match for Probe discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. properties: @@ -5333,7 +5350,7 @@ spec: x-kubernetes-map-type: atomic probeSelector: description: |- - Probes to be selected for target discovery. An empty label selector + probeSelector defines the probes to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -5390,7 +5407,7 @@ spec: x-kubernetes-map-type: atomic prometheusExternalLabelName: description: |- - Name of Prometheus external label used to denote the Prometheus instance + prometheusExternalLabelName defines the name of Prometheus external label used to denote the Prometheus instance name. The external label will _not_ be added when the field is set to the empty string (`""`). @@ -5398,14 +5415,14 @@ spec: type: string reloadStrategy: description: |- - Defines the strategy used to reload the Prometheus configuration. + reloadStrategy defines the strategy used to reload the Prometheus configuration. If not specified, the configuration is reloaded using the /-/reload HTTP endpoint. enum: - HTTP - ProcessSignal type: string remoteWrite: - description: Defines the list of remote write configurations. + description: remoteWrite defines the list of remote write configurations. items: description: |- RemoteWriteSpec defines the configuration to write samples from Prometheus @@ -5413,15 +5430,15 @@ spec: properties: authorization: description: |- - Authorization section for the URL. + authorization section for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. Cannot be set at the same time as `sigv4`, `basicAuth`, `oauth2`, or `azureAd`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -5445,12 +5462,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -5459,15 +5476,15 @@ spec: type: object azureAd: description: |- - AzureAD for the URL. + azureAd for the URL. It requires Prometheus >= v2.45.0 or Thanos >= v0.31.0. Cannot be set at the same time as `authorization`, `basicAuth`, `oauth2`, or `sigv4`. properties: cloud: - description: The Azure Cloud. Options are 'AzurePublic', - 'AzureChina', or 'AzureGovernment'. + description: cloud defines the Azure Cloud. Options are + 'AzurePublic', 'AzureChina', or 'AzureGovernment'. enum: - AzureChina - AzureGovernment @@ -5475,32 +5492,33 @@ spec: type: string managedIdentity: description: |- - ManagedIdentity defines the Azure User-assigned Managed identity. + managedIdentity defines the Azure User-assigned Managed identity. Cannot be set at the same time as `oauth` or `sdk`. properties: clientId: - description: The client id + description: clientId defines defines the Azure User-assigned + Managed identity. type: string required: - clientId type: object oauth: description: |- - OAuth defines the oauth config that is being used to authenticate. + oauth defines the oauth config that is being used to authenticate. Cannot be set at the same time as `managedIdentity` or `sdk`. It requires Prometheus >= v2.48.0 or Thanos >= v0.31.0. properties: clientId: - description: '`clientID` is the clientId of the Azure + description: clientId defines the clientId of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 type: string clientSecret: - description: '`clientSecret` specifies a key of a Secret + description: clientSecret specifies a key of a Secret containing the client secret of the Azure Active Directory - application that is being used to authenticate.' + application that is being used to authenticate. properties: key: description: The key of the secret to select from. Must @@ -5524,9 +5542,9 @@ spec: type: object x-kubernetes-map-type: atomic tenantId: - description: '`tenantId` is the tenant ID of the Azure + description: tenantId is the tenant ID of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 pattern: ^[0-9a-zA-Z-.]+$ type: string @@ -5537,29 +5555,29 @@ spec: type: object sdk: description: |- - SDK defines the Azure SDK config that is being used to authenticate. + sdk defines the Azure SDK config that is being used to authenticate. See https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication Cannot be set at the same time as `oauth` or `managedIdentity`. It requires Prometheus >= v2.52.0 or Thanos >= v0.36.0. properties: tenantId: - description: '`tenantId` is the tenant ID of the azure + description: tenantId defines the tenant ID of the azure active directory application that is being used to - authenticate.' + authenticate. pattern: ^[0-9a-zA-Z-.]+$ type: string type: object type: object basicAuth: description: |- - BasicAuth configuration for the URL. + basicAuth configuration for the URL. Cannot be set at the same time as `sigv4`, `authorization`, `oauth2`, or `azureAd`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -5585,7 +5603,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -5612,23 +5630,22 @@ spec: type: object bearerToken: description: |- + bearerToken is deprecated: this will be removed in a future release. *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File from which to read bearer token for the URL. + bearerTokenFile defines the file from which to read bearer token for the URL. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. type: boolean @@ -5636,14 +5653,14 @@ spec: additionalProperties: type: string description: |- - Custom HTTP headers to be sent along with each remote write request. + headers defines the custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. It requires Prometheus >= v2.25.0 or Thanos >= v0.24.0. type: object messageVersion: description: |- - The Remote Write message's version to use when writing to the endpoint. + messageVersion defines the Remote Write message's version to use when writing to the endpoint. `Version1.0` corresponds to the `prometheus.WriteRequest` protobuf message introduced in Remote Write 1.0. `Version2.0` corresponds to the `io.prometheus.write.v2.Request` protobuf message introduced in Remote Write 2.0. @@ -5660,37 +5677,37 @@ spec: - V2.0 type: string metadataConfig: - description: MetadataConfig configures the sending of series - metadata to the remote storage. + description: metadataConfig defines how to send a series metadata + to the remote storage. properties: maxSamplesPerSend: description: |- - MaxSamplesPerSend is the maximum number of metadata samples per send. + maxSamplesPerSend defines the maximum number of metadata samples per send. It requires Prometheus >= v2.29.0. format: int32 minimum: -1 type: integer send: - description: Defines whether metric metadata is sent to - the remote storage or not. + description: send defines whether metric metadata is sent + to the remote storage or not. type: boolean sendInterval: - description: Defines how frequently metric metadata is sent - to the remote storage. + description: sendInterval defines how frequently metric + metadata is sent to the remote storage. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object name: description: |- - The name of the remote write queue, it must be unique if specified. The + name of the remote write queue, it must be unique if specified. The name is used in metrics and logging in order to differentiate queues. It requires Prometheus >= v2.15.0 or Thanos >= 0.24.0. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5698,7 +5715,7 @@ spec: type: string oauth2: description: |- - OAuth2 configuration for the URL. + oauth2 configuration for the URL. It requires Prometheus >= v2.27.0 or Thanos >= v0.24.0. @@ -5706,12 +5723,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5734,7 +5751,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5760,7 +5778,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5788,12 +5806,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5827,7 +5845,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5835,33 +5853,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5884,8 +5901,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5910,12 +5927,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5938,8 +5955,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5964,11 +5981,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5993,7 +6011,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6004,7 +6022,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6014,12 +6032,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -6055,7 +6074,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6063,89 +6082,88 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string queueConfig: - description: QueueConfig allows tuning of the remote write queue + description: queueConfig allows tuning of the remote write queue parameters. properties: batchSendDeadline: - description: BatchSendDeadline is the maximum time a sample - will wait in buffer. + description: batchSendDeadline defines the maximum time + a sample will wait in buffer. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string capacity: description: |- - Capacity is the number of samples to buffer per shard before we start + capacity defines the number of samples to buffer per shard before we start dropping them. type: integer maxBackoff: - description: MaxBackoff is the maximum retry delay. + description: maxBackoff defines the maximum retry delay. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string maxRetries: - description: MaxRetries is the maximum number of times to - retry a batch on recoverable errors. + description: maxRetries defines the maximum number of times + to retry a batch on recoverable errors. type: integer maxSamplesPerSend: - description: MaxSamplesPerSend is the maximum number of - samples per send. + description: maxSamplesPerSend defines the maximum number + of samples per send. type: integer maxShards: - description: MaxShards is the maximum number of shards, + description: maxShards defines the maximum number of shards, i.e. amount of concurrency. type: integer minBackoff: - description: MinBackoff is the initial retry delay. Gets - doubled for every retry. + description: minBackoff defines the initial retry delay. + Gets doubled for every retry. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string minShards: - description: MinShards is the minimum number of shards, + description: minShards defines the minimum number of shards, i.e. amount of concurrency. type: integer retryOnRateLimit: description: |- - Retry upon receiving a 429 status code from the remote-write storage. + retryOnRateLimit defines the retry upon receiving a 429 status code from the remote-write storage. This is an *experimental feature*, it may change in any upcoming release in a breaking way. type: boolean sampleAgeLimit: description: |- - SampleAgeLimit drops samples older than the limit. + sampleAgeLimit drops samples older than the limit. It requires Prometheus >= v2.50.0 or Thanos >= v0.32.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object remoteTimeout: - description: Timeout for requests to the remote write endpoint. + description: remoteTimeout defines the timeout for requests + to the remote write endpoint. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string roundRobinDNS: - description: |- - When enabled: - - The remote-write mechanism will resolve the hostname via DNS. - - It will randomly select one of the resolved IP addresses and connect to it. - - When disabled (default behavior): - - The Go standard library will handle hostname resolution. - - It will attempt connections to each resolved IP address sequentially. - - Note: The connection timeout applies to the entire resolution and connection process. - If disabled, the timeout is distributed across all connection attempts. - - It requires Prometheus >= v3.1.0 or Thanos >= v0.38.0. + description: "roundRobinDNS controls the DNS resolution behavior + for remote-write connections.\nWhen enabled:\n - The remote-write + mechanism will resolve the hostname via DNS.\n - It will + randomly select one of the resolved IP addresses and connect + to it.\n\nWhen disabled (default behavior):\n - The Go standard + library will handle hostname resolution.\n - It will attempt + connections to each resolved IP address sequentially.\n\nNote: + The connection timeout applies to the entire resolution and + connection process.\n\n\tIf disabled, the timeout is distributed + across all connection attempts.\n\nIt requires Prometheus + >= v3.1.0 or Thanos >= v0.38.0." type: boolean sendExemplars: description: |- - Enables sending of exemplars over remote write. Note that + sendExemplars enables sending of exemplars over remote write. Note that exemplar-storage itself must be enabled using the `spec.enableFeatures` option for exemplars to be scraped in the first place. @@ -6153,14 +6171,14 @@ spec: type: boolean sendNativeHistograms: description: |- - Enables sending of native histograms, also known as sparse histograms + sendNativeHistograms enables sending of native histograms, also known as sparse histograms over remote write. It requires Prometheus >= v2.40.0 or Thanos >= v0.30.0. type: boolean sigv4: description: |- - Sigv4 allows to configures AWS's Signature Verification 4 for the URL. + sigv4 defines the AWS's Signature Verification 4 for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. @@ -6168,7 +6186,7 @@ spec: properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -6193,18 +6211,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used to authenticate. + description: profile defines the named AWS profile used + to authenticate. type: string region: - description: Region is the AWS region. If blank, the region - from the default credentials chain used. + description: region defines the AWS region. If blank, the + region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used to authenticate. + description: roleArn defines the named AWS profile used + to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -6228,17 +6248,22 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object tlsConfig: - description: TLS Config to use for the URL. + description: tlsConfig to use for the URL. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6261,7 +6286,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6286,15 +6312,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6317,7 +6344,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6342,19 +6370,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6379,7 +6408,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6390,7 +6419,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6400,15 +6429,18 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the endpoint to send samples to. + description: url defines the URL of the endpoint to send samples + to. minLength: 1 type: string writeRelabelConfigs: - description: The list of remote write relabel configurations. + description: writeRelabelConfigs defines the list of remote + write relabel configurations. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -6419,7 +6451,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -6451,41 +6483,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -6500,7 +6532,7 @@ spec: type: array remoteWriteReceiverMessageVersions: description: |- - List of the protobuf message versions to accept when receiving the + remoteWriteReceiverMessageVersions list of the protobuf message versions to accept when receiving the remote writes. It requires Prometheus >= v2.54.0. @@ -6514,7 +6546,7 @@ spec: x-kubernetes-list-type: set replicaExternalLabelName: description: |- - Name of Prometheus external label used to denote the replica name. + replicaExternalLabelName defines the name of Prometheus external label used to denote the replica name. The external label will _not_ be added when the field is set to the empty string (`""`). @@ -6522,7 +6554,7 @@ spec: type: string replicas: description: |- - Number of replicas of each shard to deploy for a Prometheus deployment. + replicas defines the number of replicas of each shard to deploy for a Prometheus deployment. `spec.replicas` multiplied by `spec.shards` is the total number of Pods created. @@ -6530,8 +6562,8 @@ spec: format: int32 type: integer resources: - description: Defines the resources requests and limits of the 'prometheus' - container. + description: resources defines the resources requests and limits of + the 'prometheus' container. properties: claims: description: |- @@ -6591,7 +6623,7 @@ spec: type: object routePrefix: description: |- - The route prefix Prometheus registers HTTP handlers for. + routePrefix defines the route prefix Prometheus registers HTTP handlers for. This is useful when using `spec.externalURL`, and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but @@ -6599,12 +6631,12 @@ spec: for use with `kubectl proxy`. type: string runtime: - description: RuntimeConfig configures the values for the Prometheus - process behavior + description: runtime defines the values for the Prometheus process + behavior properties: goGC: description: |- - The Go garbage collection target percentage. Lowering this number may increase the CPU usage. + goGC defines the Go garbage collection target percentage. Lowering this number may increase the CPU usage. See: https://tip.golang.org/doc/gc-guide#GOGC format: int32 minimum: -1 @@ -6612,7 +6644,7 @@ spec: type: object sampleLimit: description: |- - SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit defines per-scrape limit on number of scraped samples that will be accepted. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -6621,7 +6653,7 @@ spec: type: integer scrapeClasses: description: |- - List of scrape classes to expose to scraping objects such as + scrapeClasses defines the list of scrape classes to expose to scraping objects such as PodMonitors, ServiceMonitors, Probes and ScrapeConfigs. This is an *experimental feature*, it may change in any upcoming release @@ -6630,13 +6662,13 @@ spec: properties: attachMetadata: description: |- - AttachMetadata configures additional metadata to the discovered targets. + attachMetadata defines additional metadata to the discovered targets. When the scrape object defines its own configuration, it takes precedence over the scrape class configuration. properties: node: description: |- - When set to true, Prometheus attaches node metadata to the discovered + node when set to true, Prometheus attaches node metadata to the discovered targets. The Prometheus service account must have the `list` and `watch` @@ -6645,12 +6677,12 @@ spec: type: object authorization: description: |- - Authorization section for the ScrapeClass. + authorization section for the ScrapeClass. It will only apply if the scrape resource doesn't specify any Authorization. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -6674,12 +6706,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6688,14 +6720,14 @@ spec: type: object default: description: |- - Default indicates that the scrape applies to all scrape objects that + default defines that the scrape applies to all scrape objects that don't configure an explicit scrape class name. Only one scrape class can be set as the default. type: boolean fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It will only apply if the scrape resource doesn't specify any FallbackScrapeProtocol It requires Prometheus >= v3.0.0. @@ -6708,7 +6740,7 @@ spec: type: string metricRelabelings: description: |- - MetricRelabelings configures the relabeling rules to apply to all samples before ingestion. + metricRelabelings defines the relabeling rules to apply to all samples before ingestion. The Operator adds the scrape class metric relabelings defined here. Then the Operator adds the target-specific metric relabelings defined in ServiceMonitors, PodMonitors, Probes and ScrapeConfigs. @@ -6725,7 +6757,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -6757,41 +6789,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -6801,12 +6833,12 @@ spec: type: object type: array name: - description: Name of the scrape class. + description: name of the scrape class. minLength: 1 type: string relabelings: description: |- - Relabelings configures the relabeling rules to apply to all scrape targets. + relabelings defines the relabeling rules to apply to all scrape targets. The Operator automatically adds relabelings for a few standard Kubernetes fields like `__meta_kubernetes_namespace` and `__meta_kubernetes_service_name`. @@ -6824,7 +6856,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -6856,41 +6888,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -6901,19 +6933,19 @@ spec: type: array tlsConfig: description: |- - TLSConfig defines the TLS settings to use for the scrape. When the + tlsConfig defines the TLS settings to use for the scrape. When the scrape objects define their own CA, certificate and/or key, they take precedence over the corresponding scrape class fields. For now only the `caFile`, `certFile` and `keyFile` fields are supported. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6936,7 +6968,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6961,15 +6994,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6992,7 +7026,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7017,19 +7052,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7054,7 +7090,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7065,7 +7101,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7075,7 +7111,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -7087,7 +7124,7 @@ spec: x-kubernetes-list-type: map scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. @@ -7095,7 +7132,7 @@ spec: type: boolean scrapeConfigNamespaceSelector: description: |- - Namespaces to match for ScrapeConfig discovery. An empty label selector + scrapeConfigNamespaceSelector defines the namespaces to match for ScrapeConfig discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. @@ -7146,7 +7183,7 @@ spec: x-kubernetes-map-type: atomic scrapeConfigSelector: description: |- - ScrapeConfigs to be selected for target discovery. An empty label + scrapeConfigSelector defines the scrapeConfigs to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -7205,7 +7242,7 @@ spec: x-kubernetes-map-type: atomic scrapeFailureLogFile: description: |- - File to which scrape failures are logged. + scrapeFailureLogFile defines the file to which scrape failures are logged. Reloading the configuration will reopen the file. If the filename has an empty path, e.g. 'file.log', The Prometheus Pods @@ -7218,14 +7255,14 @@ spec: scrapeInterval: default: 30s description: |- - Interval between consecutive scrapes. + scrapeInterval defines interval between consecutive scrapes. Default: "30s" pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string scrapeProtocols: description: |- - The protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -7253,13 +7290,13 @@ spec: x-kubernetes-list-type: set scrapeTimeout: description: |- - Number of seconds to wait until a scrape request times out. + scrapeTimeout defines the number of seconds to wait until a scrape request times out. The value cannot be greater than the scrape interval otherwise the operator will reject the resource. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string secrets: description: |- - Secrets is a list of Secrets in the same namespace as the Prometheus + secrets defines a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. Each Secret is added to the StatefulSet definition as a volume named `secret-`. The Secrets are mounted into /etc/prometheus/secrets/ in the 'prometheus' container. @@ -7269,7 +7306,7 @@ spec: x-kubernetes-list-type: set securityContext: description: |- - SecurityContext holds pod-level security attributes and common container settings. + securityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: appArmorProfile: @@ -7503,12 +7540,12 @@ spec: type: object serviceAccountName: description: |- - ServiceAccountName is the name of the ServiceAccount to use to run the + serviceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods. type: string serviceDiscoveryRole: description: |- - Defines the service discovery role used to discover targets from + serviceDiscoveryRole defines the service discovery role used to discover targets from `ServiceMonitor` objects and Alertmanager endpoints. If set, the value should be either "Endpoints" or "EndpointSlice". @@ -7519,7 +7556,7 @@ spec: type: string serviceMonitorNamespaceSelector: description: |- - Namespaces to match for ServicedMonitors discovery. An empty label selector + serviceMonitorNamespaceSelector defines the namespaces to match for ServicedMonitors discovery. An empty label selector matches all namespaces. A null label selector (default value) matches the current namespace only. properties: @@ -7568,7 +7605,7 @@ spec: x-kubernetes-map-type: atomic serviceMonitorSelector: description: |- - ServiceMonitors to be selected for target discovery. An empty label + serviceMonitorSelector defines the serviceMonitors to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -7625,7 +7662,7 @@ spec: x-kubernetes-map-type: atomic serviceName: description: |- - The name of the service name used by the underlying StatefulSet(s) as the governing service. + serviceName defines the name of the service name used by the underlying StatefulSet(s) as the governing service. If defined, the Service must be created before the Prometheus/PrometheusAgent resource in the same namespace and it must define a selector that matches the pod labels. If empty, the operator will create and manage a headless service named `prometheus-operated` for Prometheus resources, or `prometheus-agent-operated` for PrometheusAgent resources. @@ -7635,7 +7672,7 @@ spec: type: string shards: description: |- - Number of shards to distribute the scraped targets onto. + shards defines the number of shards to distribute the scraped targets onto. `spec.replicas` multiplied by `spec.shards` is the total number of Pods being created. @@ -7664,15 +7701,15 @@ spec: format: int32 type: integer storage: - description: Storage defines the storage used by Prometheus. + description: storage defines the storage used by Prometheus. properties: disableMountSubPath: - description: 'Deprecated: subPath usage will be removed in a future - release.' + description: 'disableMountSubPath deprecated: subPath usage will + be removed in a future release.' type: boolean emptyDir: description: |- - EmptyDirVolumeSource to be used by the StatefulSet. + emptyDir to be used by the StatefulSet. If specified, it takes precedence over `ephemeral` and `volumeClaimTemplate`. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir properties: @@ -7699,7 +7736,7 @@ spec: type: object ephemeral: description: |- - EphemeralVolumeSource to be used by the StatefulSet. + ephemeral to be used by the StatefulSet. This is a beta field in k8s 1.21 and GA in 1.15. For lower versions, starting with k8s 1.19, it requires enabling the GenericEphemeralVolume feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes @@ -7944,7 +7981,7 @@ spec: type: object volumeClaimTemplate: description: |- - Defines the PVC spec to be used by the Prometheus StatefulSets. + volumeClaimTemplate defines the PVC spec to be used by the Prometheus StatefulSets. The easiest way to use a volume that cannot be automatically provisioned is to use a label selector alongside manually created PersistentVolumes. properties: @@ -7964,14 +8001,14 @@ spec: More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: - description: EmbeddedMetadata contains metadata relevant to - an EmbeddedResource. + description: metadata defines EmbeddedMetadata contains metadata + relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -7980,14 +8017,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -7997,7 +8034,7 @@ spec: type: object spec: description: |- - Defines the desired characteristics of a volume requested by a pod author. + spec defines the specification of the characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: accessModes: @@ -8193,7 +8230,7 @@ spec: type: string type: object status: - description: 'Deprecated: this field is never set.' + description: 'status is deprecated: this field is never set.' properties: accessModes: description: |- @@ -8378,7 +8415,7 @@ spec: type: object targetLimit: description: |- - TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit defines a limit on the number of scraped targets that will be accepted. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -8387,7 +8424,7 @@ spec: type: integer terminationGracePeriodSeconds: description: |- - Optional duration in seconds the pod needs to terminate gracefully. + terminationGracePeriodSeconds defines the optional duration in seconds the pod needs to terminate gracefully. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down) which may lead to data corruption. @@ -8396,7 +8433,7 @@ spec: minimum: 0 type: integer tolerations: - description: Defines the Pods' tolerations if specified. + description: tolerations defines the Pods' tolerations if specified. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -8435,12 +8472,14 @@ spec: type: object type: array topologySpreadConstraints: - description: Defines the pod's topology spread constraints if specified. + description: topologySpreadConstraints defines the pod's topology + spread constraints if specified. items: properties: additionalLabelSelectors: - description: Defines what Prometheus Operator managed labels - should be added to labelSelector on the topologySpreadConstraint. + description: additionalLabelSelectors Defines what Prometheus + Operator managed labels should be added to labelSelector on + the topologySpreadConstraint. enum: - OnResource - OnShard @@ -8615,61 +8654,62 @@ spec: type: array tracingConfig: description: |- - TracingConfig configures tracing in Prometheus. + tracingConfig defines tracing in Prometheus. This is an *experimental feature*, it may change in any upcoming release in a breaking way. properties: clientType: - description: Client used to export the traces. Supported values - are `http` or `grpc`. + description: clientType defines the client used to export the + traces. Supported values are `http` or `grpc`. enum: - http - grpc type: string compression: - description: Compression key for supported compression types. + description: compression key for supported compression types. The only supported value is `gzip`. enum: - gzip type: string endpoint: - description: Endpoint to send the traces to. Should be provided + description: endpoint to send the traces to. Should be provided in format :. minLength: 1 type: string headers: additionalProperties: type: string - description: Key-value pairs to be used as headers associated - with gRPC or HTTP requests. + description: headers defines the key-value pairs to be used as + headers associated with gRPC or HTTP requests. type: object insecure: - description: If disabled, the client will use a secure connection. + description: insecure if disabled, the client will use a secure + connection. type: boolean samplingFraction: anyOf: - type: integer - type: string - description: Sets the probability a given trace will be sampled. - Must be a float from 0 through 1. + description: samplingFraction defines the probability a given + trace will be sampled. Must be a float from 0 through 1. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true timeout: - description: Maximum time the exporter will wait for each batch - export. + description: timeout defines the maximum time the exporter will + wait for each batch export. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS Config to use when sending traces. + description: tlsConfig to use when sending traces. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8692,7 +8732,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8717,15 +8758,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8748,7 +8790,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8773,19 +8816,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8810,7 +8854,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8821,7 +8865,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8831,7 +8875,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -8839,12 +8884,12 @@ spec: type: object tsdb: description: |- - Defines the runtime reloadable configuration of the timeseries database(TSDB). + tsdb defines the runtime reloadable configuration of the timeseries database(TSDB). It requires Prometheus >= v2.39.0 or PrometheusAgent >= v2.54.0. properties: outOfOrderTimeWindow: description: |- - Configures how old an out-of-order/out-of-bounds sample can be with + outOfOrderTimeWindow defines how old an out-of-order/out-of-bounds sample can be with respect to the TSDB max time. An out-of-order/out-of-bounds sample is ingested into the TSDB as long as @@ -8859,7 +8904,7 @@ spec: type: object version: description: |- - Version of Prometheus being deployed. The operator uses this information + version of Prometheus being deployed. The operator uses this information to generate the Prometheus StatefulSet + configuration files. If not specified, the operator assumes the latest upstream version of @@ -8868,7 +8913,7 @@ spec: type: string volumeMounts: description: |- - VolumeMounts allows the configuration of additional VolumeMounts. + volumeMounts allows the configuration of additional VolumeMounts. VolumeMounts will be appended to other VolumeMounts in the 'prometheus' container, that are generated as a result of StorageSpec objects. @@ -8936,7 +8981,7 @@ spec: type: array volumes: description: |- - Volumes allows the configuration of additional volumes on the output + volumes allows the configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: @@ -10823,29 +10868,30 @@ spec: type: array walCompression: description: |- - Configures compression of the write-ahead log (WAL) using Snappy. + walCompression defines the compression of the write-ahead log (WAL) using Snappy. WAL compression is enabled by default for Prometheus >= 2.20.0 Requires Prometheus v2.11.0 and above. type: boolean web: - description: Defines the configuration of the Prometheus web server. + description: web defines the configuration of the Prometheus web server. properties: httpConfig: - description: Defines HTTP parameters for web server. + description: httpConfig defines HTTP parameters for web server. properties: headers: - description: List of headers that can be added to HTTP responses. + description: headers defines a list of headers that can be + added to HTTP responses. properties: contentSecurityPolicy: description: |- - Set the Content-Security-Policy header to HTTP responses. + contentSecurityPolicy defines the Content-Security-Policy header to HTTP responses. Unset if blank. type: string strictTransportSecurity: description: |- - Set the Strict-Transport-Security header to HTTP responses. + strictTransportSecurity defines the Strict-Transport-Security header to HTTP responses. Unset if blank. Please make sure that you use this with care as this header might force browsers to load Prometheus and the other applications hosted on the same @@ -10854,7 +10900,7 @@ spec: type: string xContentTypeOptions: description: |- - Set the X-Content-Type-Options header to HTTP responses. + xContentTypeOptions defines the X-Content-Type-Options header to HTTP responses. Unset if blank. Accepted value is nosniff. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options enum: @@ -10863,7 +10909,7 @@ spec: type: string xFrameOptions: description: |- - Set the X-Frame-Options header to HTTP responses. + xFrameOptions defines the X-Frame-Options header to HTTP responses. Unset if blank. Accepted values are deny and sameorigin. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options enum: @@ -10873,42 +10919,42 @@ spec: type: string xXSSProtection: description: |- - Set the X-XSS-Protection header to all responses. + xXSSProtection defines the X-XSS-Protection header to all responses. Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection type: string type: object http2: description: |- - Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. + http2 enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. When TLSConfig is not configured, HTTP/2 will be disabled. Whenever the value of the field changes, a rolling update will be triggered. type: boolean type: object maxConnections: description: |- - Defines the maximum number of simultaneous connections + maxConnections defines the maximum number of simultaneous connections A zero value means that Prometheus doesn't accept any incoming connection. format: int32 minimum: 0 type: integer pageTitle: - description: The prometheus web page title. + description: pageTitle defines the prometheus web page title. type: string tlsConfig: - description: Defines the TLS parameters for HTTPS. + description: tlsConfig defines the TLS parameters for HTTPS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10931,7 +10977,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10957,7 +11004,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -10965,7 +11012,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -10975,14 +11022,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11005,7 +11052,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11031,21 +11079,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -11055,7 +11103,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -11063,7 +11111,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -11091,14 +11139,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -11124,18 +11174,19 @@ spec: rule: '!(has(self.mode) && self.mode == ''DaemonSet'' && has(self.probeSelector))' status: description: |- - Most recent observed status of the Prometheus cluster. Read-only. + status defines the most recent observed status of the Prometheus cluster. Read-only. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment. format: int32 type: integer conditions: - description: The current state of the Prometheus deployment. + description: conditions defines the current state of the Prometheus + deployment. items: description: |- Condition represents the state of the resources associated with the @@ -11147,12 +11198,12 @@ spec: format: date-time type: string message: - description: Human-readable message indicating details for the - condition's last transition. + description: message defines human-readable message indicating + details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the @@ -11160,14 +11211,14 @@ spec: format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: - description: Type of the condition being reported. + description: type of the condition being reported. minLength: 1 type: string required: @@ -11181,45 +11232,46 @@ spec: x-kubernetes-list-type: map paused: description: |- - Represents whether any actions on the underlying managed objects are + paused defines whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: |- - Total number of non-terminated pods targeted by this Prometheus deployment + replicas defines the total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector). format: int32 type: integer selector: - description: The selector used to match the pods targeted by this - Prometheus resource. + description: selector used to match the pods targeted by this Prometheus + resource. type: string shardStatuses: - description: The list has one entry per shard. Each entry provides - a summary of the shard status. + description: shardStatuses defines the list has one entry per shard. + Each entry provides a summary of the shard status. items: properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this shard. format: int32 type: integer replicas: - description: Total number of pods targeted by this shard. + description: replicas defines the total number of pods targeted + by this shard. format: int32 type: integer shardID: - description: Identifier of the shard. + description: shardID defines the identifier of the shard. type: string unavailableReplicas: - description: Total number of unavailable pods targeted by this - shard. + description: unavailableReplicas defines the Total number of + unavailable pods targeted by this shard. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this shard + updatedReplicas defines the total number of non-terminated pods targeted by this shard that have the desired spec. format: int32 type: integer @@ -11235,26 +11287,20 @@ spec: - shardID x-kubernetes-list-type: map shards: - description: Shards is the most recently observed number of shards. + description: shards defines the most recently observed number of shards. format: int32 type: integer unavailableReplicas: - description: Total number of unavailable pods targeted by this Prometheus - deployment. + description: unavailableReplicas defines the total number of unavailable + pods targeted by this Prometheus deployment. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this Prometheus deployment + updatedReplicas defines the total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec. format: int32 type: integer - required: - - availableReplicas - - paused - - replicas - - unavailableReplicas - - updatedReplicas type: object required: - spec diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheuses.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheuses.yaml index c5420605b20..0c2d581d96b 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheuses.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheuses.yaml @@ -76,12 +76,12 @@ spec: type: object spec: description: |- - Specification of the desired behavior of the Prometheus cluster. More info: + spec defines the specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: additionalAlertManagerConfigs: description: |- - AdditionalAlertManagerConfigs specifies a key of a Secret containing + additionalAlertManagerConfigs defines a key of a Secret containing additional Prometheus Alertmanager configurations. The Alertmanager configurations are appended to the configuration generated by the Prometheus Operator. They must be formatted according to the official @@ -118,7 +118,7 @@ spec: x-kubernetes-map-type: atomic additionalAlertRelabelConfigs: description: |- - AdditionalAlertRelabelConfigs specifies a key of a Secret containing + additionalAlertRelabelConfigs defines a key of a Secret containing additional Prometheus alert relabel configurations. The alert relabel configurations are appended to the configuration generated by the Prometheus Operator. They must be formatted according to the official @@ -155,7 +155,7 @@ spec: x-kubernetes-map-type: atomic additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the 'prometheus' container. + additionalArgs allows setting additional arguments for the 'prometheus' container. It is intended for e.g. activating hidden flags which are not supported by the dedicated configuration options yet. The arguments are passed as-is to the @@ -169,12 +169,12 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for name-only - arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. Can + be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name @@ -182,7 +182,7 @@ spec: type: array additionalScrapeConfigs: description: |- - AdditionalScrapeConfigs allows specifying a key of a Secret containing + additionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified @@ -215,7 +215,8 @@ spec: type: object x-kubernetes-map-type: atomic affinity: - description: Defines the Pods' affinity scheduling rules if specified. + description: affinity defines the Pods' affinity scheduling rules + if specified. properties: nodeAffinity: description: Describes node affinity scheduling rules for the @@ -1127,10 +1128,10 @@ spec: type: object type: object alerting: - description: Defines the settings related to Alertmanager. + description: alerting defines the settings related to Alertmanager. properties: alertmanagers: - description: Alertmanager endpoints where Prometheus should send + description: alertmanagers endpoints where Prometheus should send alerts to. items: description: |- @@ -1139,7 +1140,7 @@ spec: properties: alertRelabelings: description: |- - Relabeling configs applied before sending alerts to a specific Alertmanager. + alertRelabelings defines the relabeling configs applied before sending alerts to a specific Alertmanager. It requires Prometheus >= v2.51.0. items: description: |- @@ -1151,7 +1152,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -1183,41 +1184,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the - extracted value is matched. + description: regex defines the regular expression + against which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated - SourceLabels. + description: separator defines the string between + concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -1228,7 +1229,7 @@ spec: type: array apiVersion: description: |- - Version of the Alertmanager API that Prometheus uses to send alerts. + apiVersion defines the version of the Alertmanager API that Prometheus uses to send alerts. It can be "V1" or "V2". The field has no effect for Prometheus >= v3.0.0 because only the v2 API is supported. enum: @@ -1239,13 +1240,13 @@ spec: type: string authorization: description: |- - Authorization section for Alertmanager. + authorization section for Alertmanager. Cannot be set at the same time as `basicAuth`, `bearerTokenFile` or `sigv4`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in + the namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -1270,7 +1271,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1279,13 +1280,13 @@ spec: type: object basicAuth: description: |- - BasicAuth configuration for Alertmanager. + basicAuth configuration for Alertmanager. Cannot be set at the same time as `bearerTokenFile`, `authorization` or `sigv4`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1311,7 +1312,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1338,22 +1339,22 @@ spec: type: object bearerTokenFile: description: |- - File to read bearer token for Alertmanager. + bearerTokenFile defines the file to read bearer token for Alertmanager. Cannot be set at the same time as `basicAuth`, `authorization`, or `sigv4`. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string enableHttp2: - description: Whether to enable HTTP2. + description: enableHttp2 defines whether to enable HTTP2. type: boolean name: - description: Name of the Endpoints object in the namespace. + description: name of the Endpoints object in the namespace. minLength: 1 type: string namespace: description: |- - Namespace of the Endpoints object. + namespace of the Endpoints object. If not set, the object will be discovered in the namespace of the Prometheus object. @@ -1361,21 +1362,21 @@ spec: type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string pathPrefix: - description: Prefix for the HTTP path alerts are pushed - to. + description: pathPrefix defines the prefix for the HTTP + path alerts are pushed to. type: string port: anyOf: - type: integer - type: string - description: Port on which the Alertmanager API is exposed. + description: port on which the Alertmanager API is exposed. x-kubernetes-int-or-string: true proxyConnectHeader: additionalProperties: @@ -1405,7 +1406,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1413,18 +1414,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string relabelings: - description: Relabel configuration applied to the discovered - Alertmanagers. + description: relabelings defines the relabel configuration + applied to the discovered Alertmanagers. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -1435,7 +1435,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -1467,41 +1467,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the - extracted value is matched. + description: regex defines the regular expression + against which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated - SourceLabels. + description: separator defines the string between + concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -1511,11 +1511,11 @@ spec: type: object type: array scheme: - description: Scheme to use when firing alerts. + description: scheme to use when firing alerts. type: string sigv4: description: |- - Sigv4 allows to configures AWS's Signature Verification 4 for the URL. + sigv4 defines AWS's Signature Verification 4 for the URL. It requires Prometheus >= v2.48.0. @@ -1523,7 +1523,7 @@ spec: properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -1548,20 +1548,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used to - authenticate. + description: profile defines the named AWS profile used + to authenticate. type: string region: - description: Region is the AWS region. If blank, the - region from the default credentials chain used. + description: region defines the AWS region. If blank, + the region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used to - authenticate. + description: roleArn defines the named AWS profile used + to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -1585,22 +1585,27 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object timeout: - description: Timeout is a per-target Alertmanager timeout + description: timeout defines a per-target Alertmanager timeout when pushing alerts. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS Config to use for Alertmanager. + description: tlsConfig to use for Alertmanager. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1623,8 +1628,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1649,16 +1654,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert + in the Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1681,8 +1686,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1707,19 +1712,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client + cert file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client + key file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1744,7 +1750,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1755,7 +1761,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1765,7 +1771,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object required: @@ -1778,14 +1785,14 @@ spec: type: object allowOverlappingBlocks: description: |- - AllowOverlappingBlocks enables vertical compaction and vertical query + allowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. Deprecated: this flag has no effect for Prometheus >= 2.39.0 where overlapping blocks are enabled by default. type: boolean apiserverConfig: description: |- - APIServerConfig allows specifying a host and auth methods to access the + apiserverConfig allows specifying a host and auth methods to access the Kuberntees API server. If null, Prometheus is assumed to run inside of the cluster: it will discover the API servers automatically and use the Pod's CA certificate @@ -1793,14 +1800,14 @@ spec: properties: authorization: description: |- - Authorization section for the API server. + authorization section for the API server. Cannot be set at the same time as `basicAuth`, `bearerToken`, or `bearerTokenFile`. properties: credentials: - description: Selects a key of a Secret in the namespace that - contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -1824,12 +1831,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a secret + from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1838,14 +1845,14 @@ spec: type: object basicAuth: description: |- - BasicAuth configuration for the API server. + basicAuth configuration for the API server. Cannot be set at the same time as `authorization`, `bearerToken`, or `bearerTokenFile`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -1871,7 +1878,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -1898,14 +1905,13 @@ spec: type: object bearerToken: description: |- - *Warning: this field shouldn't be used because the token value appears + bearerToken is deprecated: this will be removed in a future release. + *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File to read bearer token for accessing apiserver. + bearerTokenFile defines the file to read bearer token for accessing apiserver. Cannot be set at the same time as `basicAuth`, `authorization`, or `bearerToken`. @@ -1913,12 +1919,12 @@ spec: type: string host: description: |- - Kubernetes API address consisting of a hostname or IP address followed + host defines the Kubernetes API address consisting of a hostname or IP address followed by an optional port number. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1952,7 +1958,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1960,24 +1966,24 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string tlsConfig: - description: TLS Config to use for the API server. + description: tlsConfig to use for the API server. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2000,7 +2006,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2025,15 +2032,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2056,7 +2064,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2081,19 +2090,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2118,7 +2128,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2129,7 +2139,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2139,7 +2149,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -2147,7 +2158,7 @@ spec: type: object arbitraryFSAccessThroughSMs: description: |- - When true, ServiceMonitor, PodMonitor and Probe object are forbidden to + arbitraryFSAccessThroughSMs when true, ServiceMonitor, PodMonitor and Probe object are forbidden to reference arbitrary files on the file system of the 'prometheus' container. When a ServiceMonitor's endpoint specifies a `bearerTokenFile` value @@ -2159,22 +2170,27 @@ spec: `spec.bearerTokenSecret` field. properties: deny: + description: |- + deny prevents service monitors from accessing arbitrary files on the file system. + When true, service monitors cannot use file-based configurations like BearerTokenFile + that could potentially access sensitive files. When false (default), such access is allowed. + Setting this to true enhances security by preventing potential credential theft attacks. type: boolean type: object automountServiceAccountToken: description: |- - AutomountServiceAccountToken indicates whether a service account token should be automatically mounted in the pod. + automountServiceAccountToken defines whether a service account token should be automatically mounted in the pod. If the field isn't set, the operator mounts the service account token by default. **Warning:** be aware that by default, Prometheus requires the service account token for Kubernetes service discovery. It is possible to use strategic merge patch to project the service account token into the 'prometheus' container. type: boolean baseImage: - description: 'Deprecated: use ''spec.image'' instead.' + description: 'baseImage is deprecated: use ''spec.image'' instead.' type: string bodySizeLimit: description: |- - BodySizeLimit defines per-scrape on response body size. + bodySizeLimit defines per-scrape on response body size. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -2183,7 +2199,7 @@ spec: type: string configMaps: description: |- - ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus + configMaps defines a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. Each ConfigMap is added to the StatefulSet definition as a volume named `configmap-`. The ConfigMaps are mounted into /etc/prometheus/configmaps/ in the 'prometheus' container. @@ -2192,7 +2208,7 @@ spec: type: array containers: description: |- - Containers allows injecting additional containers or modifying operator + containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to the Pods or to change the behavior of an operator generated container. Containers described here modify an operator generated @@ -3728,23 +3744,23 @@ spec: type: array convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.4.0. type: boolean disableCompaction: description: |- - When true, the Prometheus compaction is disabled. + disableCompaction when true, the Prometheus compaction is disabled. When `spec.thanos.objectStorageConfig` or `spec.objectStorageConfigFile` are defined, the operator automatically disables block compaction to avoid race conditions during block uploads (as the Thanos documentation recommends). type: boolean dnsConfig: - description: Defines the DNS configuration for the pods. + description: dnsConfig defines the DNS configuration for the pods. properties: nameservers: description: |- - A list of DNS name server IP addresses. + nameservers defines the list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. items: minLength: 1 @@ -3753,7 +3769,7 @@ spec: x-kubernetes-list-type: set options: description: |- - A list of DNS resolver options. + options defines the list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Resolution options given in Options will override those that appear in the base DNSPolicy. @@ -3762,11 +3778,11 @@ spec: of a pod. properties: name: - description: Name is required and must be unique. + description: name is required and must be unique. minLength: 1 type: string value: - description: Value is optional. + description: value is optional. type: string required: - name @@ -3777,7 +3793,7 @@ spec: x-kubernetes-list-type: map searches: description: |- - A list of DNS search domains for host-name lookup. + searches defines the list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. items: minLength: 1 @@ -3786,7 +3802,7 @@ spec: x-kubernetes-list-type: set type: object dnsPolicy: - description: Defines the DNS policy for the pods. + description: dnsPolicy defines the DNS policy for the pods. enum: - ClusterFirstWithHostNet - ClusterFirst @@ -3795,7 +3811,7 @@ spec: type: string enableAdminAPI: description: |- - Enables access to the Prometheus web admin API. + enableAdminAPI defines access to the Prometheus web admin API. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the @@ -3807,7 +3823,7 @@ spec: type: boolean enableFeatures: description: |- - Enable access to Prometheus feature flags. By default, no features are enabled. + enableFeatures enables access to Prometheus feature flags. By default, no features are enabled. Enabling features which are disabled by default is entirely outside the scope of what the maintainers will support and by doing so, you accept @@ -3821,7 +3837,7 @@ spec: x-kubernetes-list-type: set enableOTLPReceiver: description: |- - Enable Prometheus to be used as a receiver for the OTLP Metrics protocol. + enableOTLPReceiver defines the Prometheus to be used as a receiver for the OTLP Metrics protocol. Note that the OTLP receiver endpoint is automatically enabled if `.spec.otlpConfig` is defined. @@ -3829,7 +3845,7 @@ spec: type: boolean enableRemoteWriteReceiver: description: |- - Enable Prometheus to be used as a receiver for the Prometheus remote + enableRemoteWriteReceiver defines the Prometheus to be used as a receiver for the Prometheus remote write protocol. WARNING: This is not considered an efficient way of ingesting samples. @@ -3841,12 +3857,12 @@ spec: It requires Prometheus >= v2.33.0. type: boolean enableServiceLinks: - description: Indicates whether information about services should be - injected into pod's environment variables + description: enableServiceLinks defines whether information about + services should be injected into pod's environment variables type: boolean enforcedBodySizeLimit: description: |- - When defined, enforcedBodySizeLimit specifies a global limit on the size + enforcedBodySizeLimit when defined specifies a global limit on the size of uncompressed response body that will be accepted by Prometheus. Targets responding with a body larger than this many bytes will cause the scrape to fail. @@ -3862,7 +3878,7 @@ spec: type: string enforcedKeepDroppedTargets: description: |- - When defined, enforcedKeepDroppedTargets specifies a global limit on the number of targets + enforcedKeepDroppedTargets when defined specifies a global limit on the number of targets dropped by relabeling that will be kept in memory. The value overrides any `spec.keepDroppedTargets` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.keepDroppedTargets` is @@ -3879,7 +3895,7 @@ spec: type: integer enforcedLabelLimit: description: |- - When defined, enforcedLabelLimit specifies a global limit on the number + enforcedLabelLimit when defined specifies a global limit on the number of labels per sample. The value overrides any `spec.labelLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelLimit` is greater than zero and less than `spec.enforcedLabelLimit`. @@ -3895,7 +3911,7 @@ spec: type: integer enforcedLabelNameLengthLimit: description: |- - When defined, enforcedLabelNameLengthLimit specifies a global limit on the length + enforcedLabelNameLengthLimit when defined specifies a global limit on the length of labels name per sample. The value overrides any `spec.labelNameLengthLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelNameLengthLimit` is greater than zero and less than `spec.enforcedLabelNameLengthLimit`. @@ -3911,7 +3927,7 @@ spec: type: integer enforcedLabelValueLengthLimit: description: |- - When not null, enforcedLabelValueLengthLimit defines a global limit on the length + enforcedLabelValueLengthLimit when not null defines a global limit on the length of labels value per sample. The value overrides any `spec.labelValueLengthLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.labelValueLengthLimit` is greater than zero and less than `spec.enforcedLabelValueLengthLimit`. @@ -3927,7 +3943,7 @@ spec: type: integer enforcedNamespaceLabel: description: |- - When not empty, a label will be added to: + enforcedNamespaceLabel when not empty, a label will be added to: 1. All metrics scraped from `ServiceMonitor`, `PodMonitor`, `Probe` and `ScrapeConfig` objects. 2. All metrics generated from recording rules defined in `PrometheusRule` objects. @@ -3942,7 +3958,7 @@ spec: type: string enforcedSampleLimit: description: |- - When defined, enforcedSampleLimit specifies a global limit on the number + enforcedSampleLimit when defined specifies a global limit on the number of scraped samples that will be accepted. This overrides any `spec.sampleLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.sampleLimit` is greater than zero and less than @@ -3960,7 +3976,7 @@ spec: type: integer enforcedTargetLimit: description: |- - When defined, enforcedTargetLimit specifies a global limit on the number + enforcedTargetLimit when defined specifies a global limit on the number of scraped targets. The value overrides any `spec.targetLimit` set by ServiceMonitor, PodMonitor, Probe objects unless `spec.targetLimit` is greater than zero and less than `spec.enforcedTargetLimit`. @@ -3978,13 +3994,13 @@ spec: evaluationInterval: default: 30s description: |- - Interval between rule evaluations. + evaluationInterval defines the interval between rule evaluations. Default: "30s" pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string excludedFromEnforcement: description: |- - List of references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + excludedFromEnforcement defines the list of references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects to be excluded from enforcing a namespace label of origin. It is only applicable if `spec.enforcedNamespaceLabel` set to true. @@ -3994,23 +4010,23 @@ spec: properties: group: default: monitoring.coreos.com - description: Group of the referent. When not specified, it defaults + description: group of the referent. When not specified, it defaults to `monitoring.coreos.com` enum: - monitoring.coreos.com type: string name: - description: Name of the referent. When not set, all resources + description: name of the referent. When not set, all resources in the namespace are matched. type: string namespace: description: |- - Namespace of the referent. + namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ minLength: 1 type: string resource: - description: Resource of the referent. + description: resource of the referent. enum: - prometheusrules - servicemonitors @@ -4025,12 +4041,12 @@ spec: type: array exemplars: description: |- - Exemplars related settings that are runtime reloadable. + exemplars related settings that are runtime reloadable. It requires to enable the `exemplar-storage` feature flag to be effective. properties: maxSize: description: |- - Maximum number of exemplars stored in memory for all series. + maxSize defines the maximum number of exemplars stored in memory for all series. exemplar-storage itself must be enabled using the `spec.enableFeature` option for exemplars to be scraped in the first place. @@ -4044,20 +4060,20 @@ spec: additionalProperties: type: string description: |- - The labels to add to any time series or alerts when communicating with + externalLabels defines the labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager). Labels defined by `spec.replicaExternalLabelName` and `spec.prometheusExternalLabelName` take precedence over this list. type: object externalUrl: description: |- - The external URL under which the Prometheus service is externally + externalUrl defines the external URL under which the Prometheus service is externally available. This is necessary to generate correct URLs (for instance if Prometheus is accessible behind an Ingress resource). type: string hostAliases: description: |- - Optional list of hosts and IPs that will be injected into the Pod's + hostAliases defines the optional list of hosts and IPs that will be injected into the Pod's hosts file if specified. items: description: |- @@ -4065,12 +4081,12 @@ spec: pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. + description: hostnames defines hostnames for the above IP address. items: type: string type: array ip: - description: IP address of the host file entry. + description: ip defines the IP address of the host file entry. type: string required: - hostnames @@ -4082,7 +4098,7 @@ spec: x-kubernetes-list-type: map hostNetwork: description: |- - Use the host's network namespace if true. + hostNetwork defines the host's network namespace if true. Make sure to understand the security implications if you want to enable it (https://kubernetes.io/docs/concepts/configuration/overview/ ). @@ -4093,7 +4109,7 @@ spec: type: boolean hostUsers: description: |- - HostUsers supports the user space in Kubernetes. + hostUsers supports the user space in Kubernetes. More info: https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/ @@ -4102,14 +4118,14 @@ spec: type: boolean ignoreNamespaceSelectors: description: |- - When true, `spec.namespaceSelector` from all PodMonitor, ServiceMonitor + ignoreNamespaceSelectors when true, `spec.namespaceSelector` from all PodMonitor, ServiceMonitor and Probe objects will be ignored. They will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object. type: boolean image: description: |- - Container image name for Prometheus. If specified, it takes precedence + image defines the container image name for Prometheus. If specified, it takes precedence over the `spec.baseImage`, `spec.tag` and `spec.sha` fields. Specifying `spec.version` is still necessary to ensure the Prometheus @@ -4121,7 +4137,7 @@ spec: type: string imagePullPolicy: description: |- - Image pull policy for the 'prometheus', 'init-config-reloader' and 'config-reloader' containers. + imagePullPolicy defines the image pull policy for the 'prometheus', 'init-config-reloader' and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. enum: - "" @@ -4131,7 +4147,7 @@ spec: type: string imagePullSecrets: description: |- - An optional list of references to Secrets in the same namespace + imagePullSecrets defines an optional list of references to Secrets in the same namespace to use for pulling images from registries. See http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod items: @@ -4153,7 +4169,7 @@ spec: type: array initContainers: description: |- - InitContainers allows injecting initContainers to the Pod definition. Those + initContainers allows injecting initContainers to the Pod definition. Those can be used to e.g. fetch secrets for injection into the Prometheus configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: @@ -5689,7 +5705,7 @@ spec: type: array keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -5700,7 +5716,7 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -5709,7 +5725,7 @@ spec: type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -5718,7 +5734,7 @@ spec: type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -5727,11 +5743,11 @@ spec: type: integer listenLocal: description: |- - When true, the Prometheus server listens on the loopback address + listenLocal when true, the Prometheus server listens on the loopback address instead of the Pod IP's address. type: boolean logFormat: - description: Log format for Log level for Prometheus and the config-reloader + description: logFormat for Log level for Prometheus and the config-reloader sidecar. enum: - "" @@ -5739,7 +5755,7 @@ spec: - json type: string logLevel: - description: Log level for Prometheus and the config-reloader sidecar. + description: logLevel for Prometheus and the config-reloader sidecar. enum: - "" - debug @@ -5749,14 +5765,14 @@ spec: type: string maximumStartupDurationSeconds: description: |- - Defines the maximum time that the `prometheus` container's startup probe will wait before being considered failed. The startup probe will return success after the WAL replay is complete. - If set, the value should be greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 minutes). + maximumStartupDurationSeconds defines the maximum time that the `prometheus` container's startup probe will wait before being considered failed. The startup probe will return success after the WAL replay is complete. + If set, the value should be greater than 60 (seconds). Otherwise it will be equal to 900 seconds (15 minutes). format: int32 minimum: 60 type: integer minReadySeconds: description: |- - Minimum number of seconds for which a newly created Pod should be ready + minReadySeconds defines the minimum number of seconds for which a newly created Pod should be ready without any of its container crashing for it to be considered available. If unset, pods will be considered available as soon as they are ready. @@ -5765,7 +5781,7 @@ spec: type: integer nameEscapingScheme: description: |- - Specifies the character escaping scheme that will be requested when scraping + nameEscapingScheme defines the character escaping scheme that will be requested when scraping for metric and label names that do not conform to the legacy Prometheus character set. @@ -5778,7 +5794,7 @@ spec: type: string nameValidationScheme: description: |- - Specifies the validation scheme for metric and label names. + nameValidationScheme defines the validation scheme for metric and label names. It requires Prometheus >= v2.55.0. enum: @@ -5788,21 +5804,21 @@ spec: nodeSelector: additionalProperties: type: string - description: Defines on which Nodes the Pods are scheduled. + description: nodeSelector defines on which Nodes the Pods are scheduled. type: object otlp: description: |- - Settings related to the OTLP receiver feature. + otlp defines the settings related to the OTLP receiver feature. It requires Prometheus >= v2.55.0. properties: convertHistogramsToNHCB: description: |- - Configures optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. + convertHistogramsToNHCB defines optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. It requires Prometheus >= v3.4.0. type: boolean ignoreResourceAttributes: description: |- - List of OpenTelemetry resource attributes to ignore when `promoteAllResourceAttributes` is true. + ignoreResourceAttributes defines the list of OpenTelemetry resource attributes to ignore when `promoteAllResourceAttributes` is true. It requires `promoteAllResourceAttributes` to be true. It requires Prometheus >= v3.5.0. @@ -5814,21 +5830,21 @@ spec: x-kubernetes-list-type: set keepIdentifyingResourceAttributes: description: |- - Enables adding `service.name`, `service.namespace` and `service.instance.id` + keepIdentifyingResourceAttributes enables adding `service.name`, `service.namespace` and `service.instance.id` resource attributes to the `target_info` metric, on top of converting them into the `instance` and `job` labels. It requires Prometheus >= v3.1.0. type: boolean promoteAllResourceAttributes: description: |- - Promote all resource attributes to metric labels except the ones defined in `ignoreResourceAttributes`. + promoteAllResourceAttributes promotes all resource attributes to metric labels except the ones defined in `ignoreResourceAttributes`. Cannot be true when `promoteResourceAttributes` is defined. It requires Prometheus >= v3.5.0. type: boolean promoteResourceAttributes: description: |- - List of OpenTelemetry Attributes that should be promoted to metric labels, defaults to none. + promoteResourceAttributes defines the list of OpenTelemetry Attributes that should be promoted to metric labels, defaults to none. Cannot be defined when `promoteAllResourceAttributes` is true. items: minLength: 1 @@ -5836,39 +5852,46 @@ spec: minItems: 1 type: array x-kubernetes-list-type: set + promoteScopeMetadata: + description: |- + promoteScopeMetadata controls whether to promote OpenTelemetry scope metadata (i.e. name, version, schema URL, and attributes) to metric labels. + As per the OpenTelemetry specification, the aforementioned scope metadata should be identifying, i.e. made into metric labels. + It requires Prometheus >= v3.6.0. + type: boolean translationStrategy: description: |- - Configures how the OTLP receiver endpoint translates the incoming metrics. + translationStrategy defines how the OTLP receiver endpoint translates the incoming metrics. It requires Prometheus >= v3.0.0. enum: - NoUTF8EscapingWithSuffixes - UnderscoreEscapingWithSuffixes - NoTranslation + - UnderscoreEscapingWithoutSuffixes type: string type: object overrideHonorLabels: description: |- - When true, Prometheus resolves label conflicts by renaming the labels in the scraped data + overrideHonorLabels when true, Prometheus resolves label conflicts by renaming the labels in the scraped data to “exported_” for all targets created from ServiceMonitor, PodMonitor and ScrapeConfig objects. Otherwise the HonorLabels field of the service or pod monitor applies. - In practice,`overrideHonorLaels:true` enforces `honorLabels:false` + In practice,`OverrideHonorLabels:true` enforces `honorLabels:false` for all ServiceMonitor, PodMonitor and ScrapeConfig objects. type: boolean overrideHonorTimestamps: description: |- - When true, Prometheus ignores the timestamps for all the targets created + overrideHonorTimestamps when true, Prometheus ignores the timestamps for all the targets created from service and pod monitors. Otherwise the HonorTimestamps field of the service or pod monitor applies. type: boolean paused: description: |- - When a Prometheus deployment is paused, no actions except for deletion + paused defines when a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects. type: boolean persistentVolumeClaimRetentionPolicy: description: |- - The field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. + persistentVolumeClaimRetentionPolicy defines the field controls if and how PVCs are deleted during the lifecycle of a StatefulSet. The default behavior is all PVCs are retained. This is an alpha field from kubernetes 1.23 until 1.26 and a beta field from 1.26. It requires enabling the StatefulSetAutoDeletePVC feature gate. @@ -5891,7 +5914,7 @@ spec: type: object podMetadata: description: |- - PodMetadata configures labels and annotations which are propagated to the Prometheus pods. + podMetadata defines labels and annotations which are propagated to the Prometheus pods. The following items are reserved and cannot be overridden: * "prometheus" label, set to the name of the Prometheus object. @@ -5907,7 +5930,7 @@ spec: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -5916,14 +5939,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -5933,7 +5956,7 @@ spec: type: object podMonitorNamespaceSelector: description: |- - Namespaces to match for PodMonitors discovery. An empty label selector + podMonitorNamespaceSelector defines the namespaces to match for PodMonitors discovery. An empty label selector matches all namespaces. A null label selector (default value) matches the current namespace only. properties: @@ -5982,7 +6005,7 @@ spec: x-kubernetes-map-type: atomic podMonitorSelector: description: |- - PodMonitors to be selected for target discovery. An empty label selector + podMonitorSelector defines the podMonitors to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -6039,7 +6062,7 @@ spec: x-kubernetes-map-type: atomic podTargetLabels: description: |- - PodTargetLabels are appended to the `spec.podTargetLabels` field of all + podTargetLabels are appended to the `spec.podTargetLabels` field of all PodMonitor and ServiceMonitor objects. items: type: string @@ -6047,15 +6070,15 @@ spec: portName: default: web description: |- - Port name used for the pods and governing service. + portName used for the pods and governing service. Default: "web" type: string priorityClassName: - description: Priority class assigned to the Pods. + description: priorityClassName assigned to the Pods. type: string probeNamespaceSelector: description: |- - Namespaces to match for Probe discovery. An empty label + probeNamespaceSelector defines the namespaces to match for Probe discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. properties: @@ -6104,7 +6127,7 @@ spec: x-kubernetes-map-type: atomic probeSelector: description: |- - Probes to be selected for target discovery. An empty label selector + probeSelector defines the probes to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -6161,7 +6184,7 @@ spec: x-kubernetes-map-type: atomic prometheusExternalLabelName: description: |- - Name of Prometheus external label used to denote the Prometheus instance + prometheusExternalLabelName defines the name of Prometheus external label used to denote the Prometheus instance name. The external label will _not_ be added when the field is set to the empty string (`""`). @@ -6169,7 +6192,7 @@ spec: type: string prometheusRulesExcludedFromEnforce: description: |- - Defines the list of PrometheusRule objects to which the namespace label + prometheusRulesExcludedFromEnforce defines the list of PrometheusRule objects to which the namespace label enforcement doesn't apply. This is only relevant when `spec.enforcedNamespaceLabel` is set to true. Deprecated: use `spec.excludedFromEnforcement` instead. @@ -6180,10 +6203,12 @@ spec: namespace label for alerts and metrics. properties: ruleName: - description: Name of the excluded PrometheusRule object. + description: ruleName defines the name of the excluded PrometheusRule + object. type: string ruleNamespace: - description: Namespace of the excluded PrometheusRule object. + description: ruleNamespace defines the namespace of the excluded + PrometheusRule object. type: string required: - ruleName @@ -6191,27 +6216,29 @@ spec: type: object type: array query: - description: QuerySpec defines the configuration of the Promethus - query service. + description: query defines the configuration of the Prometheus query + service. properties: lookbackDelta: - description: The delta difference allowed for retrieving metrics - during expression evaluations. + description: lookbackDelta defines the delta difference allowed + for retrieving metrics during expression evaluations. type: string maxConcurrency: - description: Number of concurrent queries that can be run at once. + description: maxConcurrency defines the number of concurrent queries + that can be run at once. format: int32 minimum: 1 type: integer maxSamples: description: |- - Maximum number of samples a single query can load into memory. Note that + maxSamples defines the maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return. format: int32 type: integer timeout: - description: Maximum time a query may take before being aborted. + description: timeout defines the maximum time a query may take + before being aborted. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object @@ -6231,14 +6258,14 @@ spec: type: string reloadStrategy: description: |- - Defines the strategy used to reload the Prometheus configuration. + reloadStrategy defines the strategy used to reload the Prometheus configuration. If not specified, the configuration is reloaded using the /-/reload HTTP endpoint. enum: - HTTP - ProcessSignal type: string remoteRead: - description: Defines the list of remote read configurations. + description: remoteRead defines the list of remote read configurations. items: description: |- RemoteReadSpec defines the configuration for Prometheus to read back samples @@ -6246,15 +6273,15 @@ spec: properties: authorization: description: |- - Authorization section for the URL. + authorization section for the URL. It requires Prometheus >= v2.26.0. Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -6278,12 +6305,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6292,13 +6319,13 @@ spec: type: object basicAuth: description: |- - BasicAuth configuration for the URL. + basicAuth configuration for the URL. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -6324,7 +6351,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -6351,26 +6378,25 @@ spec: type: object bearerToken: description: |- + bearerToken is deprecated: this will be removed in a future release. *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File from which to read the bearer token for the URL. + bearerTokenFile defines the file from which to read the bearer token for the URL. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string filterExternalLabels: description: |- - Whether to use the external labels as selectors for the remote read endpoint. + filterExternalLabels defines whether to use the external labels as selectors for the remote read endpoint. It requires Prometheus >= v2.34.0. type: boolean followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.26.0. type: boolean @@ -6378,13 +6404,13 @@ spec: additionalProperties: type: string description: |- - Custom HTTP headers to be sent along with each remote read request. + headers defines the custom HTTP headers to be sent along with each remote read request. Be aware that headers that are set by Prometheus itself can't be overwritten. Only valid in Prometheus versions 2.26.0 and newer. type: object name: description: |- - The name of the remote read queue, it must be unique if specified. The + name of the remote read queue, it must be unique if specified. The name is used in metrics and logging in order to differentiate read configurations. @@ -6392,7 +6418,7 @@ spec: type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6400,7 +6426,7 @@ spec: type: string oauth2: description: |- - OAuth2 configuration for the URL. + oauth2 configuration for the URL. It requires Prometheus >= v2.27.0. @@ -6408,12 +6434,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6436,7 +6462,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6462,7 +6489,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -6490,12 +6517,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6529,7 +6556,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6537,33 +6564,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6586,8 +6612,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -6612,12 +6638,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6640,8 +6666,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -6666,11 +6692,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6695,7 +6722,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6706,7 +6733,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6716,12 +6743,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -6757,7 +6785,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6765,40 +6793,41 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string readRecent: description: |- - Whether reads should be made for queries for time ranges that + readRecent defines whether reads should be made for queries for time ranges that the local storage should have complete data for. type: boolean remoteTimeout: - description: Timeout for requests to the remote read endpoint. + description: remoteTimeout defines the timeout for requests + to the remote read endpoint. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string requiredMatchers: additionalProperties: type: string description: |- - An optional list of equality matchers which have to be present + requiredMatchers defines an optional list of equality matchers which have to be present in a selector to query the remote read endpoint. type: object tlsConfig: - description: TLS Config to use for the URL. + description: tlsConfig to use for the URL. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6821,7 +6850,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6846,15 +6876,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6877,7 +6908,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6902,19 +6934,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6939,7 +6972,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6950,7 +6983,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6960,18 +6993,19 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the endpoint to query from. + description: url defines the URL of the endpoint to query from. type: string required: - url type: object type: array remoteWrite: - description: Defines the list of remote write configurations. + description: remoteWrite defines the list of remote write configurations. items: description: |- RemoteWriteSpec defines the configuration to write samples from Prometheus @@ -6979,15 +7013,15 @@ spec: properties: authorization: description: |- - Authorization section for the URL. + authorization section for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. Cannot be set at the same time as `sigv4`, `basicAuth`, `oauth2`, or `azureAd`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -7011,12 +7045,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -7025,15 +7059,15 @@ spec: type: object azureAd: description: |- - AzureAD for the URL. + azureAd for the URL. It requires Prometheus >= v2.45.0 or Thanos >= v0.31.0. Cannot be set at the same time as `authorization`, `basicAuth`, `oauth2`, or `sigv4`. properties: cloud: - description: The Azure Cloud. Options are 'AzurePublic', - 'AzureChina', or 'AzureGovernment'. + description: cloud defines the Azure Cloud. Options are + 'AzurePublic', 'AzureChina', or 'AzureGovernment'. enum: - AzureChina - AzureGovernment @@ -7041,32 +7075,33 @@ spec: type: string managedIdentity: description: |- - ManagedIdentity defines the Azure User-assigned Managed identity. + managedIdentity defines the Azure User-assigned Managed identity. Cannot be set at the same time as `oauth` or `sdk`. properties: clientId: - description: The client id + description: clientId defines defines the Azure User-assigned + Managed identity. type: string required: - clientId type: object oauth: description: |- - OAuth defines the oauth config that is being used to authenticate. + oauth defines the oauth config that is being used to authenticate. Cannot be set at the same time as `managedIdentity` or `sdk`. It requires Prometheus >= v2.48.0 or Thanos >= v0.31.0. properties: clientId: - description: '`clientID` is the clientId of the Azure + description: clientId defines the clientId of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 type: string clientSecret: - description: '`clientSecret` specifies a key of a Secret + description: clientSecret specifies a key of a Secret containing the client secret of the Azure Active Directory - application that is being used to authenticate.' + application that is being used to authenticate. properties: key: description: The key of the secret to select from. Must @@ -7090,9 +7125,9 @@ spec: type: object x-kubernetes-map-type: atomic tenantId: - description: '`tenantId` is the tenant ID of the Azure + description: tenantId is the tenant ID of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 pattern: ^[0-9a-zA-Z-.]+$ type: string @@ -7103,29 +7138,29 @@ spec: type: object sdk: description: |- - SDK defines the Azure SDK config that is being used to authenticate. + sdk defines the Azure SDK config that is being used to authenticate. See https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication Cannot be set at the same time as `oauth` or `managedIdentity`. It requires Prometheus >= v2.52.0 or Thanos >= v0.36.0. properties: tenantId: - description: '`tenantId` is the tenant ID of the azure + description: tenantId defines the tenant ID of the azure active directory application that is being used to - authenticate.' + authenticate. pattern: ^[0-9a-zA-Z-.]+$ type: string type: object type: object basicAuth: description: |- - BasicAuth configuration for the URL. + basicAuth configuration for the URL. Cannot be set at the same time as `sigv4`, `authorization`, `oauth2`, or `azureAd`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -7151,7 +7186,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -7178,23 +7213,22 @@ spec: type: object bearerToken: description: |- + bearerToken is deprecated: this will be removed in a future release. *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File from which to read bearer token for the URL. + bearerTokenFile defines the file from which to read bearer token for the URL. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. type: boolean @@ -7202,14 +7236,14 @@ spec: additionalProperties: type: string description: |- - Custom HTTP headers to be sent along with each remote write request. + headers defines the custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. It requires Prometheus >= v2.25.0 or Thanos >= v0.24.0. type: object messageVersion: description: |- - The Remote Write message's version to use when writing to the endpoint. + messageVersion defines the Remote Write message's version to use when writing to the endpoint. `Version1.0` corresponds to the `prometheus.WriteRequest` protobuf message introduced in Remote Write 1.0. `Version2.0` corresponds to the `io.prometheus.write.v2.Request` protobuf message introduced in Remote Write 2.0. @@ -7226,37 +7260,37 @@ spec: - V2.0 type: string metadataConfig: - description: MetadataConfig configures the sending of series - metadata to the remote storage. + description: metadataConfig defines how to send a series metadata + to the remote storage. properties: maxSamplesPerSend: description: |- - MaxSamplesPerSend is the maximum number of metadata samples per send. + maxSamplesPerSend defines the maximum number of metadata samples per send. It requires Prometheus >= v2.29.0. format: int32 minimum: -1 type: integer send: - description: Defines whether metric metadata is sent to - the remote storage or not. + description: send defines whether metric metadata is sent + to the remote storage or not. type: boolean sendInterval: - description: Defines how frequently metric metadata is sent - to the remote storage. + description: sendInterval defines how frequently metric + metadata is sent to the remote storage. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object name: description: |- - The name of the remote write queue, it must be unique if specified. The + name of the remote write queue, it must be unique if specified. The name is used in metrics and logging in order to differentiate queues. It requires Prometheus >= v2.15.0 or Thanos >= 0.24.0. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7264,7 +7298,7 @@ spec: type: string oauth2: description: |- - OAuth2 configuration for the URL. + oauth2 configuration for the URL. It requires Prometheus >= v2.27.0 or Thanos >= v0.24.0. @@ -7272,12 +7306,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7300,7 +7334,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7326,7 +7361,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -7354,12 +7389,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7393,7 +7428,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7401,33 +7436,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7450,8 +7484,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7476,12 +7510,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7504,8 +7538,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7530,11 +7564,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7559,7 +7594,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7570,7 +7605,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7580,12 +7615,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -7621,7 +7657,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7629,89 +7665,88 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string queueConfig: - description: QueueConfig allows tuning of the remote write queue + description: queueConfig allows tuning of the remote write queue parameters. properties: batchSendDeadline: - description: BatchSendDeadline is the maximum time a sample - will wait in buffer. + description: batchSendDeadline defines the maximum time + a sample will wait in buffer. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string capacity: description: |- - Capacity is the number of samples to buffer per shard before we start + capacity defines the number of samples to buffer per shard before we start dropping them. type: integer maxBackoff: - description: MaxBackoff is the maximum retry delay. + description: maxBackoff defines the maximum retry delay. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string maxRetries: - description: MaxRetries is the maximum number of times to - retry a batch on recoverable errors. + description: maxRetries defines the maximum number of times + to retry a batch on recoverable errors. type: integer maxSamplesPerSend: - description: MaxSamplesPerSend is the maximum number of - samples per send. + description: maxSamplesPerSend defines the maximum number + of samples per send. type: integer maxShards: - description: MaxShards is the maximum number of shards, + description: maxShards defines the maximum number of shards, i.e. amount of concurrency. type: integer minBackoff: - description: MinBackoff is the initial retry delay. Gets - doubled for every retry. + description: minBackoff defines the initial retry delay. + Gets doubled for every retry. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string minShards: - description: MinShards is the minimum number of shards, + description: minShards defines the minimum number of shards, i.e. amount of concurrency. type: integer retryOnRateLimit: description: |- - Retry upon receiving a 429 status code from the remote-write storage. + retryOnRateLimit defines the retry upon receiving a 429 status code from the remote-write storage. This is an *experimental feature*, it may change in any upcoming release in a breaking way. type: boolean sampleAgeLimit: description: |- - SampleAgeLimit drops samples older than the limit. + sampleAgeLimit drops samples older than the limit. It requires Prometheus >= v2.50.0 or Thanos >= v0.32.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object remoteTimeout: - description: Timeout for requests to the remote write endpoint. + description: remoteTimeout defines the timeout for requests + to the remote write endpoint. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string roundRobinDNS: - description: |- - When enabled: - - The remote-write mechanism will resolve the hostname via DNS. - - It will randomly select one of the resolved IP addresses and connect to it. - - When disabled (default behavior): - - The Go standard library will handle hostname resolution. - - It will attempt connections to each resolved IP address sequentially. - - Note: The connection timeout applies to the entire resolution and connection process. - If disabled, the timeout is distributed across all connection attempts. - - It requires Prometheus >= v3.1.0 or Thanos >= v0.38.0. + description: "roundRobinDNS controls the DNS resolution behavior + for remote-write connections.\nWhen enabled:\n - The remote-write + mechanism will resolve the hostname via DNS.\n - It will + randomly select one of the resolved IP addresses and connect + to it.\n\nWhen disabled (default behavior):\n - The Go standard + library will handle hostname resolution.\n - It will attempt + connections to each resolved IP address sequentially.\n\nNote: + The connection timeout applies to the entire resolution and + connection process.\n\n\tIf disabled, the timeout is distributed + across all connection attempts.\n\nIt requires Prometheus + >= v3.1.0 or Thanos >= v0.38.0." type: boolean sendExemplars: description: |- - Enables sending of exemplars over remote write. Note that + sendExemplars enables sending of exemplars over remote write. Note that exemplar-storage itself must be enabled using the `spec.enableFeatures` option for exemplars to be scraped in the first place. @@ -7719,14 +7754,14 @@ spec: type: boolean sendNativeHistograms: description: |- - Enables sending of native histograms, also known as sparse histograms + sendNativeHistograms enables sending of native histograms, also known as sparse histograms over remote write. It requires Prometheus >= v2.40.0 or Thanos >= v0.30.0. type: boolean sigv4: description: |- - Sigv4 allows to configures AWS's Signature Verification 4 for the URL. + sigv4 defines the AWS's Signature Verification 4 for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. @@ -7734,7 +7769,7 @@ spec: properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -7759,18 +7794,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used to authenticate. + description: profile defines the named AWS profile used + to authenticate. type: string region: - description: Region is the AWS region. If blank, the region - from the default credentials chain used. + description: region defines the AWS region. If blank, the + region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used to authenticate. + description: roleArn defines the named AWS profile used + to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -7794,17 +7831,22 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object tlsConfig: - description: TLS Config to use for the URL. + description: tlsConfig to use for the URL. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7827,7 +7869,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7852,15 +7895,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7883,7 +7927,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7908,19 +7953,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7945,7 +7991,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7956,7 +8002,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7966,15 +8012,18 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the endpoint to send samples to. + description: url defines the URL of the endpoint to send samples + to. minLength: 1 type: string writeRelabelConfigs: - description: The list of remote write relabel configurations. + description: writeRelabelConfigs defines the list of remote + write relabel configurations. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -7985,7 +8034,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -8017,41 +8066,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -8066,7 +8115,7 @@ spec: type: array remoteWriteReceiverMessageVersions: description: |- - List of the protobuf message versions to accept when receiving the + remoteWriteReceiverMessageVersions list of the protobuf message versions to accept when receiving the remote writes. It requires Prometheus >= v2.54.0. @@ -8080,7 +8129,7 @@ spec: x-kubernetes-list-type: set replicaExternalLabelName: description: |- - Name of Prometheus external label used to denote the replica name. + replicaExternalLabelName defines the name of Prometheus external label used to denote the replica name. The external label will _not_ be added when the field is set to the empty string (`""`). @@ -8088,7 +8137,7 @@ spec: type: string replicas: description: |- - Number of replicas of each shard to deploy for a Prometheus deployment. + replicas defines the number of replicas of each shard to deploy for a Prometheus deployment. `spec.replicas` multiplied by `spec.shards` is the total number of Pods created. @@ -8096,8 +8145,8 @@ spec: format: int32 type: integer resources: - description: Defines the resources requests and limits of the 'prometheus' - container. + description: resources defines the resources requests and limits of + the 'prometheus' container. properties: claims: description: |- @@ -8157,18 +8206,19 @@ spec: type: object retention: description: |- - How long to retain the Prometheus data. + retention defines how long to retain the Prometheus data. Default: "24h" if `spec.retention` and `spec.retentionSize` are empty. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string retentionSize: - description: Maximum number of bytes used by the Prometheus data. + description: retentionSize defines the maximum number of bytes used + by the Prometheus data. pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ type: string routePrefix: description: |- - The route prefix Prometheus registers HTTP handlers for. + routePrefix defines the route prefix Prometheus registers HTTP handlers for. This is useful when using `spec.externalURL`, and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but @@ -8177,7 +8227,7 @@ spec: type: string ruleNamespaceSelector: description: |- - Namespaces to match for PrometheusRule discovery. An empty label selector + ruleNamespaceSelector defines the namespaces to match for PrometheusRule discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. properties: @@ -8226,13 +8276,13 @@ spec: x-kubernetes-map-type: atomic ruleQueryOffset: description: |- - Defines the offset the rule evaluation timestamp of this particular group by the specified duration into the past. + ruleQueryOffset defines the offset the rule evaluation timestamp of this particular group by the specified duration into the past. It requires Prometheus >= v2.53.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string ruleSelector: description: |- - PrometheusRule objects to be selected for rule evaluation. An empty + ruleSelector defines the prometheusRule objects to be selected for rule evaluation. An empty label selector matches all objects. A null label selector matches no objects. properties: @@ -8280,40 +8330,41 @@ spec: type: object x-kubernetes-map-type: atomic rules: - description: Defines the configuration of the Prometheus rules' engine. + description: rules defines the configuration of the Prometheus rules' + engine. properties: alert: description: |- - Defines the parameters of the Prometheus rules' engine. + alert defines the parameters of the Prometheus rules' engine. Any update to these parameters trigger a restart of the pods. properties: forGracePeriod: description: |- - Minimum duration between alert and restored 'for' state. + forGracePeriod defines the minimum duration between alert and restored 'for' state. This is maintained only for alerts with a configured 'for' time greater than the grace period. type: string forOutageTolerance: description: |- - Max time to tolerate prometheus outage for restoring 'for' state of + forOutageTolerance defines the max time to tolerate prometheus outage for restoring 'for' state of alert. type: string resendDelay: description: |- - Minimum amount of time to wait before resending an alert to + resendDelay defines the minimum amount of time to wait before resending an alert to Alertmanager. type: string type: object type: object runtime: - description: RuntimeConfig configures the values for the Prometheus - process behavior + description: runtime defines the values for the Prometheus process + behavior properties: goGC: description: |- - The Go garbage collection target percentage. Lowering this number may increase the CPU usage. + goGC defines the Go garbage collection target percentage. Lowering this number may increase the CPU usage. See: https://tip.golang.org/doc/gc-guide#GOGC format: int32 minimum: -1 @@ -8321,7 +8372,7 @@ spec: type: object sampleLimit: description: |- - SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit defines per-scrape limit on number of scraped samples that will be accepted. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -8330,7 +8381,7 @@ spec: type: integer scrapeClasses: description: |- - List of scrape classes to expose to scraping objects such as + scrapeClasses defines the list of scrape classes to expose to scraping objects such as PodMonitors, ServiceMonitors, Probes and ScrapeConfigs. This is an *experimental feature*, it may change in any upcoming release @@ -8339,13 +8390,13 @@ spec: properties: attachMetadata: description: |- - AttachMetadata configures additional metadata to the discovered targets. + attachMetadata defines additional metadata to the discovered targets. When the scrape object defines its own configuration, it takes precedence over the scrape class configuration. properties: node: description: |- - When set to true, Prometheus attaches node metadata to the discovered + node when set to true, Prometheus attaches node metadata to the discovered targets. The Prometheus service account must have the `list` and `watch` @@ -8354,12 +8405,12 @@ spec: type: object authorization: description: |- - Authorization section for the ScrapeClass. + authorization section for the ScrapeClass. It will only apply if the scrape resource doesn't specify any Authorization. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -8383,12 +8434,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -8397,14 +8448,14 @@ spec: type: object default: description: |- - Default indicates that the scrape applies to all scrape objects that + default defines that the scrape applies to all scrape objects that don't configure an explicit scrape class name. Only one scrape class can be set as the default. type: boolean fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It will only apply if the scrape resource doesn't specify any FallbackScrapeProtocol It requires Prometheus >= v3.0.0. @@ -8417,7 +8468,7 @@ spec: type: string metricRelabelings: description: |- - MetricRelabelings configures the relabeling rules to apply to all samples before ingestion. + metricRelabelings defines the relabeling rules to apply to all samples before ingestion. The Operator adds the scrape class metric relabelings defined here. Then the Operator adds the target-specific metric relabelings defined in ServiceMonitors, PodMonitors, Probes and ScrapeConfigs. @@ -8434,7 +8485,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -8466,41 +8517,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -8510,12 +8561,12 @@ spec: type: object type: array name: - description: Name of the scrape class. + description: name of the scrape class. minLength: 1 type: string relabelings: description: |- - Relabelings configures the relabeling rules to apply to all scrape targets. + relabelings defines the relabeling rules to apply to all scrape targets. The Operator automatically adds relabelings for a few standard Kubernetes fields like `__meta_kubernetes_namespace` and `__meta_kubernetes_service_name`. @@ -8533,7 +8584,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -8565,41 +8616,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -8610,19 +8661,19 @@ spec: type: array tlsConfig: description: |- - TLSConfig defines the TLS settings to use for the scrape. When the + tlsConfig defines the TLS settings to use for the scrape. When the scrape objects define their own CA, certificate and/or key, they take precedence over the corresponding scrape class fields. For now only the `caFile`, `certFile` and `keyFile` fields are supported. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8645,7 +8696,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8670,15 +8722,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8701,7 +8754,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8726,19 +8780,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8763,7 +8818,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8774,7 +8829,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8784,7 +8839,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -8796,7 +8852,7 @@ spec: x-kubernetes-list-type: map scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. @@ -8804,7 +8860,7 @@ spec: type: boolean scrapeConfigNamespaceSelector: description: |- - Namespaces to match for ScrapeConfig discovery. An empty label selector + scrapeConfigNamespaceSelector defines the namespaces to match for ScrapeConfig discovery. An empty label selector matches all namespaces. A null label selector matches the current namespace only. @@ -8855,7 +8911,7 @@ spec: x-kubernetes-map-type: atomic scrapeConfigSelector: description: |- - ScrapeConfigs to be selected for target discovery. An empty label + scrapeConfigSelector defines the scrapeConfigs to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -8914,7 +8970,7 @@ spec: x-kubernetes-map-type: atomic scrapeFailureLogFile: description: |- - File to which scrape failures are logged. + scrapeFailureLogFile defines the file to which scrape failures are logged. Reloading the configuration will reopen the file. If the filename has an empty path, e.g. 'file.log', The Prometheus Pods @@ -8927,14 +8983,14 @@ spec: scrapeInterval: default: 30s description: |- - Interval between consecutive scrapes. + scrapeInterval defines interval between consecutive scrapes. Default: "30s" pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string scrapeProtocols: description: |- - The protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -8962,13 +9018,13 @@ spec: x-kubernetes-list-type: set scrapeTimeout: description: |- - Number of seconds to wait until a scrape request times out. + scrapeTimeout defines the number of seconds to wait until a scrape request times out. The value cannot be greater than the scrape interval otherwise the operator will reject the resource. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string secrets: description: |- - Secrets is a list of Secrets in the same namespace as the Prometheus + secrets defines a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. Each Secret is added to the StatefulSet definition as a volume named `secret-`. The Secrets are mounted into /etc/prometheus/secrets/ in the 'prometheus' container. @@ -8978,7 +9034,7 @@ spec: x-kubernetes-list-type: set securityContext: description: |- - SecurityContext holds pod-level security attributes and common container settings. + securityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: appArmorProfile: @@ -9212,12 +9268,12 @@ spec: type: object serviceAccountName: description: |- - ServiceAccountName is the name of the ServiceAccount to use to run the + serviceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods. type: string serviceDiscoveryRole: description: |- - Defines the service discovery role used to discover targets from + serviceDiscoveryRole defines the service discovery role used to discover targets from `ServiceMonitor` objects and Alertmanager endpoints. If set, the value should be either "Endpoints" or "EndpointSlice". @@ -9228,7 +9284,7 @@ spec: type: string serviceMonitorNamespaceSelector: description: |- - Namespaces to match for ServicedMonitors discovery. An empty label selector + serviceMonitorNamespaceSelector defines the namespaces to match for ServicedMonitors discovery. An empty label selector matches all namespaces. A null label selector (default value) matches the current namespace only. properties: @@ -9277,7 +9333,7 @@ spec: x-kubernetes-map-type: atomic serviceMonitorSelector: description: |- - ServiceMonitors to be selected for target discovery. An empty label + serviceMonitorSelector defines the serviceMonitors to be selected for target discovery. An empty label selector matches all objects. A null label selector matches no objects. If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`, `spec.probeSelector` @@ -9334,7 +9390,7 @@ spec: x-kubernetes-map-type: atomic serviceName: description: |- - The name of the service name used by the underlying StatefulSet(s) as the governing service. + serviceName defines the name of the service name used by the underlying StatefulSet(s) as the governing service. If defined, the Service must be created before the Prometheus/PrometheusAgent resource in the same namespace and it must define a selector that matches the pod labels. If empty, the operator will create and manage a headless service named `prometheus-operated` for Prometheus resources, or `prometheus-agent-operated` for PrometheusAgent resources. @@ -9343,12 +9399,12 @@ spec: minLength: 1 type: string sha: - description: 'Deprecated: use ''spec.image'' instead. The image''s + description: 'sha is deprecated: use ''spec.image'' instead. The image''s digest can be specified as part of the image name.' type: string shardRetentionPolicy: description: |- - ShardRetentionPolicy defines the retention policy for the Prometheus shards. + shardRetentionPolicy defines the retention policy for the Prometheus shards. (Alpha) Using this field requires the 'PrometheusShardRetentionPolicy' feature gate to be enabled. The final goals for this feature can be seen at https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/proposals/202310-shard-autoscaling.md#graceful-scale-down-of-prometheus-servers, @@ -9357,14 +9413,12 @@ spec: properties: retain: description: |- - Defines the config for retention when the retention policy is set to `Retain`. + retain defines the config for retention when the retention policy is set to `Retain`. This field is ineffective as of now. properties: retentionPeriod: - description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` + description: retentionPeriod defines the retentionPeriod for + shard retention policy. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string required: @@ -9372,7 +9426,7 @@ spec: type: object whenScaled: description: |- - Defines the retention policy when the Prometheus shards are scaled down. + whenScaled defines the retention policy when the Prometheus shards are scaled down. * `Delete`, the operator will delete the pods from the scaled-down shard(s). * `Retain`, the operator will keep the pods from the scaled-down shard(s), so the data can still be queried. @@ -9384,7 +9438,7 @@ spec: type: object shards: description: |- - Number of shards to distribute the scraped targets onto. + shards defines the number of shards to distribute the scraped targets onto. `spec.replicas` multiplied by `spec.shards` is the total number of Pods being created. @@ -9413,15 +9467,15 @@ spec: format: int32 type: integer storage: - description: Storage defines the storage used by Prometheus. + description: storage defines the storage used by Prometheus. properties: disableMountSubPath: - description: 'Deprecated: subPath usage will be removed in a future - release.' + description: 'disableMountSubPath deprecated: subPath usage will + be removed in a future release.' type: boolean emptyDir: description: |- - EmptyDirVolumeSource to be used by the StatefulSet. + emptyDir to be used by the StatefulSet. If specified, it takes precedence over `ephemeral` and `volumeClaimTemplate`. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir properties: @@ -9448,7 +9502,7 @@ spec: type: object ephemeral: description: |- - EphemeralVolumeSource to be used by the StatefulSet. + ephemeral to be used by the StatefulSet. This is a beta field in k8s 1.21 and GA in 1.15. For lower versions, starting with k8s 1.19, it requires enabling the GenericEphemeralVolume feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes @@ -9693,7 +9747,7 @@ spec: type: object volumeClaimTemplate: description: |- - Defines the PVC spec to be used by the Prometheus StatefulSets. + volumeClaimTemplate defines the PVC spec to be used by the Prometheus StatefulSets. The easiest way to use a volume that cannot be automatically provisioned is to use a label selector alongside manually created PersistentVolumes. properties: @@ -9713,14 +9767,14 @@ spec: More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: - description: EmbeddedMetadata contains metadata relevant to - an EmbeddedResource. + description: metadata defines EmbeddedMetadata contains metadata + relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -9729,14 +9783,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -9746,7 +9800,7 @@ spec: type: object spec: description: |- - Defines the desired characteristics of a volume requested by a pod author. + spec defines the specification of the characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: accessModes: @@ -9942,7 +9996,7 @@ spec: type: string type: object status: - description: 'Deprecated: this field is never set.' + description: 'status is deprecated: this field is never set.' properties: accessModes: description: |- @@ -10126,12 +10180,12 @@ spec: type: object type: object tag: - description: 'Deprecated: use ''spec.image'' instead. The image''s + description: 'tag is deprecated: use ''spec.image'' instead. The image''s tag can be specified as part of the image name.' type: string targetLimit: description: |- - TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit defines a limit on the number of scraped targets that will be accepted. Only valid in Prometheus versions 2.45.0 and newer. Note that the global limit only applies to scrape objects that don't specify an explicit limit value. @@ -10140,7 +10194,7 @@ spec: type: integer terminationGracePeriodSeconds: description: |- - Optional duration in seconds the pod needs to terminate gracefully. + terminationGracePeriodSeconds defines the optional duration in seconds the pod needs to terminate gracefully. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down) which may lead to data corruption. @@ -10149,11 +10203,12 @@ spec: minimum: 0 type: integer thanos: - description: Defines the configuration of the optional Thanos sidecar. + description: thanos defines the configuration of the optional Thanos + sidecar. properties: additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the Thanos container. + additionalArgs allows setting additional arguments for the Thanos container. The arguments are passed as-is to the Thanos container which may cause issues if they are invalid or not supported the given Thanos version. In case of an argument conflict (e.g. an argument which is already set by the @@ -10163,24 +10218,24 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for - name-only arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. + Can be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name type: object type: array baseImage: - description: 'Deprecated: use ''image'' instead.' + description: 'baseImage is deprecated: use ''image'' instead.' type: string blockSize: default: 2h description: |- - BlockDuration controls the size of TSDB blocks produced by Prometheus. + blockSize controls the size of TSDB blocks produced by Prometheus. The default value is 2h to match the upstream Prometheus defaults. WARNING: Changing the block duration can impact the performance and @@ -10191,34 +10246,35 @@ spec: pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string getConfigInterval: - description: How often to retrieve the Prometheus configuration. + description: getConfigInterval defines how often to retrieve the + Prometheus configuration. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string getConfigTimeout: - description: Maximum time to wait when retrieving the Prometheus - configuration. + description: getConfigTimeout defines the maximum time to wait + when retrieving the Prometheus configuration. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string grpcListenLocal: description: |- - When true, the Thanos sidecar listens on the loopback interface instead + grpcListenLocal defines when true, the Thanos sidecar listens on the loopback interface instead of the Pod IP's address for the gRPC endpoints. It has no effect if `listenLocal` is true. type: boolean grpcServerTlsConfig: description: |- - Configures the TLS parameters for the gRPC server providing the StoreAPI. + grpcServerTlsConfig defines the TLS parameters for the gRPC server providing the StoreAPI. Note: Currently only the `caFile`, `certFile`, and `keyFile` fields are supported. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10241,7 +10297,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10266,15 +10323,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10297,7 +10355,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10322,19 +10381,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -10359,7 +10419,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10370,7 +10430,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10380,19 +10440,20 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object httpListenLocal: description: |- - When true, the Thanos sidecar listens on the loopback interface instead + httpListenLocal when true, the Thanos sidecar listens on the loopback interface instead of the Pod IP's address for the HTTP endpoints. It has no effect if `listenLocal` is true. type: boolean image: description: |- - Container image name for Thanos. If specified, it takes precedence over + image defines the container image name for Thanos. If specified, it takes precedence over the `spec.thanos.baseImage`, `spec.thanos.tag` and `spec.thanos.sha` fields. @@ -10404,18 +10465,18 @@ spec: the time when the operator was released. type: string listenLocal: - description: 'Deprecated: use `grpcListenLocal` and `httpListenLocal` - instead.' + description: 'listenLocal is deprecated: use `grpcListenLocal` + and `httpListenLocal` instead.' type: boolean logFormat: - description: Log format for the Thanos sidecar. + description: logFormat for the Thanos sidecar. enum: - "" - logfmt - json type: string logLevel: - description: Log level for the Thanos sidecar. + description: logLevel for the Thanos sidecar. enum: - "" - debug @@ -10425,14 +10486,14 @@ spec: type: string minTime: description: |- - Defines the start of time range limit served by the Thanos sidecar's StoreAPI. + minTime defines the start of time range limit served by the Thanos sidecar's StoreAPI. The field's value should be a constant time in RFC3339 format or a time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y. type: string objectStorageConfig: description: |- - Defines the Thanos sidecar's configuration to upload TSDB blocks to object storage. + objectStorageConfig defines the Thanos sidecar's configuration to upload TSDB blocks to object storage. More info: https://thanos.io/tip/thanos/storage.md/ @@ -10461,7 +10522,7 @@ spec: x-kubernetes-map-type: atomic objectStorageConfigFile: description: |- - Defines the Thanos sidecar's configuration file to upload TSDB blocks to object storage. + objectStorageConfigFile defines the Thanos sidecar's configuration file to upload TSDB blocks to object storage. More info: https://thanos.io/tip/thanos/storage.md/ @@ -10469,13 +10530,13 @@ spec: type: string readyTimeout: description: |- - ReadyTimeout is the maximum time that the Thanos sidecar will wait for + readyTimeout defines the maximum time that the Thanos sidecar will wait for Prometheus to start. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string resources: - description: Defines the resources requests and limits of the - Thanos sidecar. + description: resources defines the resources requests and limits + of the Thanos sidecar. properties: claims: description: |- @@ -10534,16 +10595,16 @@ spec: type: object type: object sha: - description: 'Deprecated: use ''image'' instead. The image digest - can be specified as part of the image name.' + description: 'sha is deprecated: use ''image'' instead. The image + digest can be specified as part of the image name.' type: string tag: - description: 'Deprecated: use ''image'' instead. The image''s + description: 'tag is deprecated: use ''image'' instead. The image''s tag can be specified as as part of the image name.' type: string tracingConfig: description: |- - Defines the tracing configuration for the Thanos sidecar. + tracingConfig defines the tracing configuration for the Thanos sidecar. `tracingConfigFile` takes precedence over this field. @@ -10575,7 +10636,7 @@ spec: x-kubernetes-map-type: atomic tracingConfigFile: description: |- - Defines the tracing configuration file for the Thanos sidecar. + tracingConfigFile defines the tracing configuration file for the Thanos sidecar. This field takes precedence over `tracingConfig`. @@ -10586,7 +10647,7 @@ spec: type: string version: description: |- - Version of Thanos being deployed. The operator uses this information + version of Thanos being deployed. The operator uses this information to generate the Prometheus StatefulSet + configuration files. If not specified, the operator assumes the latest upstream release of @@ -10595,7 +10656,7 @@ spec: type: string volumeMounts: description: |- - VolumeMounts allows configuration of additional VolumeMounts for Thanos. + volumeMounts allows configuration of additional VolumeMounts for Thanos. VolumeMounts specified will be appended to other VolumeMounts in the 'thanos-sidecar' container. items: @@ -10662,7 +10723,7 @@ spec: type: array type: object tolerations: - description: Defines the Pods' tolerations if specified. + description: tolerations defines the Pods' tolerations if specified. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -10701,12 +10762,14 @@ spec: type: object type: array topologySpreadConstraints: - description: Defines the pod's topology spread constraints if specified. + description: topologySpreadConstraints defines the pod's topology + spread constraints if specified. items: properties: additionalLabelSelectors: - description: Defines what Prometheus Operator managed labels - should be added to labelSelector on the topologySpreadConstraint. + description: additionalLabelSelectors Defines what Prometheus + Operator managed labels should be added to labelSelector on + the topologySpreadConstraint. enum: - OnResource - OnShard @@ -10881,61 +10944,62 @@ spec: type: array tracingConfig: description: |- - TracingConfig configures tracing in Prometheus. + tracingConfig defines tracing in Prometheus. This is an *experimental feature*, it may change in any upcoming release in a breaking way. properties: clientType: - description: Client used to export the traces. Supported values - are `http` or `grpc`. + description: clientType defines the client used to export the + traces. Supported values are `http` or `grpc`. enum: - http - grpc type: string compression: - description: Compression key for supported compression types. + description: compression key for supported compression types. The only supported value is `gzip`. enum: - gzip type: string endpoint: - description: Endpoint to send the traces to. Should be provided + description: endpoint to send the traces to. Should be provided in format :. minLength: 1 type: string headers: additionalProperties: type: string - description: Key-value pairs to be used as headers associated - with gRPC or HTTP requests. + description: headers defines the key-value pairs to be used as + headers associated with gRPC or HTTP requests. type: object insecure: - description: If disabled, the client will use a secure connection. + description: insecure if disabled, the client will use a secure + connection. type: boolean samplingFraction: anyOf: - type: integer - type: string - description: Sets the probability a given trace will be sampled. - Must be a float from 0 through 1. + description: samplingFraction defines the probability a given + trace will be sampled. Must be a float from 0 through 1. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true timeout: - description: Maximum time the exporter will wait for each batch - export. + description: timeout defines the maximum time the exporter will + wait for each batch export. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS Config to use when sending traces. + description: tlsConfig to use when sending traces. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10958,7 +11022,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10983,15 +11048,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11014,7 +11080,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11039,19 +11106,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key file + in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -11076,7 +11144,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11087,7 +11155,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11097,7 +11165,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -11105,12 +11174,12 @@ spec: type: object tsdb: description: |- - Defines the runtime reloadable configuration of the timeseries database(TSDB). + tsdb defines the runtime reloadable configuration of the timeseries database(TSDB). It requires Prometheus >= v2.39.0 or PrometheusAgent >= v2.54.0. properties: outOfOrderTimeWindow: description: |- - Configures how old an out-of-order/out-of-bounds sample can be with + outOfOrderTimeWindow defines how old an out-of-order/out-of-bounds sample can be with respect to the TSDB max time. An out-of-order/out-of-bounds sample is ingested into the TSDB as long as @@ -11125,7 +11194,7 @@ spec: type: object version: description: |- - Version of Prometheus being deployed. The operator uses this information + version of Prometheus being deployed. The operator uses this information to generate the Prometheus StatefulSet + configuration files. If not specified, the operator assumes the latest upstream version of @@ -11134,7 +11203,7 @@ spec: type: string volumeMounts: description: |- - VolumeMounts allows the configuration of additional VolumeMounts. + volumeMounts allows the configuration of additional VolumeMounts. VolumeMounts will be appended to other VolumeMounts in the 'prometheus' container, that are generated as a result of StorageSpec objects. @@ -11202,7 +11271,7 @@ spec: type: array volumes: description: |- - Volumes allows the configuration of additional volumes on the output + volumes allows the configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: @@ -13089,29 +13158,30 @@ spec: type: array walCompression: description: |- - Configures compression of the write-ahead log (WAL) using Snappy. + walCompression defines the compression of the write-ahead log (WAL) using Snappy. WAL compression is enabled by default for Prometheus >= 2.20.0 Requires Prometheus v2.11.0 and above. type: boolean web: - description: Defines the configuration of the Prometheus web server. + description: web defines the configuration of the Prometheus web server. properties: httpConfig: - description: Defines HTTP parameters for web server. + description: httpConfig defines HTTP parameters for web server. properties: headers: - description: List of headers that can be added to HTTP responses. + description: headers defines a list of headers that can be + added to HTTP responses. properties: contentSecurityPolicy: description: |- - Set the Content-Security-Policy header to HTTP responses. + contentSecurityPolicy defines the Content-Security-Policy header to HTTP responses. Unset if blank. type: string strictTransportSecurity: description: |- - Set the Strict-Transport-Security header to HTTP responses. + strictTransportSecurity defines the Strict-Transport-Security header to HTTP responses. Unset if blank. Please make sure that you use this with care as this header might force browsers to load Prometheus and the other applications hosted on the same @@ -13120,7 +13190,7 @@ spec: type: string xContentTypeOptions: description: |- - Set the X-Content-Type-Options header to HTTP responses. + xContentTypeOptions defines the X-Content-Type-Options header to HTTP responses. Unset if blank. Accepted value is nosniff. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options enum: @@ -13129,7 +13199,7 @@ spec: type: string xFrameOptions: description: |- - Set the X-Frame-Options header to HTTP responses. + xFrameOptions defines the X-Frame-Options header to HTTP responses. Unset if blank. Accepted values are deny and sameorigin. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options enum: @@ -13139,42 +13209,42 @@ spec: type: string xXSSProtection: description: |- - Set the X-XSS-Protection header to all responses. + xXSSProtection defines the X-XSS-Protection header to all responses. Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection type: string type: object http2: description: |- - Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. + http2 enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. When TLSConfig is not configured, HTTP/2 will be disabled. Whenever the value of the field changes, a rolling update will be triggered. type: boolean type: object maxConnections: description: |- - Defines the maximum number of simultaneous connections + maxConnections defines the maximum number of simultaneous connections A zero value means that Prometheus doesn't accept any incoming connection. format: int32 minimum: 0 type: integer pageTitle: - description: The prometheus web page title. + description: pageTitle defines the prometheus web page title. type: string tlsConfig: - description: Defines the TLS parameters for HTTPS. + description: tlsConfig defines the TLS parameters for HTTPS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -13197,7 +13267,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -13223,7 +13294,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -13231,7 +13302,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -13241,14 +13312,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -13271,7 +13342,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -13297,21 +13369,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -13321,7 +13393,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -13329,7 +13401,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -13357,14 +13429,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -13375,18 +13449,19 @@ spec: type: object status: description: |- - Most recent observed status of the Prometheus cluster. Read-only. + status defines the most recent observed status of the Prometheus cluster. Read-only. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment. format: int32 type: integer conditions: - description: The current state of the Prometheus deployment. + description: conditions defines the current state of the Prometheus + deployment. items: description: |- Condition represents the state of the resources associated with the @@ -13398,12 +13473,12 @@ spec: format: date-time type: string message: - description: Human-readable message indicating details for the - condition's last transition. + description: message defines human-readable message indicating + details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the @@ -13411,14 +13486,14 @@ spec: format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: - description: Type of the condition being reported. + description: type of the condition being reported. minLength: 1 type: string required: @@ -13432,45 +13507,46 @@ spec: x-kubernetes-list-type: map paused: description: |- - Represents whether any actions on the underlying managed objects are + paused defines whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: |- - Total number of non-terminated pods targeted by this Prometheus deployment + replicas defines the total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector). format: int32 type: integer selector: - description: The selector used to match the pods targeted by this - Prometheus resource. + description: selector used to match the pods targeted by this Prometheus + resource. type: string shardStatuses: - description: The list has one entry per shard. Each entry provides - a summary of the shard status. + description: shardStatuses defines the list has one entry per shard. + Each entry provides a summary of the shard status. items: properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this shard. format: int32 type: integer replicas: - description: Total number of pods targeted by this shard. + description: replicas defines the total number of pods targeted + by this shard. format: int32 type: integer shardID: - description: Identifier of the shard. + description: shardID defines the identifier of the shard. type: string unavailableReplicas: - description: Total number of unavailable pods targeted by this - shard. + description: unavailableReplicas defines the Total number of + unavailable pods targeted by this shard. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this shard + updatedReplicas defines the total number of non-terminated pods targeted by this shard that have the desired spec. format: int32 type: integer @@ -13486,26 +13562,20 @@ spec: - shardID x-kubernetes-list-type: map shards: - description: Shards is the most recently observed number of shards. + description: shards defines the most recently observed number of shards. format: int32 type: integer unavailableReplicas: - description: Total number of unavailable pods targeted by this Prometheus - deployment. + description: unavailableReplicas defines the total number of unavailable + pods targeted by this Prometheus deployment. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this Prometheus deployment + updatedReplicas defines the total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec. format: int32 type: integer - required: - - availableReplicas - - paused - - replicas - - unavailableReplicas - - updatedReplicas type: object required: - spec diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusrules.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusrules.yaml index f4800ee9ed9..87285176ca8 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusrules.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_prometheusrules.yaml @@ -44,24 +44,25 @@ spec: metadata: type: object spec: - description: Specification of desired alerting rule definitions for Prometheus. + description: spec defines the specification of desired alerting rule definitions + for Prometheus. properties: groups: - description: Content of Prometheus rule file + description: groups defines the content of Prometheus rule file items: description: RuleGroup is a list of sequentially evaluated recording and alerting rules. properties: interval: - description: Interval determines how often rules in the group - are evaluated. + description: interval defines how often rules in the group are + evaluated. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string labels: additionalProperties: type: string description: |- - Labels to add or overwrite before storing the result for its rules. + labels define the labels to add or overwrite before storing the result for its rules. The labels defined at the rule level take precedence. It requires Prometheus >= 3.0.0. @@ -69,31 +70,32 @@ spec: type: object limit: description: |- - Limit the number of alerts an alerting rule and series a recording + limit defines the number of alerts an alerting rule and series a recording rule can produce. Limit is supported starting with Prometheus >= 2.31 and Thanos Ruler >= 0.24. type: integer name: - description: Name of the rule group. + description: name defines the name of the rule group. minLength: 1 type: string partial_response_strategy: description: |- - PartialResponseStrategy is only used by ThanosRuler and will + partial_response_strategy is only used by ThanosRuler and will be ignored by Prometheus instances. More info: https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response pattern: ^(?i)(abort|warn)?$ type: string query_offset: description: |- - Defines the offset the rule evaluation timestamp of this particular group by the specified duration into the past. + query_offset defines the offset the rule evaluation timestamp of this particular group by the specified duration into the past. It requires Prometheus >= v2.53.0. It is not supported for ThanosRuler. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string rules: - description: List of alerting and recording rules. + description: rules defines the list of alerting and recording + rules. items: description: |- Rule describes an alerting or recording rule @@ -101,42 +103,42 @@ spec: properties: alert: description: |- - Name of the alert. Must be a valid label value. + alert defines the name of the alert. Must be a valid label value. Only one of `record` and `alert` must be set. type: string annotations: additionalProperties: type: string description: |- - Annotations to add to each alert. + annotations defines annotations to add to each alert. Only valid for alerting rules. type: object expr: anyOf: - type: integer - type: string - description: PromQL expression to evaluate. + description: expr defines the PromQL expression to evaluate. x-kubernetes-int-or-string: true for: - description: Alerts are considered firing once they have - been returned for this long. + description: for defines how alerts are considered firing + once they have been returned for this long. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string keep_firing_for: - description: KeepFiringFor defines how long an alert will - continue firing after the condition that triggered it - has cleared. + description: keep_firing_for defines how long an alert + will continue firing after the condition that triggered + it has cleared. minLength: 1 pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string labels: additionalProperties: type: string - description: Labels to add or overwrite. + description: labels defines labels to add or overwrite. type: object record: description: |- - Name of the time series to output to. Must be a valid metric name. + record defines the name of the time series to output to. Must be a valid metric name. Only one of `record` and `alert` must be set. type: string required: @@ -151,8 +153,113 @@ spec: - name x-kubernetes-list-type: map type: object + status: + description: |- + status defines the status subresource. It is under active development and is updated only when the + "StatusForConfigurationResources" feature gate is enabled. + + Most recent observed status of the PrometheusRule. Read-only. + More info: + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + bindings: + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. + items: + description: WorkloadBinding is a link between a configuration resource + and a workload resource. + properties: + conditions: + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. + items: + description: ConfigResourceCondition describes the status + of configuration resources linked to Prometheus, PrometheusAgent, + Alertmanager or ThanosRuler. + properties: + lastTransitionTime: + description: lastTransitionTime defines the time of the + last update to the current status property. + format: date-time + type: string + message: + description: message defines the human-readable message + indicating details for the condition's last transition. + type: string + observedGeneration: + description: |- + observedGeneration defines the .metadata.generation that the + condition was set based upon. For instance, if `.metadata.generation` is + currently 12, but the `.status.conditions[].observedGeneration` is 9, the + condition is out of date with respect to the current state of the object. + format: int64 + type: integer + reason: + description: reason for the condition's last transition. + type: string + status: + description: status of the condition. + minLength: 1 + type: string + type: + description: |- + type of the condition being reported. + Currently, only "Accepted" is supported. + enum: + - Accepted + minLength: 1 + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + group: + description: group defines the group of the referenced resource. + enum: + - monitoring.coreos.com + type: string + name: + description: name defines the name of the referenced object. + minLength: 1 + type: string + namespace: + description: namespace defines the namespace of the referenced + object. + minLength: 1 + type: string + resource: + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). + enum: + - prometheuses + - prometheusagents + - thanosrulers + - alertmanagers + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map + type: object required: - spec type: object served: true storage: true + subresources: + status: {} diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_scrapeconfigs.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_scrapeconfigs.yaml index b6a61bb8bad..00f58e2c2e9 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_scrapeconfigs.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_scrapeconfigs.yaml @@ -43,15 +43,15 @@ spec: metadata: type: object spec: - description: ScrapeConfigSpec is a specification of the desired configuration - for a scrape configuration. + description: spec defines the specification of ScrapeConfigSpec. properties: authorization: - description: Authorization header to use on every scrape request. + description: authorization defines the header to use on every scrape + request. properties: credentials: - description: Selects a key of a Secret in the namespace that contains - the credentials for authentication. + description: credentials defines a key of a Secret in the namespace + that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must be @@ -76,7 +76,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -84,7 +84,7 @@ spec: type: string type: object azureSDConfigs: - description: AzureSDConfigs defines a list of Azure service discovery + description: azureSDConfigs defines a list of Azure service discovery configurations. items: description: |- @@ -93,7 +93,7 @@ spec: properties: authenticationMethod: description: |- - # The authentication method, either `OAuth` or `ManagedIdentity` or `SDK`. + authenticationMethod defines the authentication method, either `OAuth` or `ManagedIdentity` or `SDK`. See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview SDK authentication method uses environment variables by default. See https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication @@ -104,12 +104,12 @@ spec: type: string authorization: description: |- - Authorization header configuration to authenticate against the target HTTP endpoint. + authorization defines the authorization header configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `oAuth2`, or `basicAuth`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -134,7 +134,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -143,13 +143,13 @@ spec: type: object basicAuth: description: |- - BasicAuth information to authenticate against the target HTTP endpoint. + basicAuth defines the information to authenticate against the target HTTP endpoint. More info: https://prometheus.io/docs/operating/configuration/#endpoints Cannot be set at the same time as `authorization`, or `oAuth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -175,7 +175,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -201,13 +201,13 @@ spec: x-kubernetes-map-type: atomic type: object clientID: - description: Optional client ID. Only required with the OAuth - authentication method. + description: clientID defines client ID. Only required with + the OAuth authentication method. minLength: 1 type: string clientSecret: - description: Optional client secret. Only required with the - OAuth authentication method. + description: clientSecret defines client secret. Only required + with the OAuth authentication method. properties: key: description: The key of the secret to select from. Must @@ -231,37 +231,36 @@ spec: type: object x-kubernetes-map-type: atomic enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean environment: - description: The Azure environment. + description: environment defines the Azure environment. minLength: 1 type: string followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. - Cannot be set at the same time as `authorization`, or `basicAuth`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -284,7 +283,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -310,7 +310,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -338,12 +338,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -377,7 +377,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -385,33 +385,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -434,8 +433,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -460,12 +459,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -488,8 +487,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -514,11 +513,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -543,7 +543,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -554,7 +554,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -564,12 +564,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -579,7 +580,7 @@ spec: type: object port: description: |- - The port to scrape metrics from. If using the public IP address, this must + port defines the port to scrape metrics from. If using the public IP address, this must instead be specified in the relabeling rule. format: int32 maximum: 65535 @@ -613,7 +614,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -621,44 +622,47 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: RefreshInterval configures the refresh interval - at which Prometheus will re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string resourceGroup: description: |- - Optional resource group name. Limits discovery to this resource group. + resourceGroup defines resource group name. Limits discovery to this resource group. Requires Prometheus v2.35.0 and above minLength: 1 type: string subscriptionID: - description: The subscription ID. Always required. + description: subscriptionID defines subscription ID. Always + required. minLength: 1 type: string tenantID: - description: Optional tenant ID. Only required with the OAuth - authentication method. + description: tenantID defines tenant ID. Only required with + the OAuth authentication method. minLength: 1 type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defies the TLS configuration applying + to the target HTTP endpoint. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -681,7 +685,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -706,11 +711,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -733,7 +739,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -758,11 +765,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -787,7 +795,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -798,7 +806,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -808,7 +816,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -816,11 +825,12 @@ spec: type: object type: array basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -846,7 +856,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -872,7 +882,7 @@ spec: x-kubernetes-map-type: atomic type: object consulSDConfigs: - description: ConsulSDConfigs defines a list of Consul service discovery + description: consulSDConfigs defines a list of Consul service discovery configurations. items: description: |- @@ -881,17 +891,17 @@ spec: properties: allowStale: description: |- - Allow stale Consul results (see https://www.consul.io/api/features/consistency.html). Will reduce load on Consul. + allowStale Consul results (see https://www.consul.io/api/features/consistency.html). Will reduce load on Consul. If unset, Prometheus uses its default value. type: boolean authorization: description: |- - Optional Authorization header configuration to authenticate against the Consul Server. + authorization defines the header configuration to authenticate against the Consul Server. Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -916,7 +926,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -925,13 +935,13 @@ spec: type: object basicAuth: description: |- - Optional BasicAuth information to authenticate against the Consul Server. + basicAuth defines the information to authenticate against the Consul Server. More info: https://prometheus.io/docs/operating/configuration/#endpoints Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -957,7 +967,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -983,37 +993,34 @@ spec: x-kubernetes-map-type: atomic type: object datacenter: - description: Consul Datacenter name, if not provided it will - use the local Consul Agent Datacenter. + description: datacenter defines the consul Datacenter name, + if not provided it will use the local Consul Agent Datacenter. minLength: 1 type: string enableHTTP2: - description: |- - Whether to enable HTTP2. - If unset, Prometheus uses its default value. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean filter: description: |- - Filter expression used to filter the catalog results. + filter defines the filter expression used to filter the catalog results. See https://www.consul.io/api-docs/catalog#list-services It requires Prometheus >= 3.0.0. minLength: 1 type: string followRedirects: - description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. - If unset, Prometheus uses its default value. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean namespace: description: |- - Namespaces are only supported in Consul Enterprise. + namespace are only supported in Consul Enterprise. It requires Prometheus >= 2.28.0. minLength: 1 type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1023,23 +1030,23 @@ spec: additionalProperties: type: string description: |- - Node metadata key/value pairs to filter nodes for a given service. + nodeMeta defines the node metadata key/value pairs to filter nodes for a given service. Starting with Consul 1.14, it is recommended to use `filter` with the `NodeMeta` selector instead. type: object x-kubernetes-map-type: atomic oauth2: description: |- - Optional OAuth2.0 configuration. - Cannot be set at the same time as `basicAuth`, or `authorization`. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1062,7 +1069,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1088,7 +1096,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -1116,12 +1124,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1155,7 +1163,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1163,33 +1171,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1212,8 +1219,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1238,12 +1245,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1266,8 +1273,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1292,11 +1299,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1321,7 +1329,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1332,7 +1340,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1342,12 +1350,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -1356,12 +1365,13 @@ spec: - tokenUrl type: object partition: - description: Admin Partitions are only supported in Consul Enterprise. + description: partition defines the admin Partitions are only + supported in Consul Enterprise. minLength: 1 type: string pathPrefix: description: |- - Prefix for URIs for when consul is behind an API gateway (reverse proxy). + pathPrefix defines the prefix for URIs for when consul is behind an API gateway (reverse proxy). It requires Prometheus >= 2.45.0. minLength: 1 @@ -1394,7 +1404,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1402,63 +1412,64 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: description: |- - The time after which the provided names are refreshed. - On large setup it might be a good idea to increase this value because the catalog will change all the time. - If unset, Prometheus uses its default value. + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string scheme: - description: HTTP Scheme default "http" + description: scheme defines the HTTP Scheme default "http" enum: - HTTP - HTTPS type: string server: - description: Consul server address. A valid string consisting - of a hostname or IP followed by an optional port number. + description: server defines the consul server address. A valid + string consisting of a hostname or IP followed by an optional + port number. minLength: 1 type: string services: - description: A list of services for which targets are retrieved. - If omitted, all services are scraped. + description: services defines a list of services for which targets + are retrieved. If omitted, all services are scraped. items: type: string type: array x-kubernetes-list-type: set tagSeparator: description: |- - The string by which Consul tags are joined into the tag label. + tagSeparator defines the string by which Consul tags are joined into the tag label. If unset, Prometheus uses its default value. minLength: 1 type: string tags: description: |- - An optional list of tags used to filter nodes for a given service. Services must contain all tags in the list. + tags defines an optional list of tags used to filter nodes for a given service. Services must contain all tags in the list. Starting with Consul 1.14, it is recommended to use `filter` with the `ServiceTags` selector instead. items: type: string type: array x-kubernetes-list-type: set tlsConfig: - description: TLS configuration to connect to the Consul API. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1481,7 +1492,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1506,11 +1518,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1533,7 +1546,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1558,11 +1572,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1587,7 +1602,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1598,7 +1613,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1608,12 +1623,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object tokenRef: - description: Consul ACL TokenRef, if not provided it will use - the ACL from the local Consul Agent. + description: tokenRef defines the consul ACL TokenRef, if not + provided it will use the ACL from the local Consul Agent. properties: key: description: The key of the secret to select from. Must @@ -1642,11 +1658,11 @@ spec: type: array convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native histogram with custom buckets. + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.0.0. type: boolean digitalOceanSDConfigs: - description: DigitalOceanSDConfigs defines a list of DigitalOcean + description: digitalOceanSDConfigs defines a list of DigitalOcean service discovery configurations. items: description: |- @@ -1656,12 +1672,12 @@ spec: properties: authorization: description: |- - Authorization header configuration to authenticate against the DigitalOcean API. + authorization defines the header configuration to authenticate against the DigitalOcean API. Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -1686,7 +1702,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -1694,33 +1710,32 @@ spec: type: string type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1743,7 +1758,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -1769,7 +1785,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -1797,12 +1813,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -1836,7 +1852,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -1844,33 +1860,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1893,8 +1908,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1919,12 +1934,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -1947,8 +1962,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -1973,11 +1988,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2002,7 +2018,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2013,7 +2029,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2023,12 +2039,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -2037,7 +2054,8 @@ spec: - tokenUrl type: object port: - description: The port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -2070,7 +2088,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2078,28 +2096,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2122,7 +2143,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2147,11 +2169,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2174,7 +2197,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2199,11 +2223,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2228,7 +2253,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2239,7 +2264,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2249,13 +2274,14 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object type: object type: array dnsSDConfigs: - description: DNSSDConfigs defines a list of DNS service discovery + description: dnsSDConfigs defines a list of DNS service discovery configurations. items: description: |- @@ -2264,7 +2290,8 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config properties: names: - description: A list of DNS domain names to be queried. + description: names defines a list of DNS domain names to be + queried. items: minLength: 1 type: string @@ -2272,7 +2299,7 @@ spec: type: array port: description: |- - The port number used if the query type is not SRV + port defines the port to scrape metrics from. If using the public IP address, this must Ignored for SRV records format: int32 maximum: 65535 @@ -2280,13 +2307,13 @@ spec: type: integer refreshInterval: description: |- - RefreshInterval configures the time after which the provided names are refreshed. + refreshInterval defines the time after which the provided names are refreshed. If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: description: |- - The type of DNS query to perform. One of SRV, A, AAAA, MX or NS. + type defines the type of DNS query to perform. One of SRV, A, AAAA, MX or NS. If not set, Prometheus uses its default value. When set to NS, it requires Prometheus >= v2.49.0. @@ -2303,7 +2330,7 @@ spec: type: object type: array dockerSDConfigs: - description: DockerSDConfigs defines a list of Docker service discovery + description: dockerSDConfigs defines a list of Docker service discovery configurations. items: description: |- @@ -2314,12 +2341,12 @@ spec: properties: authorization: description: |- - Authorization header configuration to authenticate against the Docker API. + authorization defines the header configuration to authenticate against the DigitalOcean API. Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -2344,7 +2371,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -2352,11 +2379,12 @@ spec: type: string type: object basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -2382,7 +2410,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -2408,20 +2436,20 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean filters: - description: Optional filters to limit the discovery process - to a subset of the available resources. + description: filters defines filters to limit the discovery + process to a subset of the available resources. items: description: Filter name and value pairs to limit the discovery process to a subset of available resources. properties: name: - description: Name of the Filter. + description: name of the Filter. type: string values: - description: Value to filter on. + description: values defines values to filter on. items: minLength: 1 type: string @@ -2437,45 +2465,44 @@ spec: - name x-kubernetes-list-type: map followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean host: - description: Address of the docker daemon + description: host defines the address of the docker daemon minLength: 1 type: string hostNetworkingHost: - description: The host to use if the container is in host networking - mode. + description: hostNetworkingHost defines the host to use if the + container is in host networking mode. minLength: 1 type: string matchFirstNetwork: description: |- - Configure whether to match the first network if the container has multiple networks defined. + matchFirstNetwork defines whether to match the first network if the container has multiple networks defined. If unset, Prometheus uses true by default. It requires Prometheus >= v2.54.1. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2498,7 +2525,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2524,7 +2552,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -2552,12 +2580,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -2591,7 +2619,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2599,33 +2627,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2648,8 +2675,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -2674,12 +2701,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2702,8 +2729,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -2728,11 +2755,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2757,7 +2785,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2768,7 +2796,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2778,12 +2806,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -2792,7 +2821,8 @@ spec: - tokenUrl type: object port: - description: The port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -2825,7 +2855,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -2833,28 +2863,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Time after which the container is refreshed. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2877,7 +2910,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2902,11 +2936,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -2929,7 +2964,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2954,11 +2990,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -2983,7 +3020,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2994,7 +3031,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3004,7 +3041,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -3012,7 +3050,7 @@ spec: type: object type: array dockerSwarmSDConfigs: - description: DockerswarmSDConfigs defines a list of Dockerswarm service + description: dockerSwarmSDConfigs defines a list of Dockerswarm service discovery configurations. items: description: |- @@ -3020,12 +3058,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config properties: authorization: - description: Authorization header configuration to authenticate - against the target HTTP endpoint. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -3050,7 +3089,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -3058,11 +3097,12 @@ spec: type: string type: object basicAuth: - description: Optional HTTP basic authentication information. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -3088,7 +3128,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -3114,11 +3154,11 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean filters: description: |- - Optional filters to limit the discovery process to a subset of available + filters defines the filters to limit the discovery process to a subset of available resources. The available filters are listed in the upstream documentation: Services: https://docs.docker.com/engine/api/v1.40/#operation/ServiceList @@ -3129,10 +3169,10 @@ spec: process to a subset of available resources. properties: name: - description: Name of the Filter. + description: name of the Filter. type: string values: - description: Value to filter on. + description: values defines values to filter on. items: minLength: 1 type: string @@ -3148,16 +3188,16 @@ spec: - name x-kubernetes-list-type: map followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean host: - description: Address of the Docker daemon + description: host defines the address of the Docker daemon pattern: ^[a-zA-Z][a-zA-Z0-9+.-]*://.+$ type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3165,17 +3205,17 @@ spec: type: string oauth2: description: |- - Optional OAuth 2.0 configuration. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3198,7 +3238,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -3224,7 +3265,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -3252,12 +3293,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3291,7 +3332,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3299,33 +3340,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3348,8 +3388,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -3374,12 +3414,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3402,8 +3442,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -3428,11 +3468,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -3457,7 +3498,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -3468,7 +3509,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3478,12 +3519,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -3493,7 +3535,7 @@ spec: type: object port: description: |- - The port to scrape metrics from, when `role` is nodes, and for discovered + port defines the port to scrape metrics from. If using the public IP address, this must tasks and services that don't have published ports. format: int32 maximum: 65535 @@ -3527,7 +3569,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3535,21 +3577,22 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: The time after which the service discovery data - is refreshed. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string role: - description: Role of the targets to retrieve. Must be `Services`, + description: role of the targets to retrieve. Must be `Services`, `Tasks`, or `Nodes`. enum: - Services @@ -3557,15 +3600,16 @@ spec: - Nodes type: string tlsConfig: - description: TLS configuration to use on every scrape request + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3588,7 +3632,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -3613,11 +3658,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3640,7 +3686,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -3665,11 +3712,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -3694,7 +3742,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -3705,7 +3753,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -3715,7 +3763,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -3724,7 +3773,7 @@ spec: type: object type: array ec2SDConfigs: - description: EC2SDConfigs defines a list of EC2 service discovery + description: ec2SDConfigs defines a list of EC2 service discovery configurations. items: description: |- @@ -3737,7 +3786,7 @@ spec: BasicAuth, Authorization and OAuth2 fields are not present on purpose. properties: accessKey: - description: AccessKey is the AWS API key. + description: accessKey defines the AWS API key. properties: key: description: The key of the secret to select from. Must @@ -3762,12 +3811,12 @@ spec: x-kubernetes-map-type: atomic enableHTTP2: description: |- - Whether to enable HTTP2. + enableHTTP2 defines whether to enable HTTP2. It requires Prometheus >= v2.41.0 type: boolean filters: description: |- - Filters can be used optionally to filter the instance list by other criteria. + filters can be used optionally to filter the instance list by other criteria. Available filter criteria can be found here: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html Filter API documentation: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Filter.html @@ -3777,10 +3826,10 @@ spec: process to a subset of available resources. properties: name: - description: Name of the Filter. + description: name of the Filter. type: string values: - description: Value to filter on. + description: values defines values to filter on. items: minLength: 1 type: string @@ -3797,12 +3846,12 @@ spec: x-kubernetes-list-type: map followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.41.0 type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -3810,7 +3859,7 @@ spec: type: string port: description: |- - The port to scrape metrics from. If using the public IP address, this must + port defines the port to scrape metrics from. If using the public IP address, this must instead be specified in the relabeling rule. format: int32 maximum: 65535 @@ -3844,7 +3893,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -3852,29 +3901,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: RefreshInterval configures the refresh interval - at which Prometheus will re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: - description: The AWS region. + description: region defines the AWS region. minLength: 1 type: string roleARN: - description: AWS Role ARN, an alternative to using AWS API keys. + description: roleARN defines an alternative to using AWS API + keys. minLength: 1 type: string secretKey: - description: SecretKey is the AWS API secret. + description: secretKey defines the AWS API secret. properties: key: description: The key of the secret to select from. Must @@ -3899,16 +3950,16 @@ spec: x-kubernetes-map-type: atomic tlsConfig: description: |- - TLS configuration to connect to the AWS EC2 API. + tlsConfig defines the TLS configuration to connect to the Consul API. It requires Prometheus >= v2.41.0 properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3931,7 +3982,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -3956,11 +4008,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -3983,7 +4036,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -4008,11 +4062,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -4037,7 +4092,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4048,7 +4103,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4058,24 +4113,25 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object type: object type: array enableCompression: description: |- - When false, Prometheus will request uncompressed response from the scraped target. + enableCompression when false, Prometheus will request uncompressed response from the scraped target. It requires Prometheus >= v2.49.0. If unset, Prometheus uses true by default. type: boolean enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean eurekaSDConfigs: - description: EurekaSDConfigs defines a list of Eureka service discovery + description: eurekaSDConfigs defines a list of Eureka service discovery configurations. items: description: |- @@ -4084,11 +4140,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config properties: authorization: - description: Authorization header to use on every scrape request. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -4113,7 +4171,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -4121,11 +4179,12 @@ spec: type: string type: object basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines the BasicAuth information to + use on every scrape request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -4151,7 +4210,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -4177,33 +4236,32 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization` or `basic_auth`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4226,7 +4284,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -4252,7 +4311,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -4280,12 +4339,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -4319,7 +4378,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4327,33 +4386,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4376,8 +4434,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -4402,12 +4460,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4430,8 +4488,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -4456,11 +4514,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -4485,7 +4544,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4496,7 +4555,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4506,12 +4565,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -4547,7 +4607,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -4555,32 +4615,37 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string server: - description: The URL to connect to the Eureka server. + description: server defines the URL to connect to the Eureka + server. minLength: 1 + pattern: ^http(s)?://.+$ type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4603,7 +4668,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -4628,11 +4694,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4655,7 +4722,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -4680,11 +4748,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -4709,7 +4778,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -4720,7 +4789,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -4730,7 +4799,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -4739,7 +4809,7 @@ spec: type: array fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It requires Prometheus >= v3.0.0. enum: @@ -4750,7 +4820,7 @@ spec: - PrometheusText1.0.0 type: string fileSDConfigs: - description: FileSDConfigs defines a list of file service discovery + description: fileSDConfigs defines a list of file service discovery configurations. items: description: |- @@ -4759,7 +4829,7 @@ spec: properties: files: description: |- - List of files to be used for file discovery. Recommendation: use absolute paths. While relative paths work, the + files defines the list of files to be used for file discovery. Recommendation: use absolute paths. While relative paths work, the prometheus-operator project makes no guarantees about the working directory where the configuration file is stored. Files must be mounted using Prometheus.ConfigMaps or Prometheus.Secrets. @@ -4771,8 +4841,9 @@ spec: type: array x-kubernetes-list-type: set refreshInterval: - description: RefreshInterval configures the refresh interval - at which Prometheus will reload the content of the files. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string required: @@ -4780,7 +4851,7 @@ spec: type: object type: array gceSDConfigs: - description: GCESDConfigs defines a list of GCE service discovery + description: gceSDConfigs defines a list of GCE service discovery configurations. items: description: |- @@ -4800,36 +4871,37 @@ spec: properties: filter: description: |- - Filter can be used optionally to filter the instance list by other criteria + filter defines the filter that can be used optionally to filter the instance list by other criteria Syntax of this filter is described in the filter query parameter section: https://cloud.google.com/compute/docs/reference/latest/instances/list minLength: 1 type: string port: description: |- - The port to scrape metrics from. If using the public IP address, this must + port defines the port to scrape metrics from. If using the public IP address, this must instead be specified in the relabeling rule. format: int32 maximum: 65535 minimum: 0 type: integer project: - description: The Google Cloud Project ID + description: project defines the Google Cloud Project ID minLength: 1 type: string refreshInterval: - description: RefreshInterval configures the refresh interval - at which Prometheus will re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tagSeparator: - description: The tag separator is used to separate the tags - on concatenation + description: tagSeparator defines the tag separator is used + to separate the tags on concatenation minLength: 1 type: string zone: - description: The zone of the scrape targets. If you need multiple - zones use multiple GCESDConfigs. + description: zone defines the zone of the scrape targets. If + you need multiple zones use multiple GCESDConfigs. minLength: 1 type: string required: @@ -4838,7 +4910,7 @@ spec: type: object type: array hetznerSDConfigs: - description: HetznerSDConfigs defines a list of Hetzner service discovery + description: hetznerSDConfigs defines a list of Hetzner service discovery configurations. items: description: |- @@ -4848,12 +4920,12 @@ spec: properties: authorization: description: |- - Authorization header configuration, required when role is hcloud. - Role robot does not support bearer token authentication. + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -4878,7 +4950,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -4886,13 +4958,12 @@ spec: type: string type: object basicAuth: - description: |- - BasicAuth information to use on every scrape request, required when role is robot. - Role hcloud does not support basic auth. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -4918,7 +4989,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -4944,39 +5015,38 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean labelSelector: description: |- - Label selector used to filter the servers when fetching them from the API. + labelSelector defines the label selector used to filter the servers when fetching them from the API. It requires Prometheus >= v3.5.0. minLength: 1 type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be used at the same time as `basic_auth` or `authorization`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -4999,7 +5069,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5025,7 +5096,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5053,12 +5124,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5092,7 +5163,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5100,33 +5171,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5149,8 +5219,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5175,12 +5245,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5203,8 +5273,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5229,11 +5299,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5258,7 +5329,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5269,7 +5340,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5279,12 +5350,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -5293,7 +5365,11 @@ spec: - tokenUrl type: object port: - description: The port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must + format: int32 + maximum: 65535 + minimum: 0 type: integer proxyConnectHeader: additionalProperties: @@ -5323,7 +5399,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5331,20 +5407,23 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: The time after which the servers are refreshed. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string role: - description: The Hetzner role of entities that should be discovered. + description: role defines the Hetzner role of entities that + should be discovered. enum: - hcloud - Hcloud @@ -5352,15 +5431,16 @@ spec: - Robot type: string tlsConfig: - description: TLS configuration to use on every scrape request. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5383,7 +5463,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5408,11 +5489,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5435,7 +5517,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5460,11 +5543,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5489,7 +5573,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5500,7 +5584,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5510,7 +5594,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -5518,15 +5603,17 @@ spec: type: object type: array honorLabels: - description: HonorLabels chooses the metric's labels on collisions - with target labels. + description: |- + honorLabels defines when true the metric's labels when they collide + with the target's labels. type: boolean honorTimestamps: - description: HonorTimestamps controls whether Prometheus respects - the timestamps present in scraped data. + description: |- + honorTimestamps defines whether Prometheus preserves the timestamps + when exposed by the target. type: boolean httpSDConfigs: - description: HTTPSDConfigs defines a list of HTTP service discovery + description: httpSDConfigs defines a list of HTTP service discovery configurations. items: description: |- @@ -5535,12 +5622,12 @@ spec: properties: authorization: description: |- - Authorization header configuration to authenticate against the target HTTP endpoint. + authorization defines the authorization header configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `oAuth2`, or `basicAuth`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -5565,7 +5652,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -5574,13 +5661,13 @@ spec: type: object basicAuth: description: |- - BasicAuth information to authenticate against the target HTTP endpoint. + basicAuth defines information to use on every scrape request. More info: https://prometheus.io/docs/operating/configuration/#endpoints Cannot be set at the same time as `authorization`, or `oAuth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -5606,7 +5693,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -5632,15 +5719,15 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5648,17 +5735,17 @@ spec: type: string oauth2: description: |- - Optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5681,7 +5768,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5707,7 +5795,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5735,12 +5823,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5774,7 +5862,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5782,33 +5870,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5831,8 +5918,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5857,12 +5944,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5885,8 +5972,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5911,11 +5998,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5940,7 +6028,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5951,7 +6039,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5961,12 +6049,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -6002,7 +6091,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6010,30 +6099,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: description: |- - RefreshInterval configures the refresh interval at which Prometheus will re-query the - endpoint to update the target list. + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration applying + to the target HTTP endpoint. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6056,7 +6146,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6081,11 +6172,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6108,7 +6200,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6133,11 +6226,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6162,7 +6256,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6173,7 +6267,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6183,11 +6277,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: URL from which the targets are fetched. + description: url defines the URL from which the targets are + fetched. minLength: 1 pattern: ^http(s)?://.+$ type: string @@ -6196,7 +6292,7 @@ spec: type: object type: array ionosSDConfigs: - description: IonosSDConfigs defines a list of IONOS service discovery + description: ionosSDConfigs defines a list of IONOS service discovery configurations. items: description: |- @@ -6204,12 +6300,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ionos_sd_config properties: authorization: - description: Authorization` header configuration, required when - using IONOS. + description: |- + authorization defines the header configuration to authenticate against the IONOS. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -6234,7 +6331,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6242,35 +6339,37 @@ spec: type: string type: object datacenterID: - description: The unique ID of the IONOS data center. + description: datacenterID defines the unique ID of the IONOS + data center. minLength: 1 type: string enableHTTP2: - description: Configure whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether the HTTP requests should follow + description: followRedirects defines whether HTTP requests follow HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: Configure whether to enable OAuth2. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6293,7 +6392,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6319,7 +6419,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -6347,12 +6447,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6386,7 +6486,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6394,33 +6494,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6443,8 +6542,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -6469,12 +6568,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6497,8 +6596,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -6523,11 +6622,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6552,7 +6652,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6563,7 +6663,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6573,12 +6673,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -6587,7 +6688,8 @@ spec: - tokenUrl type: object port: - description: Port to scrape the metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -6620,7 +6722,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -6628,29 +6730,31 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the list of resources. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration to use when connecting to the - IONOS API. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6673,7 +6777,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6698,11 +6803,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -6725,7 +6831,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -6750,11 +6857,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -6779,7 +6887,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -6790,7 +6898,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -6800,7 +6908,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -6810,7 +6919,7 @@ spec: type: array jobName: description: |- - The value of the `job` label assigned to the scraped metrics by default. + jobName defines the value of the `job` label assigned to the scraped metrics by default. The `job_name` field in the rendered scrape configuration is always controlled by the operator to prevent duplicate job names, which Prometheus does not allow. Instead the @@ -6819,14 +6928,14 @@ spec: type: string keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. format: int64 type: integer kubernetesSDConfigs: - description: KubernetesSDConfigs defines a list of Kubernetes service + description: kubernetesSDConfigs defines a list of Kubernetes service discovery configurations. items: description: |- @@ -6835,7 +6944,7 @@ spec: properties: apiServer: description: |- - The API server address consisting of a hostname or IP address followed + apiServer defines the API server address consisting of a hostname or IP address followed by an optional port number. If left empty, Prometheus is assumed to run inside of the cluster. It will discover API servers automatically and use the pod's @@ -6844,13 +6953,13 @@ spec: type: string attachMetadata: description: |- - Optional metadata to attach to discovered targets. + attachMetadata defines the metadata to attach to discovered targets. It requires Prometheus >= v2.35.0 when using the `Pod` role and Prometheus >= v2.37.0 for `Endpoints` and `Endpointslice` roles. properties: node: description: |- - Attaches node metadata to discovered targets. + node attaches node metadata to discovered targets. When set to true, Prometheus must have the `get` permission on the `Nodes` objects. Only valid for Pod, Endpoint and Endpointslice roles. @@ -6858,12 +6967,12 @@ spec: type: object authorization: description: |- - Authorization header to use on every scrape request. + authorization defines the authorization header to use on every scrape request. Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -6888,7 +6997,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -6897,12 +7006,12 @@ spec: type: object basicAuth: description: |- - BasicAuth information to use on every scrape request. + basicAuth defines information to use on every scrape request. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -6928,7 +7037,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -6954,32 +7063,32 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean namespaces: - description: Optional namespace discovery. If omitted, Prometheus - discovers targets across all namespaces. + description: namespaces defines the namespace discovery. If + omitted, Prometheus discovers targets across all namespaces. properties: names: description: |- - List of namespaces where to watch for resources. + names defines a list of namespaces where to watch for resources. If empty and `ownNamespace` isn't true, Prometheus watches for resources in all namespaces. items: type: string type: array x-kubernetes-list-type: set ownNamespace: - description: Includes the namespace in which the Prometheus - pod runs to the list of watched namespaces. + description: ownNamespace includes the namespace in which + the Prometheus pod runs to the list of watched namespaces. type: boolean type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -6987,17 +7096,17 @@ spec: type: string oauth2: description: |- - Optional OAuth 2.0 configuration. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7020,7 +7129,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7046,7 +7156,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -7074,12 +7184,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7113,7 +7223,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7121,33 +7231,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7170,8 +7279,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7196,12 +7305,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7224,8 +7333,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7250,11 +7359,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7279,7 +7389,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7290,7 +7400,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7300,12 +7410,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -7341,7 +7452,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7349,17 +7460,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string role: description: |- - Role of the Kubernetes entities that should be discovered. + role defines the Kubernetes role of the entities that should be discovered. Role `Endpointslice` requires Prometheus >= v2.21.0 enum: - Pod @@ -7371,26 +7482,26 @@ spec: type: string selectors: description: |- - Selector to select objects. + selectors defines the selector to select objects. It requires Prometheus >= v2.17.0 items: description: K8SSelectorConfig is Kubernetes Selector Config properties: field: description: |- - An optional field selector to limit the service discovery to resources which have fields with specific values. + field defines an optional field selector to limit the service discovery to resources which have fields with specific values. e.g: `metadata.name=foobar` minLength: 1 type: string label: description: |- - An optional label selector to limit the service discovery to resources with specific labels and label values. + label defines an optional label selector to limit the service discovery to resources with specific labels and label values. e.g: `node.kubernetes.io/instance-type=master` minLength: 1 type: string role: description: |- - Role specifies the type of Kubernetes resource to limit the service discovery to. + role defines the type of Kubernetes resource to limit the service discovery to. Accepted values are: Node, Pod, Endpoints, EndpointSlice, Service, Ingress. enum: - Pod @@ -7408,16 +7519,16 @@ spec: - role x-kubernetes-list-type: map tlsConfig: - description: TLS configuration to connect to the Kubernetes - API. + description: tlsConfig defines the TLS configuration to connect + to the Kubernetes API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7440,7 +7551,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7465,11 +7577,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7492,7 +7605,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7517,11 +7631,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7546,7 +7661,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -7557,7 +7672,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -7567,7 +7682,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -7575,7 +7691,7 @@ spec: type: object type: array kumaSDConfigs: - description: KumaSDConfigs defines a list of Kuma service discovery + description: kumaSDConfigs defines a list of Kuma service discovery configurations. items: description: |- @@ -7583,11 +7699,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kuma_sd_config properties: authorization: - description: Authorization header to use on every scrape request. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -7612,7 +7730,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -7620,11 +7738,12 @@ spec: type: string type: object basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -7650,7 +7769,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -7676,42 +7795,43 @@ spec: x-kubernetes-map-type: atomic type: object clientID: - description: Client id is used by Kuma Control Plane to compute - Monitoring Assignment for specific Prometheus backend. + description: |- + clientID is used by Kuma Control Plane to compute Monitoring Assignment for specific Prometheus backend. + It requires Prometheus >= v2.50.0. + minLength: 1 type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean fetchTimeout: - description: The time after which the monitoring assignments - are refreshed. + description: fetchTimeout defines the time after which the monitoring + assignments are refreshed. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization`, or `basicAuth`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7734,7 +7854,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -7760,7 +7881,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -7788,12 +7909,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -7827,7 +7948,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -7835,33 +7956,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7884,8 +8004,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7910,12 +8030,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -7938,8 +8058,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -7964,11 +8084,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -7993,7 +8114,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8004,7 +8125,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8014,12 +8135,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -8055,7 +8177,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8063,32 +8185,36 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: The time to wait between polling update requests. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string server: - description: Address of the Kuma Control Plane's MADS xDS server. - minLength: 1 + description: server defines the address of the Kuma Control + Plane's MADS xDS server. + pattern: ^https?://.+$ type: string tlsConfig: - description: TLS configuration to use on every scrape request + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8111,7 +8237,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8136,11 +8263,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8163,7 +8291,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8188,11 +8317,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8217,7 +8347,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8228,7 +8358,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8238,7 +8368,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -8247,24 +8378,24 @@ spec: type: array labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines the per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. format: int64 type: integer lightSailSDConfigs: - description: LightsailSDConfigs defines a list of Lightsail service + description: lightSailSDConfigs defines a list of Lightsail service discovery configurations. items: description: |- @@ -8272,7 +8403,7 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#lightsail_sd_config properties: accessKey: - description: AccessKey is the AWS API key. + description: accessKey defines the AWS API key. properties: key: description: The key of the secret to select from. Must @@ -8297,12 +8428,12 @@ spec: x-kubernetes-map-type: atomic authorization: description: |- - Optional `authorization` HTTP header configuration. - Cannot be set at the same time as `basicAuth`, or `oauth2`. + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -8327,7 +8458,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -8336,12 +8467,12 @@ spec: type: object basicAuth: description: |- - Optional HTTP basic authentication information. + basicAuth defines information to use on every scrape request. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -8367,7 +8498,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -8393,19 +8524,19 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Configure whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean endpoint: - description: Custom endpoint to be used. + description: endpoint defines the custom endpoint to be used. minLength: 1 type: string followRedirects: - description: Configure whether the HTTP requests should follow + description: followRedirects defines whether HTTP requests follow HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -8413,17 +8544,17 @@ spec: type: string oauth2: description: |- - Optional OAuth2.0 configuration. - Cannot be set at the same time as `basicAuth`, or `authorization`. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8446,7 +8577,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8472,7 +8604,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -8500,12 +8632,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -8539,7 +8671,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8547,33 +8679,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8596,8 +8727,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -8622,12 +8753,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8650,8 +8781,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -8676,11 +8807,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8705,7 +8837,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8716,7 +8848,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8726,12 +8858,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -8740,9 +8873,8 @@ spec: - tokenUrl type: object port: - description: |- - Port to scrape the metrics from. - If using the public IP address, this must instead be specified in the relabeling rule. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -8775,7 +8907,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -8783,27 +8915,30 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the list of instances. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: - description: The AWS region. + description: region defines the AWS region. minLength: 1 type: string roleARN: - description: AWS Role ARN, an alternative to using AWS API keys. + description: roleARN defines the AWS Role ARN, an alternative + to using AWS API keys. type: string secretKey: - description: SecretKey is the AWS API secret. + description: secretKey defines the AWS API secret. properties: key: description: The key of the secret to select from. Must @@ -8827,15 +8962,16 @@ spec: type: object x-kubernetes-map-type: atomic tlsConfig: - description: TLS configuration to connect to the Puppet DB. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8858,7 +8994,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8883,11 +9020,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -8910,7 +9048,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -8935,11 +9074,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -8964,7 +9104,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -8975,7 +9115,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -8985,13 +9125,14 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object type: object type: array linodeSDConfigs: - description: LinodeSDConfigs defines a list of Linode service discovery + description: linodeSDConfigs defines a list of Linode service discovery configurations. items: description: |- @@ -8999,11 +9140,13 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#linode_sd_config properties: authorization: - description: Authorization header configuration. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -9028,7 +9171,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -9036,15 +9179,15 @@ spec: type: string type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -9052,17 +9195,17 @@ spec: type: string oauth2: description: |- - Optional OAuth 2.0 configuration. - Cannot be used at the same time as `authorization`. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9085,7 +9228,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9111,7 +9255,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -9139,12 +9283,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -9178,7 +9322,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9186,33 +9330,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9235,8 +9378,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -9261,12 +9404,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9289,8 +9432,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -9315,11 +9458,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -9344,7 +9488,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9355,7 +9499,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9365,12 +9509,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -9379,7 +9524,8 @@ spec: - tokenUrl type: object port: - description: Default port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -9412,7 +9558,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -9420,37 +9566,40 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Time after which the linode instances are refreshed. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: - description: Optional region to filter on. + description: region defines the region to filter on. minLength: 1 type: string tagSeparator: - description: The string by which Linode Instance tags are joined - into the tag label. + description: tagSeparator defines the string by which Linode + Instance tags are joined into the tag label.el. minLength: 1 type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9473,7 +9622,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9498,11 +9648,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9525,7 +9676,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9550,11 +9702,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -9579,7 +9732,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -9590,7 +9743,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -9600,13 +9753,15 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object type: object type: array metricRelabelings: - description: MetricRelabelConfigs to apply to samples before ingestion. + description: metricRelabelings defines the metricRelabelings to apply + to samples before ingestion. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -9617,7 +9772,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -9649,40 +9804,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against which + the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated SourceLabels. + description: separator defines the string between concatenated + SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -9693,13 +9849,13 @@ spec: minItems: 1 type: array metricsPath: - description: MetricsPath HTTP path to scrape for metrics. If empty, - Prometheus uses the default value (e.g. /metrics). + description: metricsPath defines the HTTP path to scrape for metrics. + If empty, Prometheus uses the default value (e.g. /metrics). minLength: 1 type: string nameEscapingScheme: description: |- - Metric name escaping mode to request through content negotiation. + nameEscapingScheme defines the metric name escaping mode to request through content negotiation. It requires Prometheus >= v3.4.0. enum: @@ -9710,7 +9866,7 @@ spec: type: string nameValidationScheme: description: |- - Specifies the validation scheme for metric and label names. + nameValidationScheme defines the validation scheme for metric and label names. It requires Prometheus >= v3.0.0. enum: @@ -9719,7 +9875,7 @@ spec: type: string nativeHistogramBucketLimit: description: |- - If there are more than this many buckets in a native histogram, + nativeHistogramBucketLimit defines ff there are more than this many buckets in a native histogram, buckets will be merged to stay within the limit. It requires Prometheus >= v2.45.0. format: int64 @@ -9729,21 +9885,21 @@ spec: - type: integer - type: string description: |- - If the growth factor of one bucket to the next is smaller than this, + nativeHistogramMinBucketFactor defines if the growth factor of one bucket to the next is smaller than this, buckets will be merged to increase the factor sufficiently. It requires Prometheus >= v2.50.0. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string nomadSDConfigs: - description: NomadSDConfigs defines a list of Nomad service discovery + description: nomadSDConfigs defines a list of Nomad service discovery configurations. items: description: |- @@ -9752,15 +9908,17 @@ spec: properties: allowStale: description: |- - The information to access the Nomad API. It is to be defined + allowStale defines the information to access the Nomad API. It is to be defined as the Nomad documentation requires. type: boolean authorization: - description: Authorization header to use on every scrape request. + description: |- + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -9785,7 +9943,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -9793,11 +9951,12 @@ spec: type: string type: object basicAuth: - description: BasicAuth information to use on every scrape request. + description: basicAuth defines information to use on every scrape + request. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -9823,7 +9982,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -9849,35 +10008,37 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean namespace: + description: |- + namespace defines the Nomad namespace to query for service discovery. + When specified, only resources within this namespace will be discovered. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string oauth2: - description: |- - Optional OAuth 2.0 configuration. - Cannot be set at the same time as `authorization` or `basic_auth`. + description: oauth2 defines the configuration to use on every + scrape request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9900,7 +10061,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9926,7 +10088,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -9954,12 +10116,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -9993,7 +10155,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10001,33 +10163,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10050,8 +10211,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -10076,12 +10237,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10104,8 +10265,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -10130,11 +10291,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -10159,7 +10321,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10170,7 +10332,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10180,12 +10342,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -10221,7 +10384,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10229,38 +10392,47 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: description: |- - Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. - Supported units: y, w, d, h, m, s, ms - Examples: `30s`, `1m`, `1h20m15s`, `15d` + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: + description: |- + region defines the Nomad region to query for service discovery. + When specified, only resources within this region will be discovered. type: string server: + description: |- + server defines the Nomad server address to connect to for service discovery. + This should be the full URL including protocol (e.g., "https://nomad.example.com:4646"). minLength: 1 type: string tagSeparator: + description: |- + tagSeparator defines the separator used to join multiple tags. + This determines how Nomad service tags are concatenated into Prometheus labels. type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10283,7 +10455,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10308,11 +10481,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10335,7 +10509,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10360,11 +10535,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -10389,7 +10565,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10400,7 +10576,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10410,7 +10586,8 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object required: @@ -10418,15 +10595,17 @@ spec: type: object type: array oauth2: - description: OAuth2 configuration to use on every scrape request. + description: oauth2 defines the configuration to use on every scrape + request. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -10449,7 +10628,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10475,7 +10655,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -10503,12 +10683,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -10542,7 +10722,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -10550,32 +10730,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for the - token request.' + description: scopes defines the OAuth2 scopes used for the token + request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10598,7 +10778,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10623,11 +10804,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10650,7 +10832,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10675,11 +10858,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -10704,7 +10888,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -10715,7 +10899,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -10725,12 +10909,12 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the token - from.' + description: tokenUrl defines the URL to fetch the token from. minLength: 1 type: string required: @@ -10739,7 +10923,7 @@ spec: - tokenUrl type: object openstackSDConfigs: - description: OpenStackSDConfigs defines a list of OpenStack service + description: openstackSDConfigs defines a list of OpenStack service discovery configurations. items: description: |- @@ -10748,15 +10932,15 @@ spec: properties: allTenants: description: |- - Whether the service discovery should list all instances for all projects. + allTenants defines whether the service discovery should list all instances for all projects. It is only relevant for the 'instance' role and usually requires admin permissions. type: boolean applicationCredentialId: - description: ApplicationCredentialID + description: applicationCredentialId defines the OpenStack applicationCredentialId. type: string applicationCredentialName: description: |- - The ApplicationCredentialID or ApplicationCredentialName fields are + applicationCredentialName defines the ApplicationCredentialID or ApplicationCredentialName fields are required if using an application credential to authenticate. Some providers allow you to create an application credential to authenticate rather than a password. @@ -10764,7 +10948,7 @@ spec: type: string applicationCredentialSecret: description: |- - The applicationCredentialSecret field is required if using an application + applicationCredentialSecret defines the required field if using an application credential to authenticate. properties: key: @@ -10789,7 +10973,8 @@ spec: type: object x-kubernetes-map-type: atomic availability: - description: Availability of the endpoint to connect to. + description: availability defines the availability of the endpoint + to connect to. enum: - Public - public @@ -10799,24 +10984,24 @@ spec: - internal type: string domainID: - description: DomainID + description: domainID defines The OpenStack domainID. minLength: 1 type: string domainName: description: |- - At most one of domainId and domainName must be provided if using username + domainName defines at most one of domainId and domainName that must be provided if using username with Identity V3. Otherwise, either are optional. minLength: 1 type: string identityEndpoint: description: |- - IdentityEndpoint specifies the HTTP endpoint that is required to work with + identityEndpoint defines the HTTP endpoint that is required to work with the Identity API of the appropriate version. pattern: ^http(s)?:\/\/.+$ type: string password: description: |- - Password for the Identity V2 and V3 APIs. Consult with your provider's + password defines the password for the Identity V2 and V3 APIs. Consult with your provider's control panel to discover your account's preferred method of authentication. properties: key: @@ -10842,35 +11027,37 @@ spec: x-kubernetes-map-type: atomic port: description: |- - The port to scrape metrics from. If using the public IP address, this must + port defines the port to scrape metrics from. If using the public IP address, this must instead be specified in the relabeling rule. format: int32 maximum: 65535 minimum: 0 type: integer projectID: - description: ' ProjectID' + description: projectID defines the OpenStack projectID. minLength: 1 type: string projectName: description: |- - The ProjectId and ProjectName fields are optional for the Identity V2 API. + projectName defines an optional field for the Identity V2 API. Some providers allow you to specify a ProjectName instead of the ProjectId. Some require both. Your provider's authentication policies will determine how these fields influence authentication. minLength: 1 type: string refreshInterval: - description: Refresh interval to re-read the instance list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string region: - description: The OpenStack Region. + description: region defines the OpenStack Region. minLength: 1 type: string role: description: |- - The OpenStack role of entities that should be discovered. + role defines the OpenStack role of entities that should be discovered. Note: The `LoadBalancer` role requires Prometheus >= v3.2.0. enum: @@ -10879,15 +11066,16 @@ spec: - LoadBalancer type: string tlsConfig: - description: TLS configuration applying to the target HTTP endpoint. + description: tlsConfig defines the TLS configuration applying + to the target HTTP endpoint. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10910,7 +11098,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10935,11 +11124,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -10962,7 +11152,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -10987,11 +11178,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -11016,7 +11208,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11027,7 +11219,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11037,16 +11229,17 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object userid: - description: UserID + description: userid defines the OpenStack userid. minLength: 1 type: string username: description: |- - Username is required if using Identity V2 API. Consult with your provider's + username defines the username required if using Identity V2 API. Consult with your provider's control panel to discover your account's username. In Identity V3, either userid or a combination of username and domainId or domainName are needed @@ -11058,7 +11251,7 @@ spec: type: object type: array ovhcloudSDConfigs: - description: OVHCloudSDConfigs defines a list of OVHcloud service + description: ovhcloudSDConfigs defines a list of OVHcloud service discovery configurations. items: description: |- @@ -11066,11 +11259,15 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ovhcloud_sd_config properties: applicationKey: - description: Access key to use. https://api.ovh.com. + description: |- + applicationKey defines the access key to use for OVHCloud API authentication. + This is obtained from the OVHCloud API credentials at https://api.ovh.com. minLength: 1 type: string applicationSecret: - description: SecretKeySelector selects a key of a Secret. + description: |- + applicationSecret defines the secret key for OVHCloud API authentication. + This contains the application secret obtained during OVHCloud API credential creation. properties: key: description: The key of the secret to select from. Must @@ -11094,7 +11291,9 @@ spec: type: object x-kubernetes-map-type: atomic consumerKey: - description: SecretKeySelector selects a key of a Secret. + description: |- + consumerKey defines the consumer key for OVHCloud API authentication. + This is the third component of OVHCloud's three-key authentication system. properties: key: description: The key of the secret to select from. Must @@ -11118,23 +11317,24 @@ spec: type: object x-kubernetes-map-type: atomic endpoint: - description: Custom endpoint to be used. + description: |- + endpoint defines a custom API endpoint to be used. + When not specified, defaults to the standard OVHCloud API endpoint for the region. minLength: 1 type: string refreshInterval: - description: Refresh interval to re-read the resources list. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string service: - allOf: - - enum: - - VPS - - DedicatedServer - - enum: - - VPS - - DedicatedServer - description: Service of the targets to retrieve. Must be `VPS` - or `DedicatedServer`. + description: |- + service defines the service type of the targets to retrieve. + Must be either `VPS` or `DedicatedServer` to specify which OVHCloud resources to discover. + enum: + - VPS + - DedicatedServer type: string required: - applicationKey @@ -11148,7 +11348,7 @@ spec: items: type: string type: array - description: Optional HTTP URL parameters + description: params defines optional HTTP URL parameters type: object x-kubernetes-map-type: atomic proxyConnectHeader: @@ -11179,7 +11379,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11187,16 +11387,16 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string puppetDBSDConfigs: - description: PuppetDBSDConfigs defines a list of PuppetDB service + description: puppetDBSDConfigs defines a list of PuppetDB service discovery configurations. items: description: |- @@ -11205,12 +11405,12 @@ spec: properties: authorization: description: |- - Optional `authorization` HTTP header configuration. - Cannot be set at the same time as `basicAuth`, or `oauth2`. + authorization defines the header configuration to authenticate against the DigitalOcean API. + Cannot be set at the same time as `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -11235,7 +11435,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -11244,12 +11444,12 @@ spec: type: object basicAuth: description: |- - Optional HTTP basic authentication information. + basicAuth defines information to use on every scrape request. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -11275,7 +11475,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -11301,21 +11501,21 @@ spec: x-kubernetes-map-type: atomic type: object enableHTTP2: - description: Configure whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether the HTTP requests should follow + description: followRedirects defines whether HTTP requests follow HTTP 3xx redirects. type: boolean includeParameters: description: |- - Whether to include the parameters as meta labels. + includeParameters defines whether to include the parameters as meta labels. Note: Enabling this exposes parameters in the Prometheus UI and API. Make sure that you don't have secrets exposed as parameters if you enable this. type: boolean noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -11323,17 +11523,17 @@ spec: type: string oauth2: description: |- - Optional OAuth2.0 configuration. - Cannot be set at the same time as `basicAuth`, or `authorization`. + oauth2 defines the optional OAuth 2.0 configuration to authenticate against the target HTTP endpoint. + Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11356,7 +11556,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11382,7 +11583,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -11410,12 +11611,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -11449,7 +11650,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11457,33 +11658,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11506,8 +11706,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -11532,12 +11732,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11560,8 +11760,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -11586,11 +11786,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -11615,7 +11816,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11626,7 +11827,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11636,12 +11837,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -11650,7 +11852,8 @@ spec: - tokenUrl type: object port: - description: Port to scrape the metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 @@ -11683,7 +11886,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -11691,34 +11894,37 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string query: description: |- - Puppet Query Language (PQL) query. Only resources are supported. + query defines the Puppet Query Language (PQL) query. Only resources are supported. https://puppet.com/docs/puppetdb/latest/api/query/v4/pql.html minLength: 1 type: string refreshInterval: - description: Refresh interval to re-read the list of resources. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string tlsConfig: - description: TLS configuration to connect to the Puppet DB. + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11741,7 +11947,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11766,11 +11973,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -11793,7 +12001,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -11818,11 +12027,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -11847,7 +12057,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -11858,7 +12068,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -11868,11 +12078,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the PuppetDB root query endpoint. + description: url defines the URL of the PuppetDB root query + endpoint. minLength: 1 pattern: ^http(s)?://.+$ type: string @@ -11883,7 +12095,7 @@ spec: type: array relabelings: description: |- - RelabelConfigs defines how to rewrite the target's labels before scraping. + relabelings defines how to rewrite the target's labels before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields. The original scrape job's name is available via the `__tmp_prometheus_job_name` label. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config @@ -11897,7 +12109,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -11929,40 +12141,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against which + the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated SourceLabels. + description: separator defines the string between concatenated + SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -11973,12 +12186,12 @@ spec: minItems: 1 type: array sampleLimit: - description: SampleLimit defines per-scrape limit on number of scraped + description: sampleLimit defines per-scrape limit on number of scraped samples that will be accepted. format: int64 type: integer scalewaySDConfigs: - description: ScalewaySDConfigs defines a list of Scaleway instances + description: scalewaySDConfigs defines a list of Scaleway instances and baremetal service discovery configurations. items: description: |- @@ -11986,41 +12199,43 @@ spec: See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scaleway_sd_config properties: accessKey: - description: Access key to use. https://console.scaleway.com/project/credentials + description: accessKey defines the access key to use. https://console.scaleway.com/project/credentials minLength: 1 type: string apiURL: - description: API URL to use when doing the server listing requests. + description: apiURL defines the API URL to use when doing the + server listing requests. pattern: ^http(s)?://.+$ type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: - description: Configure whether HTTP requests follow HTTP 3xx - redirects. + description: followRedirects defines whether HTTP requests follow + HTTP 3xx redirects. type: boolean nameFilter: - description: NameFilter specify a name filter (works as a LIKE) + description: nameFilter defines a name filter (works as a LIKE) to apply on the server listing request. minLength: 1 type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: string port: - description: The port to scrape metrics from. + description: port defines the port to scrape metrics from. If + using the public IP address, this must format: int32 maximum: 65535 minimum: 0 type: integer projectID: - description: Project ID of the targets. + description: projectID defines the Project ID of the targets. minLength: 1 type: string proxyConnectHeader: @@ -12051,7 +12266,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -12059,27 +12274,30 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string refreshInterval: - description: Refresh interval to re-read the list of instances. + description: |- + refreshInterval defines the time after which the provided names are refreshed. + If not set, Prometheus uses its default value. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string role: - description: Service of the targets to retrieve. Must be `Instance` - or `Baremetal`. + description: role defines the service of the targets to retrieve. + Must be `Instance` or `Baremetal`. enum: - Instance - Baremetal type: string secretKey: - description: Secret key to use when listing targets. + description: secretKey defines the secret key to use when listing + targets. properties: key: description: The key of the secret to select from. Must @@ -12103,7 +12321,7 @@ spec: type: object x-kubernetes-map-type: atomic tagsFilter: - description: TagsFilter specify a tag filter (a server needs + description: tagsFilter defines a tag filter (a server needs to have all defined tags to be listed) to apply on the server listing request. items: @@ -12113,15 +12331,16 @@ spec: type: array x-kubernetes-list-type: set tlsConfig: - description: TLS configuration to use on every scrape request + description: tlsConfig defines the TLS configuration to connect + to the Consul API. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -12144,7 +12363,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -12169,11 +12389,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -12196,7 +12417,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -12221,11 +12443,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -12250,7 +12473,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -12261,7 +12484,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -12271,12 +12494,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object zone: - description: Zone is the availability zone of your targets (e.g. - fr-par-1). + description: zone defines the availability zone of your targets + (e.g. fr-par-1). minLength: 1 type: string required: @@ -12288,30 +12512,31 @@ spec: type: array scheme: description: |- - Configures the protocol scheme used for requests. + scheme defines the protocol scheme used for requests. If empty, Prometheus uses HTTP by default. enum: - HTTP - HTTPS type: string scrapeClass: - description: The scrape class to apply. + description: scrapeClass defines the scrape class to apply. minLength: 1 type: string scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. It requires Prometheus >= v2.45.0. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. type: boolean scrapeInterval: - description: ScrapeInterval is the interval between consecutive scrapes. + description: scrapeInterval defines the interval between consecutive + scrapes. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string scrapeProtocols: description: |- - The protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -12338,12 +12563,12 @@ spec: x-kubernetes-list-type: set scrapeTimeout: description: |- - ScrapeTimeout is the number of seconds to wait until a scrape request times out. + scrapeTimeout defines the number of seconds to wait until a scrape request times out. The value cannot be greater than the scrape interval otherwise the operator will reject the resource. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string staticConfigs: - description: StaticConfigs defines a list of static targets with a + description: staticConfigs defines a list of static targets with a common label set. items: description: |- @@ -12353,12 +12578,13 @@ spec: labels: additionalProperties: type: string - description: Labels assigned to all metrics scraped from the - targets. + description: labels defines labels assigned to all metrics scraped + from the targets. type: object x-kubernetes-map-type: atomic targets: - description: List of targets for this static configuration. + description: targets defines the list of targets for this static + configuration. items: description: |- Target represents a target for Prometheus to scrape @@ -12372,19 +12598,21 @@ spec: type: object type: array targetLimit: - description: TargetLimit defines a limit on the number of scraped + description: targetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer tlsConfig: - description: TLS configuration to use on every scrape request + description: tlsConfig defines the TLS configuration to use on every + scrape request properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when verifying + server certificates. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -12407,7 +12635,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -12432,10 +12661,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present when + doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -12458,7 +12689,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -12483,10 +12715,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keySecret: - description: Secret containing the client key file for the targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must be @@ -12511,7 +12745,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -12522,7 +12756,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -12532,19 +12766,125 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for the + targets. type: string type: object trackTimestampsStaleness: description: |- - TrackTimestampsStaleness whether Prometheus tracks staleness of + trackTimestampsStaleness defines whether Prometheus tracks staleness of the metrics that have an explicit timestamp present in scraped data. Has no effect if `honorTimestamps` is false. It requires Prometheus >= v2.48.0. type: boolean type: object + status: + description: |- + status defines the status subresource. It is under active development and is updated only when the + "StatusForConfigurationResources" feature gate is enabled. + + Most recent observed status of the ScrapeConfig. Read-only. + More info: + https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + bindings: + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. + items: + description: WorkloadBinding is a link between a configuration resource + and a workload resource. + properties: + conditions: + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. + items: + description: ConfigResourceCondition describes the status + of configuration resources linked to Prometheus, PrometheusAgent, + Alertmanager or ThanosRuler. + properties: + lastTransitionTime: + description: lastTransitionTime defines the time of the + last update to the current status property. + format: date-time + type: string + message: + description: message defines the human-readable message + indicating details for the condition's last transition. + type: string + observedGeneration: + description: |- + observedGeneration defines the .metadata.generation that the + condition was set based upon. For instance, if `.metadata.generation` is + currently 12, but the `.status.conditions[].observedGeneration` is 9, the + condition is out of date with respect to the current state of the object. + format: int64 + type: integer + reason: + description: reason for the condition's last transition. + type: string + status: + description: status of the condition. + minLength: 1 + type: string + type: + description: |- + type of the condition being reported. + Currently, only "Accepted" is supported. + enum: + - Accepted + minLength: 1 + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + group: + description: group defines the group of the referenced resource. + enum: + - monitoring.coreos.com + type: string + name: + description: name defines the name of the referenced object. + minLength: 1 + type: string + namespace: + description: namespace defines the namespace of the referenced + object. + minLength: 1 + type: string + resource: + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). + enum: + - prometheuses + - prometheusagents + - thanosrulers + - alertmanagers + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map + type: object required: - spec type: object served: true storage: true + subresources: + status: {} diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_servicemonitors.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_servicemonitors.yaml index 5913993c581..f1457ee9271 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_servicemonitors.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_servicemonitors.yaml @@ -50,19 +50,19 @@ spec: type: object spec: description: |- - Specification of desired Service selection for target discovery by + spec defines the specification of desired Service selection for target discovery by Prometheus. properties: attachMetadata: description: |- - `attachMetadata` defines additional metadata which is added to the + attachMetadata defines additional metadata which is added to the discovered targets. It requires Prometheus >= v2.37.0. properties: node: description: |- - When set to true, Prometheus attaches node metadata to the discovered + node when set to true, Prometheus attaches node metadata to the discovered targets. The Prometheus service account must have the `list` and `watch` @@ -71,7 +71,7 @@ spec: type: object bodySizeLimit: description: |- - When defined, bodySizeLimit specifies a job level limit on the size + bodySizeLimit when defined, bodySizeLimit specifies a job level limit on the size of uncompressed response body that will be accepted by Prometheus. It requires Prometheus >= v2.28.0. @@ -79,12 +79,12 @@ spec: type: string convertClassicHistogramsToNHCB: description: |- - Whether to convert all scraped classic histograms into a native histogram with custom buckets. + convertClassicHistogramsToNHCB defines whether to convert all scraped classic histograms into a native histogram with custom buckets. It requires Prometheus >= v3.0.0. type: boolean endpoints: description: |- - List of endpoints part of this ServiceMonitor. + endpoints defines the list of endpoints part of this ServiceMonitor. Defines how to scrape metrics from Kubernetes [Endpoints](https://kubernetes.io/docs/concepts/services-networking/service/#endpoints) objects. In most cases, an Endpoints object is backed by a Kubernetes [Service](https://kubernetes.io/docs/concepts/services-networking/service/) object with the same name and labels. items: @@ -94,14 +94,14 @@ spec: properties: authorization: description: |- - `authorization` configures the Authorization header credentials to use when + authorization configures the Authorization header credentials to use when scraping the target. Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -126,7 +126,7 @@ spec: x-kubernetes-map-type: atomic type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -135,14 +135,14 @@ spec: type: object basicAuth: description: |- - `basicAuth` configures the Basic Authentication credentials to use when + basicAuth defines the Basic Authentication credentials to use when scraping the target. Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -168,7 +168,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -195,13 +195,13 @@ spec: type: object bearerTokenFile: description: |- - File to read bearer token for scraping the target. + bearerTokenFile defines the file to read bearer token for scraping the target. Deprecated: use `authorization` instead. type: string bearerTokenSecret: description: |- - `bearerTokenSecret` specifies a key of a Secret containing the bearer + bearerTokenSecret defines a key of a Secret containing the bearer token for scraping targets. The secret needs to be in the same namespace as the ServiceMonitor object and readable by the Prometheus Operator. @@ -229,12 +229,12 @@ spec: type: object x-kubernetes-map-type: atomic enableHttp2: - description: '`enableHttp2` can be used to disable HTTP2 when - scraping the target.' + description: enableHttp2 can be used to disable HTTP2 when scraping + the target. type: boolean filterRunning: description: |- - When true, the pods which are not running (e.g. either in Failed or + filterRunning when true, the pods which are not running (e.g. either in Failed or Succeeded state) are dropped during the target discovery. If unset, the filtering is enabled. @@ -243,29 +243,29 @@ spec: type: boolean followRedirects: description: |- - `followRedirects` defines whether the scrape requests should follow HTTP + followRedirects defines whether the scrape requests should follow HTTP 3xx redirects. type: boolean honorLabels: description: |- - When true, `honorLabels` preserves the metric's labels when they collide + honorLabels defines when true the metric's labels when they collide with the target's labels. type: boolean honorTimestamps: description: |- - `honorTimestamps` controls whether Prometheus preserves the timestamps + honorTimestamps defines whether Prometheus preserves the timestamps when exposed by the target. type: boolean interval: description: |- - Interval at which Prometheus scrapes the metrics from the target. + interval at which Prometheus scrapes the metrics from the target. If empty, Prometheus uses the global scrape interval. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string metricRelabelings: description: |- - `metricRelabelings` configures the relabeling rules to apply to the + metricRelabelings defines the relabeling rules to apply to the samples before ingestion. items: description: |- @@ -277,7 +277,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -309,41 +309,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -354,7 +354,7 @@ spec: type: array noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -362,7 +362,7 @@ spec: type: string oauth2: description: |- - `oauth2` configures the OAuth2 settings to use when scraping the target. + oauth2 defines the OAuth2 settings to use when scraping the target. It requires Prometheus >= 2.27.0. @@ -370,12 +370,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -398,7 +398,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -424,7 +425,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -452,12 +453,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -491,7 +492,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -499,33 +500,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -548,8 +548,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -574,12 +574,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -602,8 +602,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -628,11 +628,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -657,7 +658,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -668,7 +669,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -678,12 +679,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -700,13 +702,13 @@ spec: type: object path: description: |- - HTTP path from which to scrape for metrics. + path defines the HTTP path from which to scrape for metrics. If empty, Prometheus uses the default value (e.g. `/metrics`). type: string port: description: |- - Name of the Service port which this endpoint refers to. + port defines the name of the Service port which this endpoint refers to. It takes precedence over `targetPort`. type: string @@ -738,7 +740,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -746,17 +748,17 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string relabelings: description: |- - `relabelings` configures the relabeling rules to apply the target's + relabelings defines the relabeling rules to apply the target's metadata labels. The Operator automatically adds relabelings for a few standard Kubernetes fields. @@ -774,7 +776,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -806,41 +808,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -851,7 +853,7 @@ spec: type: array scheme: description: |- - HTTP scheme to use for scraping. + scheme defines the HTTP scheme to use for scraping. `http` and `https` are the expected values unless you rewrite the `__scheme__` label via relabeling. @@ -863,7 +865,7 @@ spec: type: string scrapeTimeout: description: |- - Timeout after which Prometheus considers the scrape to be failed. + scrapeTimeout defines the timeout after which Prometheus considers the scrape to be failed. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used. @@ -875,19 +877,20 @@ spec: - type: integer - type: string description: |- - Name or number of the target port of the `Pod` object behind the + targetPort defines the name or number of the target port of the `Pod` object behind the Service. The port must be specified with the container's port property. x-kubernetes-int-or-string: true tlsConfig: - description: TLS configuration to use when scraping the target. + description: tlsConfig defines the TLS configuration to use + when scraping the target. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -910,7 +913,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -935,15 +939,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -966,7 +971,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -991,19 +997,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -1028,7 +1035,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -1039,7 +1046,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -1049,12 +1056,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object trackTimestampsStaleness: description: |- - `trackTimestampsStaleness` defines whether Prometheus tracks staleness of + trackTimestampsStaleness defines whether Prometheus tracks staleness of the metrics that have an explicit timestamp present in scraped data. Has no effect if `honorTimestamps` is false. @@ -1064,7 +1072,7 @@ spec: type: array fallbackScrapeProtocol: description: |- - The protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. + fallbackScrapeProtocol defines the protocol to use if a scrape returns blank, unparseable, or otherwise invalid Content-Type. It requires Prometheus >= v3.0.0. enum: @@ -1076,7 +1084,7 @@ spec: type: string jobLabel: description: |- - `jobLabel` selects the label from the associated Kubernetes `Service` + jobLabel selects the label from the associated Kubernetes `Service` object which will be used as the `job` label for all metrics. For example if `jobLabel` is set to `foo` and the Kubernetes `Service` @@ -1089,7 +1097,7 @@ spec: type: string keepDroppedTargets: description: |- - Per-scrape limit on the number of targets dropped by relabeling + keepDroppedTargets defines the per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. It requires Prometheus >= v2.47.0. @@ -1097,44 +1105,45 @@ spec: type: integer labelLimit: description: |- - Per-scrape limit on number of labels that will be accepted for a sample. + labelLimit defines the per-scrape limit on number of labels that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer labelNameLengthLimit: description: |- - Per-scrape limit on length of labels name that will be accepted for a sample. + labelNameLengthLimit defines the per-scrape limit on length of labels name that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer labelValueLengthLimit: description: |- - Per-scrape limit on length of labels value that will be accepted for a sample. + labelValueLengthLimit defines the per-scrape limit on length of labels value that will be accepted for a sample. It requires Prometheus >= v2.27.0. format: int64 type: integer namespaceSelector: description: |- - `namespaceSelector` defines in which namespace(s) Prometheus should discover the services. + namespaceSelector defines in which namespace(s) Prometheus should discover the services. By default, the services are discovered in the same namespace as the `ServiceMonitor` object but it is possible to select pods across different/all namespaces. properties: any: description: |- - Boolean describing whether all namespaces are selected in contrast to a + any defines the boolean describing whether all namespaces are selected in contrast to a list restricting them. type: boolean matchNames: - description: List of namespace names to select from. + description: matchNames defines the list of namespace names to + select from. items: type: string type: array type: object nativeHistogramBucketLimit: description: |- - If there are more than this many buckets in a native histogram, + nativeHistogramBucketLimit defines ff there are more than this many buckets in a native histogram, buckets will be merged to stay within the limit. It requires Prometheus >= v2.45.0. format: int64 @@ -1144,38 +1153,38 @@ spec: - type: integer - type: string description: |- - If the growth factor of one bucket to the next is smaller than this, + nativeHistogramMinBucketFactor defines if the growth factor of one bucket to the next is smaller than this, buckets will be merged to increase the factor sufficiently. It requires Prometheus >= v2.50.0. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true podTargetLabels: description: |- - `podTargetLabels` defines the labels which are transferred from the + podTargetLabels defines the labels which are transferred from the associated Kubernetes `Pod` object onto the ingested metrics. items: type: string type: array sampleLimit: description: |- - `sampleLimit` defines a per-scrape limit on the number of scraped samples + sampleLimit defines a per-scrape limit on the number of scraped samples that will be accepted. format: int64 type: integer scrapeClass: - description: The scrape class to apply. + description: scrapeClass defines the scrape class to apply. minLength: 1 type: string scrapeClassicHistograms: description: |- - Whether to scrape a classic histogram that is also exposed as a native histogram. + scrapeClassicHistograms defines whether to scrape a classic histogram that is also exposed as a native histogram. It requires Prometheus >= v2.45.0. Notice: `scrapeClassicHistograms` corresponds to the `always_scrape_classic_histograms` field in the Prometheus configuration. type: boolean scrapeProtocols: description: |- - `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the + scrapeProtocols defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). If unset, Prometheus uses its default value. @@ -1200,8 +1209,8 @@ spec: type: array x-kubernetes-list-type: set selector: - description: Label selector to select the Kubernetes `Endpoints` objects - to scrape metrics from. + description: selector defines the label selector to select the Kubernetes + `Endpoints` objects to scrape metrics from. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. @@ -1248,7 +1257,7 @@ spec: x-kubernetes-map-type: atomic selectorMechanism: description: |- - Mechanism used to select the endpoints to scrape. + selectorMechanism defines the mechanism used to select the endpoints to scrape. By default, the selection process relies on relabel configurations to filter the discovered targets. Alternatively, you can opt in for role selectors, which may offer better efficiency in large clusters. Which strategy is best for your use case needs to be carefully evaluated. @@ -1258,16 +1267,27 @@ spec: - RelabelConfig - RoleSelector type: string + serviceDiscoveryRole: + description: |- + serviceDiscoveryRole defines the service discovery role used to discover targets. + + If set, the value should be either "Endpoints" or "EndpointSlice". + Otherwise it defaults to the value defined in the + Prometheus/PrometheusAgent resource. + enum: + - Endpoints + - EndpointSlice + type: string targetLabels: description: |- - `targetLabels` defines the labels which are transferred from the + targetLabels defines the labels which are transferred from the associated Kubernetes `Service` object onto the ingested metrics. items: type: string type: array targetLimit: description: |- - `targetLimit` defines a limit on the number of scraped targets that will + targetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer @@ -1277,7 +1297,7 @@ spec: type: object status: description: |- - This Status subresource is under active development and is updated only when the + status defines the status subresource. It is under active development and is updated only when the "StatusForConfigurationResources" feature gate is enabled. Most recent observed status of the ServiceMonitor. Read-only. @@ -1285,47 +1305,48 @@ spec: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: bindings: - description: The list of workload resources (Prometheus or PrometheusAgent) - which select the configuration resource. + description: bindings defines the list of workload resources (Prometheus, + PrometheusAgent, ThanosRuler or Alertmanager) which select the configuration + resource. items: description: WorkloadBinding is a link between a configuration resource and a workload resource. properties: conditions: - description: The current state of the configuration resource - when bound to the referenced Prometheus object. + description: conditions defines the current state of the configuration + resource when bound to the referenced Workload object. items: description: ConfigResourceCondition describes the status of configuration resources linked to Prometheus, PrometheusAgent, - Alertmanager, or ThanosRuler. + Alertmanager or ThanosRuler. properties: lastTransitionTime: - description: LastTransitionTime is the time of the last - update to the current status property. + description: lastTransitionTime defines the time of the + last update to the current status property. format: date-time type: string message: - description: Human-readable message indicating details - for the condition's last transition. + description: message defines the human-readable message + indicating details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the object. format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: description: |- - Type of the condition being reported. + type of the condition being reported. Currently, only "Accepted" is supported. enum: - Accepted @@ -1341,24 +1362,27 @@ spec: - type x-kubernetes-list-type: map group: - description: The group of the referenced resource. + description: group defines the group of the referenced resource. enum: - monitoring.coreos.com type: string name: - description: The name of the referenced object. + description: name defines the name of the referenced object. minLength: 1 type: string namespace: - description: The namespace of the referenced object. + description: namespace defines the namespace of the referenced + object. minLength: 1 type: string resource: - description: The type of resource being referenced (e.g. Prometheus - or PrometheusAgent). + description: resource defines the type of resource being referenced + (e.g. Prometheus, PrometheusAgent, ThanosRuler or Alertmanager). enum: - prometheuses - prometheusagents + - thanosrulers + - alertmanagers type: string required: - group @@ -1367,6 +1391,12 @@ spec: - resource type: object type: array + x-kubernetes-list-map-keys: + - group + - resource + - name + - namespace + x-kubernetes-list-type: map type: object required: - spec diff --git a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_thanosrulers.yaml b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_thanosrulers.yaml index 2285730da9d..6cb7205c60f 100644 --- a/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_thanosrulers.yaml +++ b/pkg/component/observability/monitoring/prometheusoperator/templates/crd-monitoring.coreos.com_thanosrulers.yaml @@ -74,12 +74,12 @@ spec: type: object spec: description: |- - Specification of the desired behavior of the ThanosRuler cluster. More info: + spec defines the specification of the desired behavior of the ThanosRuler cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: additionalArgs: description: |- - AdditionalArgs allows setting additional arguments for the ThanosRuler container. + additionalArgs defines how to add additional arguments for the ThanosRuler container. It is intended for e.g. activating hidden flags which are not supported by the dedicated configuration options yet. The arguments are passed as-is to the ThanosRuler container which may cause issues if they are invalid or not supported @@ -91,19 +91,20 @@ spec: description: Argument as part of the AdditionalArgs list. properties: name: - description: Name of the argument, e.g. "scrape.discovery-reload-interval". + description: name of the argument, e.g. "scrape.discovery-reload-interval". minLength: 1 type: string value: - description: Argument value, e.g. 30s. Can be empty for name-only - arguments (e.g. --storage.tsdb.no-lockfile) + description: value defines the argument value, e.g. 30s. Can + be empty for name-only arguments (e.g. --storage.tsdb.no-lockfile) type: string required: - name type: object type: array affinity: - description: If specified, the pod's scheduling constraints. + description: affinity defines when specified, the pod's scheduling + constraints. properties: nodeAffinity: description: Describes node affinity scheduling rules for the @@ -1016,7 +1017,7 @@ spec: type: object alertDropLabels: description: |- - Configures the label names which should be dropped in Thanos Ruler + alertDropLabels defines the label names which should be dropped in Thanos Ruler alerts. The replica label `thanos_ruler_replica` will always be dropped from the alerts. @@ -1025,13 +1026,13 @@ spec: type: array alertQueryUrl: description: |- - The external Query URL the Thanos Ruler will set in the 'Source' field + alertQueryUrl defines how Thanos Ruler will set in the 'Source' field of all alerts. Maps to the '--alert.query-url' CLI arg. type: string alertRelabelConfigFile: description: |- - Configures the path to the alert relabeling configuration file. + alertRelabelConfigFile defines the path to the alert relabeling configuration file. Alert relabel configuration must have the form as specified in the official Prometheus documentation: @@ -1043,7 +1044,7 @@ spec: type: string alertRelabelConfigs: description: |- - Configures alert relabeling in Thanos Ruler. + alertRelabelConfigs defines the alert relabeling in Thanos Ruler. Alert relabel configuration must have the form as specified in the official Prometheus documentation: @@ -1075,7 +1076,7 @@ spec: x-kubernetes-map-type: atomic alertmanagersConfig: description: |- - Configures the list of Alertmanager endpoints to send alerts to. + alertmanagersConfig defines the list of Alertmanager endpoints to send alerts to. The configuration format is defined at https://thanos.io/tip/components/rule.md/#alertmanager. @@ -1107,7 +1108,7 @@ spec: x-kubernetes-map-type: atomic alertmanagersUrl: description: |- - Configures the list of Alertmanager endpoints to send alerts to. + alertmanagersUrl defines the list of Alertmanager endpoints to send alerts to. For Thanos >= v0.10.0, it is recommended to use `alertmanagersConfig` instead. @@ -1117,7 +1118,7 @@ spec: type: array containers: description: |- - Containers allows injecting additional containers or modifying operator generated + containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a ThanosRuler pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a @@ -2644,11 +2645,12 @@ spec: type: object type: array dnsConfig: - description: Defines the DNS configuration for the pods. + description: dnsConfig defines Defines the DNS configuration for the + pods. properties: nameservers: description: |- - A list of DNS name server IP addresses. + nameservers defines the list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. items: minLength: 1 @@ -2657,7 +2659,7 @@ spec: x-kubernetes-list-type: set options: description: |- - A list of DNS resolver options. + options defines the list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Resolution options given in Options will override those that appear in the base DNSPolicy. @@ -2666,11 +2668,11 @@ spec: of a pod. properties: name: - description: Name is required and must be unique. + description: name is required and must be unique. minLength: 1 type: string value: - description: Value is optional. + description: value is optional. type: string required: - name @@ -2681,7 +2683,7 @@ spec: x-kubernetes-list-type: map searches: description: |- - A list of DNS search domains for host-name lookup. + searches defines the list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. items: minLength: 1 @@ -2690,7 +2692,7 @@ spec: x-kubernetes-list-type: set type: object dnsPolicy: - description: Defines the DNS policy for the pods. + description: dnsPolicy defines the DNS policy for the pods. enum: - ClusterFirstWithHostNet - ClusterFirst @@ -2699,7 +2701,7 @@ spec: type: string enableFeatures: description: |- - Enable access to Thanos Ruler feature flags. By default, no features are enabled. + enableFeatures defines how to setup Thanos Ruler feature flags. By default, no features are enabled. Enabling features which are disabled by default is entirely outside the scope of what the maintainers will support and by doing so, you accept @@ -2714,23 +2716,24 @@ spec: type: array x-kubernetes-list-type: set enableServiceLinks: - description: Indicates whether information about services should be - injected into pod's environment variables + description: enableServiceLinks defines whether information about + services should be injected into pod's environment variables type: boolean enforcedNamespaceLabel: description: |- - EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert + enforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created. type: string evaluationInterval: default: 15s - description: Interval between consecutive evaluations. + description: evaluationInterval defines the interval between consecutive + evaluations. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string excludedFromEnforcement: description: |- - List of references to PrometheusRule objects + excludedFromEnforcement defines the list of references to PrometheusRule objects to be excluded from enforcing a namespace label of origin. Applies only if enforcedNamespaceLabel set to true. items: @@ -2739,23 +2742,23 @@ spec: properties: group: default: monitoring.coreos.com - description: Group of the referent. When not specified, it defaults + description: group of the referent. When not specified, it defaults to `monitoring.coreos.com` enum: - monitoring.coreos.com type: string name: - description: Name of the referent. When not set, all resources + description: name of the referent. When not set, all resources in the namespace are matched. type: string namespace: description: |- - Namespace of the referent. + namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ minLength: 1 type: string resource: - description: Resource of the referent. + description: resource of the referent. enum: - prometheusrules - servicemonitors @@ -2770,23 +2773,24 @@ spec: type: array externalPrefix: description: |- - The external URL the Thanos Ruler instances will be available under. This is + externalPrefix defines the Thanos Ruler instances will be available under. This is necessary to generate correct URLs. This is necessary if Thanos Ruler is not served from root of a DNS name. type: string grpcServerTlsConfig: description: |- - GRPCServerTLSConfig configures the gRPC server from which Thanos Querier reads + grpcServerTlsConfig defines the gRPC server from which Thanos Querier reads recorded rule data. Note: Currently only the CAFile, CertFile, and KeyFile fields are supported. Maps to the '--grpc-server-tls-*' CLI args. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when verifying + server certificates. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -2809,7 +2813,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2834,14 +2839,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container to - use for the targets. + description: caFile defines the path to the CA cert in the Prometheus + container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present when + doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the targets. + description: configMap defines the ConfigMap containing data + to use for the targets. properties: key: description: The key to select. @@ -2864,7 +2871,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data to + use for the targets. properties: key: description: The key of the secret to select from. Must @@ -2889,18 +2897,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus container - for the targets. + description: certFile defines the path to the client cert file + in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus container - for the targets. + description: keyFile defines the path to the client key file in + the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the targets. + description: keySecret defines the Secret containing the client + key file for the targets. properties: key: description: The key of the secret to select from. Must be @@ -2925,7 +2935,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -2936,7 +2946,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -2946,23 +2956,24 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for the + targets. type: string type: object hostAliases: - description: Pods' hostAliases configuration + description: hostAliases defines pods' hostAliases configuration items: description: |- HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. + description: hostnames defines hostnames for the above IP address. items: type: string type: array ip: - description: IP address of the host file entry. + description: ip defines the IP address of the host file entry. type: string required: - hostnames @@ -2974,7 +2985,7 @@ spec: x-kubernetes-list-type: map hostUsers: description: |- - HostUsers supports the user space in Kubernetes. + hostUsers supports the user space in Kubernetes. More info: https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/ @@ -2982,11 +2993,11 @@ spec: Starting Kubernetes 1.33, the feature is enabled by default. type: boolean image: - description: Thanos container image URL. + description: image defines Thanos container image URL. type: string imagePullPolicy: description: |- - Image pull policy for the 'thanos', 'init-config-reloader' and 'config-reloader' containers. + imagePullPolicy defines for the 'thanos', 'init-config-reloader' and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. enum: - "" @@ -2996,7 +3007,7 @@ spec: type: string imagePullSecrets: description: |- - An optional list of references to secrets in the same namespace + imagePullSecrets defines an optional list of references to secrets in the same namespace to use for pulling thanos images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod items: @@ -3018,7 +3029,7 @@ spec: type: array initContainers: description: |- - InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. + initContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the ThanosRuler configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ @@ -4548,25 +4559,25 @@ spec: additionalProperties: type: string description: |- - Configures the external label pairs of the ThanosRuler resource. + labels defines the external label pairs of the ThanosRuler resource. A default replica label `thanos_ruler_replica` will be always added as a label with the value of the pod's name. type: object listenLocal: description: |- - ListenLocal makes the Thanos ruler listen on loopback, so that it + listenLocal defines the Thanos ruler listen on loopback, so that it does not bind against the Pod IP. type: boolean logFormat: - description: Log format for ThanosRuler to be configured with. + description: logFormat for ThanosRuler to be configured with. enum: - "" - logfmt - json type: string logLevel: - description: Log level for ThanosRuler to be configured with. + description: logLevel for ThanosRuler to be configured with. enum: - "" - debug @@ -4576,7 +4587,7 @@ spec: type: string minReadySeconds: description: |- - Minimum number of seconds for which a newly created pod should be ready + minReadySeconds defines the minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. If unset, pods will be considered available as soon as they are ready. @@ -4586,13 +4597,12 @@ spec: nodeSelector: additionalProperties: type: string - description: Define which Nodes the Pods are scheduled on. + description: nodeSelector defines which Nodes the Pods are scheduled + on. type: object objectStorageConfig: description: |- - Configures object storage. - - The configuration format is defined at https://thanos.io/tip/thanos/storage.md/#configuring-access-to-object-storage + objectStorageConfig defines the configuration format is defined at https://thanos.io/tip/thanos/storage.md/#configuring-access-to-object-storage The operator performs no validation of the configuration. @@ -4620,7 +4630,7 @@ spec: x-kubernetes-map-type: atomic objectStorageConfigFile: description: |- - Configures the path of the object storage configuration file. + objectStorageConfigFile defines the path of the object storage configuration file. The configuration format is defined at https://thanos.io/tip/thanos/storage.md/#configuring-access-to-object-storage @@ -4630,12 +4640,12 @@ spec: type: string paused: description: |- - When a ThanosRuler deployment is paused, no actions except for deletion + paused defines when a ThanosRuler deployment is paused, no actions except for deletion will be performed on the underlying objects. type: boolean podMetadata: description: |- - PodMetadata configures labels and annotations which are propagated to the ThanosRuler pods. + podMetadata defines labels and annotations which are propagated to the ThanosRuler pods. The following items are reserved and cannot be overridden: * "app.kubernetes.io/name" label, set to "thanos-ruler". @@ -4648,7 +4658,7 @@ spec: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -4657,14 +4667,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -4675,15 +4685,16 @@ spec: portName: default: web description: |- - Port name used for the pods and governing service. + portName defines the port name used for the pods and governing service. Defaults to `web`. type: string priorityClassName: - description: Priority class assigned to the Pods + description: priorityClassName defines the priority class assigned + to the Pods type: string prometheusRulesExcludedFromEnforce: description: |- - PrometheusRulesExcludedFromEnforce - list of Prometheus rules to be excluded from enforcing + prometheusRulesExcludedFromEnforce defines a list of Prometheus rules to be excluded from enforcing of adding namespace labels. Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair Deprecated: use excludedFromEnforcement instead. @@ -4694,10 +4705,12 @@ spec: namespace label for alerts and metrics. properties: ruleName: - description: Name of the excluded PrometheusRule object. + description: ruleName defines the name of the excluded PrometheusRule + object. type: string ruleNamespace: - description: Namespace of the excluded PrometheusRule object. + description: ruleNamespace defines the namespace of the excluded + PrometheusRule object. type: string required: - ruleName @@ -4706,7 +4719,7 @@ spec: type: array queryConfig: description: |- - Configures the list of Thanos Query endpoints from which to query metrics. + queryConfig defines the list of Thanos Query endpoints from which to query metrics. The configuration format is defined at https://thanos.io/tip/components/rule.md/#query-api @@ -4738,7 +4751,7 @@ spec: x-kubernetes-map-type: atomic queryEndpoints: description: |- - Configures the list of Thanos Query endpoints from which to query metrics. + queryEndpoints defines the list of Thanos Query endpoints from which to query metrics. For Thanos >= v0.11.0, it is recommended to use `queryConfig` instead. @@ -4748,7 +4761,7 @@ spec: type: array remoteWrite: description: |- - Defines the list of remote write configurations. + remoteWrite defines the list of remote write configurations. When the list isn't empty, the ruler is configured with stateless mode. @@ -4760,15 +4773,15 @@ spec: properties: authorization: description: |- - Authorization section for the URL. + authorization section for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. Cannot be set at the same time as `sigv4`, `basicAuth`, `oauth2`, or `azureAd`. properties: credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. + description: credentials defines a key of a Secret in the + namespace that contains the credentials for authentication. properties: key: description: The key of the secret to select from. Must @@ -4792,12 +4805,12 @@ spec: type: object x-kubernetes-map-type: atomic credentialsFile: - description: File to read a secret from, mutually exclusive - with `credentials`. + description: credentialsFile defines the file to read a + secret from, mutually exclusive with `credentials`. type: string type: description: |- - Defines the authentication type. The value is case-insensitive. + type defines the authentication type. The value is case-insensitive. "Basic" is not a supported value. @@ -4806,15 +4819,15 @@ spec: type: object azureAd: description: |- - AzureAD for the URL. + azureAd for the URL. It requires Prometheus >= v2.45.0 or Thanos >= v0.31.0. Cannot be set at the same time as `authorization`, `basicAuth`, `oauth2`, or `sigv4`. properties: cloud: - description: The Azure Cloud. Options are 'AzurePublic', - 'AzureChina', or 'AzureGovernment'. + description: cloud defines the Azure Cloud. Options are + 'AzurePublic', 'AzureChina', or 'AzureGovernment'. enum: - AzureChina - AzureGovernment @@ -4822,32 +4835,33 @@ spec: type: string managedIdentity: description: |- - ManagedIdentity defines the Azure User-assigned Managed identity. + managedIdentity defines the Azure User-assigned Managed identity. Cannot be set at the same time as `oauth` or `sdk`. properties: clientId: - description: The client id + description: clientId defines defines the Azure User-assigned + Managed identity. type: string required: - clientId type: object oauth: description: |- - OAuth defines the oauth config that is being used to authenticate. + oauth defines the oauth config that is being used to authenticate. Cannot be set at the same time as `managedIdentity` or `sdk`. It requires Prometheus >= v2.48.0 or Thanos >= v0.31.0. properties: clientId: - description: '`clientID` is the clientId of the Azure + description: clientId defines the clientId of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 type: string clientSecret: - description: '`clientSecret` specifies a key of a Secret + description: clientSecret specifies a key of a Secret containing the client secret of the Azure Active Directory - application that is being used to authenticate.' + application that is being used to authenticate. properties: key: description: The key of the secret to select from. Must @@ -4871,9 +4885,9 @@ spec: type: object x-kubernetes-map-type: atomic tenantId: - description: '`tenantId` is the tenant ID of the Azure + description: tenantId is the tenant ID of the Azure Active Directory application that is being used to - authenticate.' + authenticate. minLength: 1 pattern: ^[0-9a-zA-Z-.]+$ type: string @@ -4884,29 +4898,29 @@ spec: type: object sdk: description: |- - SDK defines the Azure SDK config that is being used to authenticate. + sdk defines the Azure SDK config that is being used to authenticate. See https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication Cannot be set at the same time as `oauth` or `managedIdentity`. It requires Prometheus >= v2.52.0 or Thanos >= v0.36.0. properties: tenantId: - description: '`tenantId` is the tenant ID of the azure + description: tenantId defines the tenant ID of the azure active directory application that is being used to - authenticate.' + authenticate. pattern: ^[0-9a-zA-Z-.]+$ type: string type: object type: object basicAuth: description: |- - BasicAuth configuration for the URL. + basicAuth configuration for the URL. Cannot be set at the same time as `sigv4`, `authorization`, `oauth2`, or `azureAd`. properties: password: description: |- - `password` specifies a key of a Secret containing the password for + password defines a key of a Secret containing the password for authentication. properties: key: @@ -4932,7 +4946,7 @@ spec: x-kubernetes-map-type: atomic username: description: |- - `username` specifies a key of a Secret containing the username for + username defines a key of a Secret containing the username for authentication. properties: key: @@ -4959,23 +4973,22 @@ spec: type: object bearerToken: description: |- + bearerToken is deprecated: this will be removed in a future release. *Warning: this field shouldn't be used because the token value appears in clear-text. Prefer using `authorization`.* - - Deprecated: this will be removed in a future release. type: string bearerTokenFile: description: |- - File from which to read bearer token for the URL. + bearerTokenFile defines the file from which to read bearer token for the URL. Deprecated: this will be removed in a future release. Prefer using `authorization`. type: string enableHTTP2: - description: Whether to enable HTTP2. + description: enableHTTP2 defines whether to enable HTTP2. type: boolean followRedirects: description: |- - Configure whether HTTP requests follow HTTP 3xx redirects. + followRedirects defines whether HTTP requests follow HTTP 3xx redirects. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. type: boolean @@ -4983,14 +4996,14 @@ spec: additionalProperties: type: string description: |- - Custom HTTP headers to be sent along with each remote write request. + headers defines the custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. It requires Prometheus >= v2.25.0 or Thanos >= v0.24.0. type: object messageVersion: description: |- - The Remote Write message's version to use when writing to the endpoint. + messageVersion defines the Remote Write message's version to use when writing to the endpoint. `Version1.0` corresponds to the `prometheus.WriteRequest` protobuf message introduced in Remote Write 1.0. `Version2.0` corresponds to the `io.prometheus.write.v2.Request` protobuf message introduced in Remote Write 2.0. @@ -5007,37 +5020,37 @@ spec: - V2.0 type: string metadataConfig: - description: MetadataConfig configures the sending of series - metadata to the remote storage. + description: metadataConfig defines how to send a series metadata + to the remote storage. properties: maxSamplesPerSend: description: |- - MaxSamplesPerSend is the maximum number of metadata samples per send. + maxSamplesPerSend defines the maximum number of metadata samples per send. It requires Prometheus >= v2.29.0. format: int32 minimum: -1 type: integer send: - description: Defines whether metric metadata is sent to - the remote storage or not. + description: send defines whether metric metadata is sent + to the remote storage or not. type: boolean sendInterval: - description: Defines how frequently metric metadata is sent - to the remote storage. + description: sendInterval defines how frequently metric + metadata is sent to the remote storage. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object name: description: |- - The name of the remote write queue, it must be unique if specified. The + name of the remote write queue, it must be unique if specified. The name is used in metrics and logging in order to differentiate queues. It requires Prometheus >= v2.15.0 or Thanos >= 0.24.0. type: string noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5045,7 +5058,7 @@ spec: type: string oauth2: description: |- - OAuth2 configuration for the URL. + oauth2 configuration for the URL. It requires Prometheus >= v2.27.0 or Thanos >= v0.24.0. @@ -5053,12 +5066,12 @@ spec: properties: clientId: description: |- - `clientId` specifies a key of a Secret or ConfigMap containing the + clientId defines a key of a Secret or ConfigMap containing the OAuth2 client's ID. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5081,7 +5094,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5107,7 +5121,7 @@ spec: type: object clientSecret: description: |- - `clientSecret` specifies a key of a Secret containing the OAuth2 + clientSecret defines a key of a Secret containing the OAuth2 client's secret. properties: key: @@ -5135,12 +5149,12 @@ spec: additionalProperties: type: string description: |- - `endpointParams` configures the HTTP parameters to append to the token + endpointParams configures the HTTP parameters to append to the token URL. type: object noProxy: description: |- - `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + noProxy defines a comma-separated string that can contain IPs, CIDR notation, domain names that should be excluded from proxying. IP and domain names can contain port numbers. @@ -5174,7 +5188,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5182,33 +5196,32 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to - use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string scopes: - description: '`scopes` defines the OAuth2 scopes used for - the token request.' + description: scopes defines the OAuth2 scopes used for the + token request. items: type: string type: array tlsConfig: description: |- - TLS configuration to use when connecting to the OAuth2 server. + tlsConfig defines the TLS configuration to use when connecting to the OAuth2 server. It requires Prometheus >= v2.43.0. properties: ca: - description: Certificate authority used when verifying - server certificates. + description: ca defines the Certificate authority used + when verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5231,8 +5244,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5257,12 +5270,12 @@ spec: x-kubernetes-map-type: atomic type: object cert: - description: Client certificate to present when doing - client-authentication. + description: cert defines the Client certificate to + present when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for - the targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5285,8 +5298,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the - targets. + description: secret defines the Secret containing + data to use for the targets. properties: key: description: The key of the secret to select @@ -5311,11 +5324,12 @@ spec: x-kubernetes-map-type: atomic type: object insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable + target certificate validation. type: boolean keySecret: - description: Secret containing the client key file for - the targets. + description: keySecret defines the Secret containing + the client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5340,7 +5354,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5351,7 +5365,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5361,12 +5375,13 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname + for the targets. type: string type: object tokenUrl: - description: '`tokenURL` configures the URL to fetch the - token from.' + description: tokenUrl defines the URL to fetch the token + from. minLength: 1 type: string required: @@ -5402,7 +5417,7 @@ spec: x-kubernetes-map-type: atomic type: array description: |- - ProxyConnectHeader optionally specifies headers to send to + proxyConnectHeader optionally specifies headers to send to proxies during CONNECT requests. It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. @@ -5410,89 +5425,88 @@ spec: x-kubernetes-map-type: atomic proxyFromEnvironment: description: |- - Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + proxyFromEnvironment defines whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). It requires Prometheus >= v2.43.0, Alertmanager >= v0.25.0 or Thanos >= v0.32.0. type: boolean proxyUrl: - description: '`proxyURL` defines the HTTP proxy server to use.' + description: proxyUrl defines the HTTP proxy server to use. pattern: ^(http|https|socks5)://.+$ type: string queueConfig: - description: QueueConfig allows tuning of the remote write queue + description: queueConfig allows tuning of the remote write queue parameters. properties: batchSendDeadline: - description: BatchSendDeadline is the maximum time a sample - will wait in buffer. + description: batchSendDeadline defines the maximum time + a sample will wait in buffer. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string capacity: description: |- - Capacity is the number of samples to buffer per shard before we start + capacity defines the number of samples to buffer per shard before we start dropping them. type: integer maxBackoff: - description: MaxBackoff is the maximum retry delay. + description: maxBackoff defines the maximum retry delay. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string maxRetries: - description: MaxRetries is the maximum number of times to - retry a batch on recoverable errors. + description: maxRetries defines the maximum number of times + to retry a batch on recoverable errors. type: integer maxSamplesPerSend: - description: MaxSamplesPerSend is the maximum number of - samples per send. + description: maxSamplesPerSend defines the maximum number + of samples per send. type: integer maxShards: - description: MaxShards is the maximum number of shards, + description: maxShards defines the maximum number of shards, i.e. amount of concurrency. type: integer minBackoff: - description: MinBackoff is the initial retry delay. Gets - doubled for every retry. + description: minBackoff defines the initial retry delay. + Gets doubled for every retry. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string minShards: - description: MinShards is the minimum number of shards, + description: minShards defines the minimum number of shards, i.e. amount of concurrency. type: integer retryOnRateLimit: description: |- - Retry upon receiving a 429 status code from the remote-write storage. + retryOnRateLimit defines the retry upon receiving a 429 status code from the remote-write storage. This is an *experimental feature*, it may change in any upcoming release in a breaking way. type: boolean sampleAgeLimit: description: |- - SampleAgeLimit drops samples older than the limit. + sampleAgeLimit drops samples older than the limit. It requires Prometheus >= v2.50.0 or Thanos >= v0.32.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object remoteTimeout: - description: Timeout for requests to the remote write endpoint. + description: remoteTimeout defines the timeout for requests + to the remote write endpoint. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string roundRobinDNS: - description: |- - When enabled: - - The remote-write mechanism will resolve the hostname via DNS. - - It will randomly select one of the resolved IP addresses and connect to it. - - When disabled (default behavior): - - The Go standard library will handle hostname resolution. - - It will attempt connections to each resolved IP address sequentially. - - Note: The connection timeout applies to the entire resolution and connection process. - If disabled, the timeout is distributed across all connection attempts. - - It requires Prometheus >= v3.1.0 or Thanos >= v0.38.0. + description: "roundRobinDNS controls the DNS resolution behavior + for remote-write connections.\nWhen enabled:\n - The remote-write + mechanism will resolve the hostname via DNS.\n - It will + randomly select one of the resolved IP addresses and connect + to it.\n\nWhen disabled (default behavior):\n - The Go standard + library will handle hostname resolution.\n - It will attempt + connections to each resolved IP address sequentially.\n\nNote: + The connection timeout applies to the entire resolution and + connection process.\n\n\tIf disabled, the timeout is distributed + across all connection attempts.\n\nIt requires Prometheus + >= v3.1.0 or Thanos >= v0.38.0." type: boolean sendExemplars: description: |- - Enables sending of exemplars over remote write. Note that + sendExemplars enables sending of exemplars over remote write. Note that exemplar-storage itself must be enabled using the `spec.enableFeatures` option for exemplars to be scraped in the first place. @@ -5500,14 +5514,14 @@ spec: type: boolean sendNativeHistograms: description: |- - Enables sending of native histograms, also known as sparse histograms + sendNativeHistograms enables sending of native histograms, also known as sparse histograms over remote write. It requires Prometheus >= v2.40.0 or Thanos >= v0.30.0. type: boolean sigv4: description: |- - Sigv4 allows to configures AWS's Signature Verification 4 for the URL. + sigv4 defines the AWS's Signature Verification 4 for the URL. It requires Prometheus >= v2.26.0 or Thanos >= v0.24.0. @@ -5515,7 +5529,7 @@ spec: properties: accessKey: description: |- - AccessKey is the AWS API key. If not specified, the environment variable + accessKey defines the AWS API key. If not specified, the environment variable `AWS_ACCESS_KEY_ID` is used. properties: key: @@ -5540,18 +5554,20 @@ spec: type: object x-kubernetes-map-type: atomic profile: - description: Profile is the named AWS profile used to authenticate. + description: profile defines the named AWS profile used + to authenticate. type: string region: - description: Region is the AWS region. If blank, the region - from the default credentials chain used. + description: region defines the AWS region. If blank, the + region from the default credentials chain used. type: string roleArn: - description: RoleArn is the named AWS profile used to authenticate. + description: roleArn defines the named AWS profile used + to authenticate. type: string secretKey: description: |- - SecretKey is the AWS API secret. If not specified, the environment + secretKey defines the AWS API secret. If not specified, the environment variable `AWS_SECRET_ACCESS_KEY` is used. properties: key: @@ -5575,17 +5591,22 @@ spec: - key type: object x-kubernetes-map-type: atomic + useFIPSSTSEndpoint: + description: |- + useFIPSSTSEndpoint defines FIPS mode for AWS STS endpoint. + It requires Prometheus >= v2.54.0. + type: boolean type: object tlsConfig: - description: TLS Config to use for the URL. + description: tlsConfig to use for the URL. properties: ca: - description: Certificate authority used when verifying server - certificates. + description: ca defines the Certificate authority used when + verifying server certificates. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5608,7 +5629,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5633,15 +5655,16 @@ spec: x-kubernetes-map-type: atomic type: object caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. + description: caFile defines the path to the CA cert in the + Prometheus container to use for the targets. type: string cert: - description: Client certificate to present when doing client-authentication. + description: cert defines the Client certificate to present + when doing client-authentication. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -5664,7 +5687,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -5689,19 +5713,20 @@ spec: x-kubernetes-map-type: atomic type: object certFile: - description: Path to the client cert file in the Prometheus - container for the targets. + description: certFile defines the path to the client cert + file in the Prometheus container for the targets. type: string insecureSkipVerify: - description: Disable target certificate validation. + description: insecureSkipVerify defines how to disable target + certificate validation. type: boolean keyFile: - description: Path to the client key file in the Prometheus - container for the targets. + description: keyFile defines the path to the client key + file in the Prometheus container for the targets. type: string keySecret: - description: Secret containing the client key file for the - targets. + description: keySecret defines the Secret containing the + client key file for the targets. properties: key: description: The key of the secret to select from. Must @@ -5726,7 +5751,7 @@ spec: x-kubernetes-map-type: atomic maxVersion: description: |- - Maximum acceptable TLS version. + maxVersion defines the maximum acceptable TLS version. It requires Prometheus >= v2.41.0 or Thanos >= v0.31.0. enum: @@ -5737,7 +5762,7 @@ spec: type: string minVersion: description: |- - Minimum acceptable TLS version. + minVersion defines the minimum acceptable TLS version. It requires Prometheus >= v2.35.0 or Thanos >= v0.28.0. enum: @@ -5747,15 +5772,18 @@ spec: - TLS13 type: string serverName: - description: Used to verify the hostname for the targets. + description: serverName is used to verify the hostname for + the targets. type: string type: object url: - description: The URL of the endpoint to send samples to. + description: url defines the URL of the endpoint to send samples + to. minLength: 1 type: string writeRelabelConfigs: - description: The list of remote write relabel configurations. + description: writeRelabelConfigs defines the list of remote + write relabel configurations. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -5766,7 +5794,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -5798,41 +5826,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -5846,17 +5874,18 @@ spec: type: object type: array replicas: - description: Number of thanos ruler instances to deploy. + description: replicas defines the number of thanos ruler instances + to deploy. format: int32 type: integer resendDelay: - description: Minimum amount of time to wait before resending an alert - to Alertmanager. + description: resendDelay defines the minimum amount of time to wait + before resending an alert to Alertmanager. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string resources: description: |- - Resources defines the resource requirements for single Pods. + resources defines the resource requirements for single Pods. If not provided, no requests/limits will be set properties: claims: @@ -5918,7 +5947,7 @@ spec: retention: default: 24h description: |- - Time duration ThanosRuler shall retain data for. Default is '24h', and + retention defines the time duration ThanosRuler shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years). @@ -5927,26 +5956,26 @@ spec: pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string routePrefix: - description: The route prefix ThanosRuler registers HTTP handlers - for. This allows thanos UI to be served on a sub-path. + description: routePrefix defines the route prefix ThanosRuler registers + HTTP handlers for. This allows thanos UI to be served on a sub-path. type: string ruleConcurrentEval: description: |- - How many rules can be evaluated concurrently. + ruleConcurrentEval defines how many rules can be evaluated concurrently. It requires Thanos >= v0.37.0. format: int32 minimum: 1 type: integer ruleGracePeriod: description: |- - Minimum duration between alert and restored "for" state. + ruleGracePeriod defines the minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. It requires Thanos >= v0.30.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string ruleNamespaceSelector: description: |- - Namespaces to be selected for Rules discovery. If unspecified, only + ruleNamespaceSelector defines the namespaces to be selected for Rules discovery. If unspecified, only the same namespace as the ThanosRuler object is in is used. properties: matchExpressions: @@ -5994,19 +6023,19 @@ spec: x-kubernetes-map-type: atomic ruleOutageTolerance: description: |- - Max time to tolerate prometheus outage for restoring "for" state of alert. + ruleOutageTolerance defines the max time to tolerate prometheus outage for restoring "for" state of alert. It requires Thanos >= v0.30.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string ruleQueryOffset: description: |- - The default rule group's query offset duration to use. + ruleQueryOffset defines the default rule group's query offset duration to use. It requires Thanos >= v0.38.0. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string ruleSelector: description: |- - PrometheusRule objects to be selected for rule evaluation. An empty + ruleSelector defines the PrometheusRule objects to be selected for rule evaluation. An empty label selector matches all objects. A null label selector matches no objects. properties: @@ -6055,7 +6084,7 @@ spec: x-kubernetes-map-type: atomic securityContext: description: |- - SecurityContext holds pod-level security attributes and common container settings. + securityContext defines the pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: appArmorProfile: @@ -6289,12 +6318,12 @@ spec: type: object serviceAccountName: description: |- - ServiceAccountName is the name of the ServiceAccount to use to run the + serviceAccountName defines the name of the ServiceAccount to use to run the Thanos Ruler Pods. type: string serviceName: description: |- - The name of the service name used by the underlying StatefulSet(s) as the governing service. + serviceName defines the name of the service name used by the underlying StatefulSet(s) as the governing service. If defined, the Service must be created before the ThanosRuler resource in the same namespace and it must define a selector that matches the pod labels. If empty, the operator will create and manage a headless service named `thanos-ruler-operated` for ThanosRuler resources. When deploying multiple ThanosRuler resources in the same namespace, it is recommended to specify a different value for each. @@ -6302,15 +6331,16 @@ spec: minLength: 1 type: string storage: - description: Storage spec to specify how storage shall be used. + description: storage defines the specification of how storage shall + be used. properties: disableMountSubPath: - description: 'Deprecated: subPath usage will be removed in a future - release.' + description: 'disableMountSubPath deprecated: subPath usage will + be removed in a future release.' type: boolean emptyDir: description: |- - EmptyDirVolumeSource to be used by the StatefulSet. + emptyDir to be used by the StatefulSet. If specified, it takes precedence over `ephemeral` and `volumeClaimTemplate`. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir properties: @@ -6337,7 +6367,7 @@ spec: type: object ephemeral: description: |- - EphemeralVolumeSource to be used by the StatefulSet. + ephemeral to be used by the StatefulSet. This is a beta field in k8s 1.21 and GA in 1.15. For lower versions, starting with k8s 1.19, it requires enabling the GenericEphemeralVolume feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes @@ -6582,7 +6612,7 @@ spec: type: object volumeClaimTemplate: description: |- - Defines the PVC spec to be used by the Prometheus StatefulSets. + volumeClaimTemplate defines the PVC spec to be used by the Prometheus StatefulSets. The easiest way to use a volume that cannot be automatically provisioned is to use a label selector alongside manually created PersistentVolumes. properties: @@ -6602,14 +6632,14 @@ spec: More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: - description: EmbeddedMetadata contains metadata relevant to - an EmbeddedResource. + description: metadata defines EmbeddedMetadata contains metadata + relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: |- - Annotations is an unstructured key value map stored with a resource that may be + annotations defines an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -6618,14 +6648,14 @@ spec: additionalProperties: type: string description: |- - Map of string keys and values that can be used to organize and categorize + labels define the map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ type: object name: description: |- - Name must be unique within a namespace. Is required when creating resources, although + name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. @@ -6635,7 +6665,7 @@ spec: type: object spec: description: |- - Defines the desired characteristics of a volume requested by a pod author. + spec defines the specification of the characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: accessModes: @@ -6831,7 +6861,7 @@ spec: type: string type: object status: - description: 'Deprecated: this field is never set.' + description: 'status is deprecated: this field is never set.' properties: accessModes: description: |- @@ -7016,7 +7046,7 @@ spec: type: object terminationGracePeriodSeconds: description: |- - Optional duration in seconds the pod needs to terminate gracefully. + terminationGracePeriodSeconds defines the optional duration in seconds the pod needs to terminate gracefully. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down) which may lead to data corruption. @@ -7025,7 +7055,7 @@ spec: minimum: 0 type: integer tolerations: - description: If specified, the pod's tolerations. + description: tolerations defines when specified, the pod's tolerations. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -7064,7 +7094,8 @@ spec: type: object type: array topologySpreadConstraints: - description: If specified, the pod's topology spread constraints. + description: topologySpreadConstraints defines the pod's topology + spread constraints. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. @@ -7239,7 +7270,7 @@ spec: type: array tracingConfig: description: |- - Configures tracing. + tracingConfig defines the tracing configuration. The configuration format is defined at https://thanos.io/tip/thanos/tracing.md/#configuration @@ -7272,7 +7303,7 @@ spec: x-kubernetes-map-type: atomic tracingConfigFile: description: |- - Configures the path of the tracing configuration file. + tracingConfigFile defines the path of the tracing configuration file. The configuration format is defined at https://thanos.io/tip/thanos/tracing.md/#configuration @@ -7284,11 +7315,11 @@ spec: This field takes precedence over `tracingConfig`. type: string version: - description: Version of Thanos to be deployed. + description: version of Thanos to be deployed. type: string volumeMounts: description: |- - VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. + volumeMounts defines how the configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the ruler container, that are generated as a result of StorageSpec objects. items: @@ -7355,7 +7386,7 @@ spec: type: array volumes: description: |- - Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will + volumes defines how configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: description: Volume represents a named volume in a pod that may @@ -9240,22 +9271,24 @@ spec: type: object type: array web: - description: Defines the configuration of the ThanosRuler web server. + description: web defines the configuration of the ThanosRuler web + server. properties: httpConfig: - description: Defines HTTP parameters for web server. + description: httpConfig defines HTTP parameters for web server. properties: headers: - description: List of headers that can be added to HTTP responses. + description: headers defines a list of headers that can be + added to HTTP responses. properties: contentSecurityPolicy: description: |- - Set the Content-Security-Policy header to HTTP responses. + contentSecurityPolicy defines the Content-Security-Policy header to HTTP responses. Unset if blank. type: string strictTransportSecurity: description: |- - Set the Strict-Transport-Security header to HTTP responses. + strictTransportSecurity defines the Strict-Transport-Security header to HTTP responses. Unset if blank. Please make sure that you use this with care as this header might force browsers to load Prometheus and the other applications hosted on the same @@ -9264,7 +9297,7 @@ spec: type: string xContentTypeOptions: description: |- - Set the X-Content-Type-Options header to HTTP responses. + xContentTypeOptions defines the X-Content-Type-Options header to HTTP responses. Unset if blank. Accepted value is nosniff. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options enum: @@ -9273,7 +9306,7 @@ spec: type: string xFrameOptions: description: |- - Set the X-Frame-Options header to HTTP responses. + xFrameOptions defines the X-Frame-Options header to HTTP responses. Unset if blank. Accepted values are deny and sameorigin. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options enum: @@ -9283,32 +9316,32 @@ spec: type: string xXSSProtection: description: |- - Set the X-XSS-Protection header to all responses. + xXSSProtection defines the X-XSS-Protection header to all responses. Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection type: string type: object http2: description: |- - Enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. + http2 enable HTTP/2 support. Note that HTTP/2 is only supported with TLS. When TLSConfig is not configured, HTTP/2 will be disabled. Whenever the value of the field changes, a rolling update will be triggered. type: boolean type: object tlsConfig: - description: Defines the TLS parameters for HTTPS. + description: tlsConfig defines the TLS parameters for HTTPS. properties: cert: description: |- - Secret or ConfigMap containing the TLS certificate for the web server. + cert defines the Secret or ConfigMap containing the TLS certificate for the web server. Either `keySecret` or `keyFile` must be defined. It is mutually exclusive with `certFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9331,7 +9364,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9357,7 +9391,7 @@ spec: type: object certFile: description: |- - Path to the TLS certificate file in the container for the web server. + certFile defines the path to the TLS certificate file in the container for the web server. Either `keySecret` or `keyFile` must be defined. @@ -9365,7 +9399,7 @@ spec: type: string cipherSuites: description: |- - List of supported cipher suites for TLS versions up to TLS 1.2. + cipherSuites defines the list of supported cipher suites for TLS versions up to TLS 1.2. If not defined, the Go default cipher suites are used. Available cipher suites are documented in the Go documentation: @@ -9375,14 +9409,14 @@ spec: type: array client_ca: description: |- - Secret or ConfigMap containing the CA certificate for client certificate + client_ca defines the Secret or ConfigMap containing the CA certificate for client certificate authentication to the server. It is mutually exclusive with `clientCAFile`. properties: configMap: - description: ConfigMap containing data to use for the - targets. + description: configMap defines the ConfigMap containing + data to use for the targets. properties: key: description: The key to select. @@ -9405,7 +9439,8 @@ spec: type: object x-kubernetes-map-type: atomic secret: - description: Secret containing data to use for the targets. + description: secret defines the Secret containing data + to use for the targets. properties: key: description: The key of the secret to select from. Must @@ -9431,21 +9466,21 @@ spec: type: object clientAuthType: description: |- - The server policy for client TLS authentication. + clientAuthType defines the server policy for client TLS authentication. For more detail on clientAuth options: https://golang.org/pkg/crypto/tls/#ClientAuthType type: string clientCAFile: description: |- - Path to the CA certificate file for client certificate authentication to + clientCAFile defines the path to the CA certificate file for client certificate authentication to the server. It is mutually exclusive with `client_ca`. type: string curvePreferences: description: |- - Elliptic curves that will be used in an ECDHE handshake, in preference + curvePreferences defines elliptic curves that will be used in an ECDHE handshake, in preference order. Available curves are documented in the Go documentation: @@ -9455,7 +9490,7 @@ spec: type: array keyFile: description: |- - Path to the TLS private key file in the container for the web server. + keyFile defines the path to the TLS private key file in the container for the web server. If defined, either `cert` or `certFile` must be defined. @@ -9463,7 +9498,7 @@ spec: type: string keySecret: description: |- - Secret containing the TLS private key for the web server. + keySecret defines the secret containing the TLS private key for the web server. Either `cert` or `certFile` must be defined. @@ -9491,14 +9526,16 @@ spec: type: object x-kubernetes-map-type: atomic maxVersion: - description: Maximum TLS version that is acceptable. + description: maxVersion defines the Maximum TLS version that + is acceptable. type: string minVersion: - description: Minimum TLS version that is acceptable. + description: minVersion defines the minimum TLS version that + is acceptable. type: string preferServerCipherSuites: description: |- - Controls whether the server selects the client's most preferred cipher + preferServerCipherSuites defines whether the server selects the client's most preferred cipher suite, or the server's most preferred cipher suite. If true then the server's preference, as expressed in @@ -9509,18 +9546,19 @@ spec: type: object status: description: |- - Most recent observed status of the ThanosRuler cluster. Read-only. + status defines the most recent observed status of the ThanosRuler cluster. Read-only. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: availableReplicas: description: |- - Total number of available pods (ready for at least minReadySeconds) + availableReplicas defines the total number of available pods (ready for at least minReadySeconds) targeted by this ThanosRuler deployment. format: int32 type: integer conditions: - description: The current state of the ThanosRuler object. + description: conditions defines the current state of the ThanosRuler + object. items: description: |- Condition represents the state of the resources associated with the @@ -9532,12 +9570,12 @@ spec: format: date-time type: string message: - description: Human-readable message indicating details for the - condition's last transition. + description: message defines human-readable message indicating + details for the condition's last transition. type: string observedGeneration: description: |- - ObservedGeneration represents the .metadata.generation that the + observedGeneration defines the .metadata.generation that the condition was set based upon. For instance, if `.metadata.generation` is currently 12, but the `.status.conditions[].observedGeneration` is 9, the condition is out of date with respect to the current state of the @@ -9545,14 +9583,14 @@ spec: format: int64 type: integer reason: - description: Reason for the condition's last transition. + description: reason for the condition's last transition. type: string status: - description: Status of the condition. + description: status of the condition. minLength: 1 type: string type: - description: Type of the condition being reported. + description: type of the condition being reported. minLength: 1 type: string required: @@ -9566,32 +9604,26 @@ spec: x-kubernetes-list-type: map paused: description: |- - Represents whether any actions on the underlying managed objects are + paused defines whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: |- - Total number of non-terminated pods targeted by this ThanosRuler deployment + replicas defines the total number of non-terminated pods targeted by this ThanosRuler deployment (their labels match the selector). format: int32 type: integer unavailableReplicas: - description: Total number of unavailable pods targeted by this ThanosRuler - deployment. + description: unavailableReplicas defines the total number of unavailable + pods targeted by this ThanosRuler deployment. format: int32 type: integer updatedReplicas: description: |- - Total number of non-terminated pods targeted by this ThanosRuler deployment + updatedReplicas defines the total number of non-terminated pods targeted by this ThanosRuler deployment that have the desired version spec. format: int32 type: integer - required: - - availableReplicas - - paused - - replicas - - unavailableReplicas - - updatedReplicas type: object required: - spec From 81a1adf120158268e31208445059f69523984e95 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:56:34 +0200 Subject: [PATCH 097/176] Correct the `gardener-admission-controller` VPA name (#13430) Co-authored-by: Rada Dimitrova <117522044+RadaBDimitrova@users.noreply.github.com> --- .../gardener/admissioncontroller/admission_controller_test.go | 2 +- pkg/component/gardener/admissioncontroller/vpa.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/component/gardener/admissioncontroller/admission_controller_test.go b/pkg/component/gardener/admissioncontroller/admission_controller_test.go index c593d8f9392..93cc92a655c 100644 --- a/pkg/component/gardener/admissioncontroller/admission_controller_test.go +++ b/pkg/component/gardener/admissioncontroller/admission_controller_test.go @@ -817,7 +817,7 @@ func vpa(namespace string) *vpaautoscalingv1.VerticalPodAutoscaler { return &vpaautoscalingv1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: "gardener-admission-controller", + Name: "gardener-admission-controller-vpa", Namespace: namespace, Labels: map[string]string{ "app": "gardener", diff --git a/pkg/component/gardener/admissioncontroller/vpa.go b/pkg/component/gardener/admissioncontroller/vpa.go index fcd679ec36a..5534068c532 100644 --- a/pkg/component/gardener/admissioncontroller/vpa.go +++ b/pkg/component/gardener/admissioncontroller/vpa.go @@ -17,7 +17,7 @@ func (a *gardenerAdmissionController) vpa() *vpaautoscalingv1.VerticalPodAutosca updateMode := vpaautoscalingv1.UpdateModeRecreate return &vpaautoscalingv1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: DeploymentName, + Name: DeploymentName + "-vpa", Namespace: a.namespace, Labels: GetLabels(), }, From 687d3204f797d70e3658a67bc4663d6ae7e62462 Mon Sep 17 00:00:00 2001 From: Luca Bernstein Date: Wed, 12 Nov 2025 16:26:03 +0100 Subject: [PATCH 098/176] Prepare next Dev Cycle `v1.133.0-dev` (#13409) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 02b5dd754e8..ed537fe725d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.132.0-dev +v1.133.0-dev From 3498d6f068b89a10673ca6cf5abea4e3097294ad Mon Sep 17 00:00:00 2001 From: Stoyan Vitanov Date: Wed, 12 Nov 2025 20:11:02 +0200 Subject: [PATCH 099/176] Add `resource-manager` webhook mutating `vpa` resources with `InPlaceOrRecreate` update mode (#12940) * feat(resourcemanager/webhook): Add a webhook to modify VerticalPodAutoscaler update mode This commit adds a new webhook to the resource manager that aims to modify VerticalPodAutoscaler resources and set `updateMode` values to the newly introduced `InPlaceOrRecreate`. * feat(resourcemanager/apis): Add `InPlaceOrRecreate` enabling webhook configuration This commit adds a dedicated configuration for the newly introduced webhook. * feat(resourcemanager): Register `InPlaceOrRecreate` enabling webhook * feat(apis): Add `skip` label for `InPlaceOrRecreate` enabling webhook This commit adds a dedicated skip label, used on VerticalPodAutoscaler resources that should NOT get modified by the in-place enabling webhook ( i.e. continue to use their current update mode setting ). * feat(gardener/resourcemanager): Add `InPlaceOrRecreate` webhook to GRM webhooks configuration This commit adds the InPlaceOrRecreate enabling webhook to the GRM webhooks configuration via: * new utility function to create admissionregistrationv1.MutatingWebhook resource referencing the webhook implementation * a new flag within the GRM values structure based on which the webhook is added/omitted from the deployment * feat(features): Add a new feature gate for in-place Pod resource-updates-enabling webhook This commit adds a new feature gate used for enabling/disabling a dedicated GRM webhook that updates VPA resources updateModes to InPlaceOrRecreate. * feat(gardenlet/features): Register `in-place` Pod resource-updates-enabling webhook gate This commit registers the feature gate enabling the in-place Pod resource updates webhook. * feat(gardenlet/botanist): Enable `updateMode` modifying GRM webhook based on feature gate This commit instruments the `botanist` utilities to enable the in-place Pod resource-updates-enabling webhook based on the dedicated `VpaInPlaceOrRecreateUpdateMode` feature gate. * feat(shared/resourcemanager): Add default value for `InPlaceOrRecreate` GRM webhook flag This commit adds a default value to the `InPlaceOrRecreate` webhook enabling flag for the shared GRM utilities. * feat(gardenlet/seed): Enable `updateMode` modifying GRM webhook based on feature gate This commit enables the `updateMode` modifying webhook to the Seed resource manager deployment based on a dedicated feature gate. * feat(operator/features): Register `InPlaceOrRecreate` webhook enabling feature gate This commit registers the InPlaceOrRecreate webhook enabling feature gate to the gardener-operator set of supported gates. * feat(operator/components): Enable `updateMode` modifying GRM webhook based on `Garden` feature gate This commit enables the updateMode modifying webhook within the `Garden` resource reconciliation flow. The webhook activation is based on a newly introduced feature gate added to the `Garden` resource. * feat(test/integration/resourcemanager): Add test for in-place enabling webhook This commit adds integration tests to the resource manager component that aim to validate the InPlaceOrRecreate enabling webhook operation. The main goal is to validate that: - the usage of `skip` labels prevents mutation of existing vpa resources - update modes Auto and Recreate get successfully mutated to InPlaceOrRecreate when no `skip` label is used * feat(docs): Add feature gate and usage guide for enabling in-place Pod resource updates This commit udpates the feature gates document with an entry for the gardenlet and gardener operator feature gate, enabling the deployment of the webhook mutating VPA resources. In addition, a new in-place updates enablement guide is added to document the process of enabling the new feature gate and configuring the `Shoot` and `Seed` cluster's Vertical Pod Autoscaler setups. * feat(apis/config/v1alpha1): update generated `resourcemanager` webhook config This commit updates the `v1alpha1` generated sources with the newly introduced resource manager flag. * feat(skaffold): update `skaffold` dependencies with the new `grm` webhook * fix(botanist/resource_manager_test): fix test title typo * chore(autoscaling/vpa): make `autoscaling` VPAs use update mode `Recreate` * chore(resourcemanager/webhook): update `PodUpdatePolicy{}` at a single place This commit makes the setting of PodUpdatePolicy{} happen in a single place to ensure that each updateMode edit are being logged. * feat(resourcemanager/webhook/test): validate `Initial` and `Off` modes are not modified * Update pkg/resourcemanager/webhook/vpainplaceorrecreateupdatemode/vpainplaceorrecreateupdatemode_suite_test.go Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> * Update pkg/resourcemanager/webhook/vpainplaceorrecreateupdatemode/vpainplaceorrecreateupdatemode_suite_test.go Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> * Update pkg/resourcemanager/apis/config/v1alpha1/types.go Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> * chore(autoscaling/vpa/test): instrument tests with update mode `Recreate` * chore(test/integration): replace `webhook` unit tests with `integration` tests This commit enriches the GRM integration tests with additional cases that cover the scnarious of the webhook unit tests. Also: - remove the webhook unit tests in favour of additional integration tests * chore(component/resourcemanager): use upper case `VPA` prefix for `webhook` skip label * feat(charts): include `in-place` Pod updates `webhook` to `resource manager` charts This commit updates the Gardener Resource Manager charts with the additional in-place enabling webhook configuration. * chore(pkg/features): include`ialidzhikov` as a 2nd feature gate owner Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> * chore(botanist/resource_manager): validate if `InPlaceOrRecreate` VPA feature gates is explicitly disabled This commit checks if the Shoot's Vertical Pod Autoscaler configuration has the InPlaceOrRecreate VPA feature gate explicitly disabled. * chore(feature): rename feature gate to `VPAInPlaceUpdates` * Update docs/usage/autoscaling/enabling-in-place-resource-updates.md Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> * Update docs/usage/autoscaling/enabling-in-place-resource-updates.md Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> * chore(operation/botanist): remove redundant test focus * chore(pkg/apis): remove redundant `prometheus` folder cleanup annotation * chore(gardenlet/resource_manager): check if `VPA` feature gate is defined and enabled This commit updates the logic that check if the VPA feature gate for in-place updates is enabled. As of VPA 1.5.0, the in-place updates feature gate is enabled by default, so we shall preserve the same behaviour and disable the webhook only if the VPA feature gate is explicitly disabled. * chore(docs): remove redundant line Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> * chore(features): target Gardener `v1.132.0` for the webhook feature gate release * chore(pkg/resourcemanager): update webhook name to `vpainplaceupdates` This commit updates the name of the GRM webhook to `vpainplaceupdates` and updates all related packages, tests, charts and manifests to match the new name. * chore(resourcemanager/webhook): use `kubernetesutils` to setup structured logging * Update docs/usage/autoscaling/enabling-in-place-resource-updates.md Co-authored-by: Nikolay Boshnakov * Update docs/usage/autoscaling/enabling-in-place-resource-updates.md Co-authored-by: Nikolay Boshnakov * Update docs/usage/autoscaling/enabling-in-place-resource-updates.md Co-authored-by: Nikolay Boshnakov * Update docs/usage/autoscaling/enabling-in-place-resource-updates.md Co-authored-by: Nikolay Boshnakov * Update docs/usage/autoscaling/enabling-in-place-resource-updates.md Co-authored-by: Nikolay Boshnakov * Update docs/concepts/resource-manager.md Co-authored-by: Nikolay Boshnakov * Update docs/concepts/resource-manager.md Co-authored-by: Nikolay Boshnakov * feat(charts): update GRM `mutatingwebhookconfiguration` with webhook definition * chore(gardener-local): do not enable webhook feature gate in local `gardenlet` setup * feat(docs): move `in-place` enabling doc to `docs/operations` * Update pkg/features/features.go Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> * feat(docs): update `in-place` enabling link in `docs/README.md` --------- Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Co-authored-by: Nikolay Boshnakov --- .../mutatingwebhookconfiguration.yaml | 35 ++++ .../charts/runtime/templates/configmap.yaml | 2 + charts/gardener/resource-manager/values.yaml | 2 + docs/README.md | 1 + docs/concepts/resource-manager.md | 26 +++ docs/deployment/feature_gates.md | 2 + .../enabling-in-place-resource-updates.md | 69 ++++++++ pkg/apis/resources/v1alpha1/types.go | 4 + .../autoscaling/vpa/admissioncontroller.go | 2 +- pkg/component/autoscaling/vpa/recommender.go | 2 +- pkg/component/autoscaling/vpa/updater.go | 2 +- pkg/component/autoscaling/vpa/vpa_test.go | 8 +- .../resourcemanager/resource_manager.go | 49 +++++ .../resourcemanager/resource_manager_test.go | 40 ++++- pkg/component/shared/resourcemanager.go | 1 + pkg/component/shared/resourcemanager_test.go | 1 + pkg/features/features.go | 7 + .../controller/seed/seed/components.go | 3 + pkg/gardenlet/features/features.go | 1 + .../operation/botanist/resource_manager.go | 20 +++ .../botanist/resource_manager_test.go | 49 +++++ .../controller/garden/garden/components.go | 1 + pkg/operator/features/features.go | 1 + .../apis/config/v1alpha1/types.go | 8 + .../config/v1alpha1/zz_generated.deepcopy.go | 17 ++ pkg/resourcemanager/webhook/add.go | 9 + .../webhook/vpainplaceupdates/add.go | 28 +++ .../webhook/vpainplaceupdates/handler.go | 50 ++++++ skaffold-gardenadm.yaml | 3 + skaffold-operator.yaml | 2 + skaffold-seed.yaml | 2 + skaffold.yaml | 2 + .../vpainplaceupdates_suite_test.go | 167 ++++++++++++++++++ .../vpainplaceupdates_test.go | 111 ++++++++++++ 34 files changed, 719 insertions(+), 8 deletions(-) create mode 100644 docs/operations/enabling-in-place-resource-updates.md create mode 100644 pkg/resourcemanager/webhook/vpainplaceupdates/add.go create mode 100644 pkg/resourcemanager/webhook/vpainplaceupdates/handler.go create mode 100644 test/integration/resourcemanager/vpainplaceupdates/vpainplaceupdates_suite_test.go create mode 100644 test/integration/resourcemanager/vpainplaceupdates/vpainplaceupdates_test.go diff --git a/charts/gardener/resource-manager/charts/application/templates/mutatingwebhookconfiguration.yaml b/charts/gardener/resource-manager/charts/application/templates/mutatingwebhookconfiguration.yaml index e03178aa7e2..78261e66fe2 100644 --- a/charts/gardener/resource-manager/charts/application/templates/mutatingwebhookconfiguration.yaml +++ b/charts/gardener/resource-manager/charts/application/templates/mutatingwebhookconfiguration.yaml @@ -282,3 +282,38 @@ webhooks: sideEffects: None timeoutSeconds: 10 {{- end }} +{{- if .Values.global.config.webhooks.vpaInPlaceUpdates.enabled }} +- admissionReviewVersions: + - v1beta1 + - v1 + clientConfig: + {{- if .Values.global.config.server.webhooks.ca }} + caBundle: {{ b64enc .Values.global.config.server.webhooks.ca }} + {{- end }} + service: + name: gardener-resource-manager + namespace: {{ .Release.Namespace }} + path: /webhooks/vpa-in-place-updates + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: vpa-in-place-updates.resources.gardener.cloud + namespaceSelector: {} + objectSelector: + matchExpressions: + - key: vpa-in-place-updates.resources.gardener.cloud/skip + operator: DoesNotExist + reinvocationPolicy: Never + rules: + - apiGroups: + - autoscaling.k8s.io + apiVersions: + - v1 + resources: + - verticalpodautoscalers + operations: + - CREATE + - UPDATE + sideEffects: None + timeoutSeconds: 10 +{{- end }} diff --git a/charts/gardener/resource-manager/charts/runtime/templates/configmap.yaml b/charts/gardener/resource-manager/charts/runtime/templates/configmap.yaml index eb89f7d9e73..df651d99dd9 100644 --- a/charts/gardener/resource-manager/charts/runtime/templates/configmap.yaml +++ b/charts/gardener/resource-manager/charts/runtime/templates/configmap.yaml @@ -190,3 +190,5 @@ data: machineNamespace: {{ .Values.global.config.webhooks.nodeAgentAuthorizer.machineNamespace }} {{- end }} authorizeWithSelectors: {{ .Values.global.config.webhooks.nodeAgentAuthorizer.authorizeWithSelectors }} + vpaInPlaceUpdates: + enabled: {{ .Values.global.config.webhooks.vpaInPlaceUpdates.enabled }} diff --git a/charts/gardener/resource-manager/values.yaml b/charts/gardener/resource-manager/values.yaml index 999ff8b77fc..3604d0da91c 100644 --- a/charts/gardener/resource-manager/values.yaml +++ b/charts/gardener/resource-manager/values.yaml @@ -145,3 +145,5 @@ global: enabled: false # machineNamespace: shoot--foo--bar # authorizeWithSelectors: false + vpaInPlaceUpdates: + enabled: false diff --git a/docs/README.md b/docs/README.md index cf916537e95..cd46d8763f8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -255,6 +255,7 @@ * [Gardener configuration and usage](operations/configuration.md) * [Control Plane Migration](operations/control_plane_migration.md) +* [Enabling In-place Resource Updates](operations/enabling-in-place-resource-updates.md) * [Immutable Backup Buckets](operations/immutable-backup-buckets.md) * [Istio](operations/istio.md) * [Kube API server load balancing](operations/kube_apiserver_loadbalancing.md) diff --git a/docs/concepts/resource-manager.md b/docs/concepts/resource-manager.md index e2d8cdbd4ae..b322ec2b93b 100644 --- a/docs/concepts/resource-manager.md +++ b/docs/concepts/resource-manager.md @@ -1130,6 +1130,32 @@ spec: For mutating pods, the webhook needs to know the namespace of the istio ingress gateway responsible for the kube-apiserver and its host names. These values are stored in `istio-internal-load-balancing` configmap in the same namespace as the pod being mutated. +#### Vertical Pod Autoscaler In-Place Or Recreate Update Mode + +When enabled, this webhook provides a mechanism of mutating `VerticalPodAutoscaler` [resources](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/deploy/vpa-crd.yaml) + +```yaml +.spec.updatePolicy.updateMode +``` + +configured with `Auto` or `Recreate` to switch to update mode `InPlaceOrRecreate`, enabling _in-place_ `Pod` [resources updates](https://kubernetes.io/blog/2025/05/16/kubernetes-v1-33-in-place-pod-resize-beta/). Preserving the `resource-manager` scope, the following constraints apply: +- _mutates_ all `vpa` resources __inside__ `kube-system` and `kubernetes-dashboards` namespaces on `Shoot` clusters +- _mutates_ all `vpa` resources __outside__ `kube-system` and `kubernetes-dashboards` namespaces on `Seed` clusters + +In addition, to prevent the _mutation_ of certain `VerticalPodAutoscaler` [resources](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/deploy/vpa-crd.yaml) falling within the webhook's scope, the following _label_ + +``` +vpa-in-place-updates.resources.gardener.cloud/skip +``` + +could be appended to the resource metadata. With the _label_ specified, the webhook will filter out the resources, leaving its current `updateMode` configuration. + +Available for deployment with both [gardenlet](https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md) and [gardener-operator](https://github.com/gardener/gardener/blob/master/docs/concepts/operator.md), enabling the webhook happens by activating a dedicated _feature gate_ within the respective component manifest: + +``` +VPAInPlaceUpdates +``` + ### Validating Webhooks #### Unconfirmed Deletion Prevention For Custom Resources And Definitions diff --git a/docs/deployment/feature_gates.md b/docs/deployment/feature_gates.md index 195e391b596..da24f969470 100644 --- a/docs/deployment/feature_gates.md +++ b/docs/deployment/feature_gates.md @@ -32,6 +32,7 @@ The following tables are a summary of the feature gates that you can set on diff | DoNotCopyBackupCredentials | `true` | `Beta` | `1.123` | | | OpenTelemetryCollector | `false` | `Alpha` | `1.124` | | | UseUnifiedHTTPProxyPort | `false` | `Alpha` | `1.130` | | +| VPAInPlaceUpdates | `false` | `Alpha` | `1.132` | | ## Feature Gates for Graduated or Deprecated Features @@ -255,3 +256,4 @@ A *General Availability* (GA) feature is also referred to as a *stable* feature. | DoNotCopyBackupCredentials | `gardenlet` | Disables the copying of Shoot infrastructure credentials as backup credentials when the Shoot is used as a ManagedSeed. Operators are responsible for providing the credentials for backup explicitly. Credentials that were already copied will be labeled with `secret.backup.gardener.cloud/status=previously-managed` and would have to be cleaned up by operators. | | OpenTelemetryCollector | `gardenlet` | Routes logs through an instance of an `OpenTelemetry Collector` in the control-plane of `Shoots`. | | UseUnifiedHTTPProxyPort | `gardenlet` | Enables the gardenlet to set up the unified HTTP proxy network infrastructure. Gardenlet will also reconfigure the API server proxy and shoot VPN client to connect to the unified port using the new X-Gardener-Destination header. | +| VPAInPlaceUpdates | `gardenlet`, `gardener-operator` | Enables the usage of in-place Pod resource updates in `Shoot`, `Seed` and `Garden` cluster's Vertical Pod Autoscaler deployments. | diff --git a/docs/operations/enabling-in-place-resource-updates.md b/docs/operations/enabling-in-place-resource-updates.md new file mode 100644 index 00000000000..8680600b668 --- /dev/null +++ b/docs/operations/enabling-in-place-resource-updates.md @@ -0,0 +1,69 @@ +--- +title: Enabling In-Place Pod Resource Updates +description: Enablement of in-place Pod resource updates within Vertical Pod Autoscaler deployments +categories: + - Operators +--- + +# Enabling In-Place Updates of Pod Resources + +This is a short guide covering the adoption mechanism of `in-place` Pod resource updates in Gardener [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler) deployments. + +## Compatibility + +Refer to the [in-place resource updates](./in-place-resource-updates.md) guide for details on Kubernetes clusters compatibility, [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler) feature gate definition and availability. + +## Configuration + +Gardener provides a dedicated [resource manager](../../concepts/resource-manager.md) [webhook](../../concepts/resource-manager.md#webhooks) capable of _mutating_ VerticalPodAutoscaler resources, configured with [update mode](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/docs/quickstart.md#contents) `Auto` or `Recreate`, with the `in-place` updates enabling `InPlaceOrRecreate`. +Available for deployment with both [gardenlet](../../concepts/gardenlet.md) and [gardener operator](../../concepts/operator.md), the _mutating_ webhook can be activated with the following feature gate, listed within the respective component manifest. Refer to the Gardener [feature gates](../../deployment/feature_gates.md) page for additional details: + +``` +VPAInPlaceUpdates +``` + +To keep a VerticalPodAutoscaler resource out of the _mutating_ webhook scope, add the following `skip` label, indicating that the resource should preserve its current configuration and __not__ get updated: + +``` +vpa-in-place-updates.resources.gardener.cloud/skip +``` + +### gardenlet + +To enable the _mutating_ [resource manager](../../concepts/resource-manager.md) webhook, the `VPAInPlaceUpdates` feature gate must be set to `true`: + +```yaml +featureGates: + VPAInPlaceUpdates: true +``` + +Refer to the `gardenlet` component configuration [manifest](../../../example/20-componentconfig-gardenlet.yaml) for an overview. + +#### Shoot + +> When deployed in a `Shoot` cluster, the _mutating_ webhook targets `vertical pod autoscaler` resources __inside__ the `kube-system` and `kubernetes-dashboard` namespaces. + +To make use of the _mutating_ resource manager webhook, the `Shoot`'s [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler) deployment must have the `InPlaceOrRecreate` feature gate enabled. Follow the [in-place resource updates](./in-place-resource-updates.md#shoot) guide for more details about the Vertical Pod Autoscaler components setup. + +#### Seed + +> When deployed in a `Seed` cluster, the _mutating_ webhook targets `vertical pod autoscaler` resources __outside__ the `kube-system` and `kubernetes-dashboard` namespaces. + +To make use of the _mutating_ resource manager webhook, the `Seed`'s [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler) deployment must have the `InPlaceOrRecreate` feature gate enabled. Follow the [in-place resource updates](./in-place-resource-updates.md#seed) guide for more details about the Vertical Pod Autoscaler components setup. + +### Gardener Operator + +To enable the _mutating_ [resource manager](../../concepts/resource-manager.md) webhook, the `VPAInPlaceUpdates` feature gate must be set to `true`: + +```yaml +featureGates: + VPAInPlaceUpdates: true +``` + +Refer to the `operator` component configuration [manifest](../../../example/operator/10-componentconfig.yaml) for an overview. + +## References + +- [Gardener Feature Gates](../../deployment/feature_gates.md) +- [Vertical Pod Autoscaling In-Place Updates](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/docs/features.md#in-place-updates-inplaceorrecreate) +- [Vertical Pod Autoscaling In-Place Updates Guide](./in-place-resource-updates.md) diff --git a/pkg/apis/resources/v1alpha1/types.go b/pkg/apis/resources/v1alpha1/types.go index 436259dd357..719cc9dcdca 100644 --- a/pkg/apis/resources/v1alpha1/types.go +++ b/pkg/apis/resources/v1alpha1/types.go @@ -191,6 +191,10 @@ const ( // NetworkingServiceNamespace is a constant for a label on a NetworkPolicy which contains the namespace of the // Service is has been created for. NetworkingServiceNamespace = NetworkPolicyLabelKeyPrefix + "service-namespace" + // VPAInPlaceUpdatesSkip is a constant for a label on a VerticalPodAutoscaler resource which indicates that + // it's updateMode field option should not be modified to "InPlaceOrRecreate" + // TODO(vitanovs): Remove the VPAInPlaceUpdatesSkip label constant once the VPAInPlaceUpdates feature gates is deprecated. + VPAInPlaceUpdatesSkip = "vpa-in-place-updates.resources.gardener.cloud/skip" ) // +kubebuilder:resource:shortName="mr" diff --git a/pkg/component/autoscaling/vpa/admissioncontroller.go b/pkg/component/autoscaling/vpa/admissioncontroller.go index 1d0220f9c54..571911c54f9 100644 --- a/pkg/component/autoscaling/vpa/admissioncontroller.go +++ b/pkg/component/autoscaling/vpa/admissioncontroller.go @@ -307,7 +307,7 @@ func (v *vpa) reconcileAdmissionControllerVPA(vpa *vpaautoscalingv1.VerticalPodA Kind: "Deployment", Name: deployment.Name, }, - UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto)}, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeRecreate)}, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ { diff --git a/pkg/component/autoscaling/vpa/recommender.go b/pkg/component/autoscaling/vpa/recommender.go index b373744a344..b9af150c9d9 100644 --- a/pkg/component/autoscaling/vpa/recommender.go +++ b/pkg/component/autoscaling/vpa/recommender.go @@ -310,7 +310,7 @@ func (v *vpa) reconcileRecommenderVPA(vpa *vpaautoscalingv1.VerticalPodAutoscale Kind: "Deployment", Name: deployment.Name, }, - UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto)}, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeRecreate)}, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ { diff --git a/pkg/component/autoscaling/vpa/updater.go b/pkg/component/autoscaling/vpa/updater.go index 130c3febb88..968819e501b 100644 --- a/pkg/component/autoscaling/vpa/updater.go +++ b/pkg/component/autoscaling/vpa/updater.go @@ -275,7 +275,7 @@ func (v *vpa) reconcileUpdaterVPA(vpa *vpaautoscalingv1.VerticalPodAutoscaler, d Kind: "Deployment", Name: deployment.Name, }, - UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto)}, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeRecreate)}, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ { diff --git a/pkg/component/autoscaling/vpa/vpa_test.go b/pkg/component/autoscaling/vpa/vpa_test.go index 8b78011ccf1..686aaf7feed 100644 --- a/pkg/component/autoscaling/vpa/vpa_test.go +++ b/pkg/component/autoscaling/vpa/vpa_test.go @@ -103,7 +103,7 @@ var _ = Describe("VPA", func() { valuesRecommender ValuesRecommender valuesUpdater ValuesUpdater - vpaUpdateModeAuto = vpaautoscalingv1.UpdateModeAuto + vpaUpdateMode = vpaautoscalingv1.UpdateModeRecreate vpaControlledValues = vpaautoscalingv1.ContainerControlledValuesRequestsOnly maxUnavailable = intstr.FromInt32(1) @@ -577,7 +577,7 @@ var _ = Describe("VPA", func() { Kind: "Deployment", Name: "vpa-updater", }, - UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: &vpaUpdateModeAuto}, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: &vpaUpdateMode}, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ { @@ -961,7 +961,7 @@ var _ = Describe("VPA", func() { Kind: "Deployment", Name: "vpa-recommender", }, - UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: &vpaUpdateModeAuto}, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: &vpaUpdateMode}, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ { @@ -1312,7 +1312,7 @@ var _ = Describe("VPA", func() { Kind: "Deployment", Name: "vpa-admission-controller", }, - UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: &vpaUpdateModeAuto}, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{UpdateMode: &vpaUpdateMode}, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ { diff --git a/pkg/component/gardener/resourcemanager/resource_manager.go b/pkg/component/gardener/resourcemanager/resource_manager.go index 34cae0dd327..f64f42f3f2a 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager.go +++ b/pkg/component/gardener/resourcemanager/resource_manager.go @@ -67,6 +67,7 @@ import ( "github.com/gardener/gardener/pkg/resourcemanager/webhook/projectedtokenmount" "github.com/gardener/gardener/pkg/resourcemanager/webhook/seccompprofile" "github.com/gardener/gardener/pkg/resourcemanager/webhook/systemcomponentsconfig" + "github.com/gardener/gardener/pkg/resourcemanager/webhook/vpainplaceupdates" "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/flow" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" @@ -333,6 +334,8 @@ type Values struct { NodeAgentAuthorizerAuthorizeWithSelectors *bool // PodKubeAPIServerLoadBalancingWebhook specifies the settings of pod-kube-apiserver-load-balancing webhook. PodKubeAPIServerLoadBalancingWebhook PodKubeAPIServerLoadBalancingWebhook + // VPAInPlaceUpdatesEnabled specifies if a vpa-in-place-pod-vertical-scaling webhook should be enabled. + VPAInPlaceUpdatesEnabled bool } // PodKubeAPIServerLoadBalancingWebhook specifies the settings of pod-kube-apiserver-load-balancing webhook. @@ -618,6 +621,9 @@ func (r *resourceManager) ensureConfigMap(ctx context.Context, configMap *corev1 SeccompProfile: resourcemanagerconfigv1alpha1.SeccompProfileWebhookConfig{ Enabled: r.values.DefaultSeccompProfileEnabled, }, + VPAInPlaceUpdates: resourcemanagerconfigv1alpha1.VPAInPlaceUpdatesConfig{ + Enabled: r.values.VPAInPlaceUpdatesEnabled, + }, }, } @@ -1395,6 +1401,10 @@ func (r *resourceManager) newMutatingWebhookConfigurationWebhooks( webhooks = append(webhooks, NewPodTopologySpreadConstraintsMutatingWebhook(r.values.NamePrefix, namespaceSelector, objectSelector, secretServerCA, buildClientConfigFn)) } + if r.values.VPAInPlaceUpdatesEnabled { + webhooks = append(webhooks, NewInPlaceUpdatesWebhook(namespaceSelector, secretServerCA, buildClientConfigFn)) + } + r.skipStaticPods(webhooks) return webhooks } @@ -1938,6 +1948,45 @@ func NewHighAvailabilityConfigMutatingWebhook(namespaceSelector, objectSelector } } +// NewInPlaceUpdatesWebhook returns the VerticalPodAutoscaler mutating webhook for the resourcemanager component for reuse +// between the component and integration tests. +func NewInPlaceUpdatesWebhook( + namespaceSelector *metav1.LabelSelector, + secretServerCA *corev1.Secret, + buildClientConfigFn func(*corev1.Secret, string) admissionregistrationv1.WebhookClientConfig, +) admissionregistrationv1.MutatingWebhook { + return admissionregistrationv1.MutatingWebhook{ + Name: "vpa-in-place-updates.resources.gardener.cloud", + Rules: []admissionregistrationv1.RuleWithOperations{{ + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{vpaautoscalingv1.SchemeGroupVersion.Group}, + APIVersions: []string{vpaautoscalingv1.SchemeGroupVersion.Version}, + Resources: []string{"verticalpodautoscalers"}, + }, + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, + }, + }}, + NamespaceSelector: namespaceSelector, + ObjectSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: resourcesv1alpha1.VPAInPlaceUpdatesSkip, + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + ClientConfig: buildClientConfigFn(secretServerCA, vpainplaceupdates.WebhookPath), + AdmissionReviewVersions: []string{admissionv1beta1.SchemeGroupVersion.Version, admissionv1.SchemeGroupVersion.Version}, + ReinvocationPolicy: ptr.To(admissionregistrationv1.NeverReinvocationPolicy), + FailurePolicy: ptr.To(admissionregistrationv1.Fail), + MatchPolicy: ptr.To(admissionregistrationv1.Equivalent), + SideEffects: ptr.To(admissionregistrationv1.SideEffectClassNone), + TimeoutSeconds: ptr.To[int32](10), + } +} + // NewEndpointSliceHintsMutatingWebhook returns the EndpointSlice hints mutating webhook for the resourcemanager component for reuse // between the component and integration tests. func NewEndpointSliceHintsMutatingWebhook( diff --git a/pkg/component/gardener/resourcemanager/resource_manager_test.go b/pkg/component/gardener/resourcemanager/resource_manager_test.go index 05f18397f2b..afff356eac4 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager_test.go +++ b/pkg/component/gardener/resourcemanager/resource_manager_test.go @@ -338,6 +338,7 @@ var _ = Describe("ResourceManager", func() { DefaultSeccompProfileEnabled: false, EndpointSliceHintsEnabled: false, PodTopologySpreadConstraintsEnabled: true, + VPAInPlaceUpdatesEnabled: false, LogLevel: "info", LogFormat: "json", Zones: []string{"a", "b"}, @@ -504,6 +505,7 @@ var _ = Describe("ResourceManager", func() { config.Webhooks.CRDDeletionProtection.Enabled = true config.Webhooks.ExtensionValidation.Enabled = true config.Webhooks.SeccompProfile.Enabled = true + config.Webhooks.VPAInPlaceUpdates.Enabled = true } if responsibilityMode == ForSource { @@ -1299,7 +1301,41 @@ var _ = Describe("ResourceManager", func() { MatchPolicy: &matchPolicyExact, SideEffects: &sideEffect, TimeoutSeconds: ptr.To[int32](10), - }) + }, + admissionregistrationv1.MutatingWebhook{ + Name: "vpa-in-place-updates.resources.gardener.cloud", + Rules: []admissionregistrationv1.RuleWithOperations{{ + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"autoscaling.k8s.io"}, + APIVersions: []string{"v1"}, + Resources: []string{"verticalpodautoscalers"}, + }, + Operations: []admissionregistrationv1.OperationType{"CREATE", "UPDATE"}, + }}, + NamespaceSelector: &metav1.LabelSelector{MatchExpressions: namespaceSelectorMatchExpressions}, + ObjectSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: resourcesv1alpha1.VPAInPlaceUpdatesSkip, + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Name: "gardener-resource-manager", + Namespace: deployNamespace, + Path: ptr.To("/webhooks/vpa-in-place-updates"), + }, + }, + AdmissionReviewVersions: []string{"v1beta1", "v1"}, + ReinvocationPolicy: &reinvocationPolicyNever, + FailurePolicy: &failurePolicyFail, + MatchPolicy: &matchPolicyEquivalent, + SideEffects: &sideEffect, + TimeoutSeconds: ptr.To[int32](10), + }, + ) return obj } @@ -2511,6 +2547,7 @@ subjects: cfg.SchedulingProfile = nil cfg.ResponsibilityMode = ForSource cfg.PodKubeAPIServerLoadBalancingWebhook.Enabled = true + cfg.VPAInPlaceUpdatesEnabled = true resourceManager = New(c, deployNamespace, sm, cfg) resourceManager.SetSecrets(secrets) }) @@ -2610,6 +2647,7 @@ subjects: utilruntime.Must(references.InjectAnnotations(deployment)) cfg.DefaultSeccompProfileEnabled = true + cfg.VPAInPlaceUpdatesEnabled = true cfg.SchedulingProfile = nil cfg.ResponsibilityMode = ForSourceAndTarget resourceManager = New(c, deployNamespace, sm, cfg) diff --git a/pkg/component/shared/resourcemanager.go b/pkg/component/shared/resourcemanager.go index bb7f7aa523b..69182864e81 100644 --- a/pkg/component/shared/resourcemanager.go +++ b/pkg/component/shared/resourcemanager.go @@ -49,6 +49,7 @@ func runtimeGardenerResourceManagerDefaultValues() resourcemanager.Values { }}, }, PodTopologySpreadConstraintsEnabled: false, + VPAInPlaceUpdatesEnabled: false, Replicas: ptr.To[int32](2), ResourceClass: ptr.To(v1beta1constants.SeedResourceManagerClass), ResponsibilityMode: resourcemanager.ForSource, diff --git a/pkg/component/shared/resourcemanager_test.go b/pkg/component/shared/resourcemanager_test.go index ec4b525b079..d14299aed01 100644 --- a/pkg/component/shared/resourcemanager_test.go +++ b/pkg/component/shared/resourcemanager_test.go @@ -71,6 +71,7 @@ var _ = Describe("ResourceManager", func() { }}, }, PodTopologySpreadConstraintsEnabled: false, + VPAInPlaceUpdatesEnabled: false, Replicas: ptr.To[int32](2), ResourceClass: ptr.To("seed"), ResponsibilityMode: resourcemanager.ForSource, diff --git a/pkg/features/features.go b/pkg/features/features.go index 5ddd1e08cda..ce140bad45f 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -74,6 +74,12 @@ const ( // owner: @hown3d // alpha: v1.130.0 UseUnifiedHTTPProxyPort featuregate.Feature = "UseUnifiedHTTPProxyPort" + + // VPAInPlaceUpdates enables the usage of in-place Pod resource updates in the Vertical Pod Autoscaler resources + // to perform in-place Pod resource updates. + // owner: @vitanovs @ialidzhikov + // alpha: v1.132.0 + VPAInPlaceUpdates featuregate.Feature = "VPAInPlaceUpdates" ) // DefaultFeatureGate is the central feature gate map used by all gardener components. @@ -110,6 +116,7 @@ var AllFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ DoNotCopyBackupCredentials: {Default: true, PreRelease: featuregate.Beta}, OpenTelemetryCollector: {Default: false, PreRelease: featuregate.Alpha}, UseUnifiedHTTPProxyPort: {Default: false, PreRelease: featuregate.Alpha}, + VPAInPlaceUpdates: {Default: false, PreRelease: featuregate.Alpha}, } // GetFeatures returns a feature gate map with the respective specifications. Non-existing feature gates are ignored. diff --git a/pkg/gardenlet/controller/seed/seed/components.go b/pkg/gardenlet/controller/seed/seed/components.go index f6df0b2ca97..94de02ed0ef 100644 --- a/pkg/gardenlet/controller/seed/seed/components.go +++ b/pkg/gardenlet/controller/seed/seed/components.go @@ -312,6 +312,9 @@ func (r *Reconciler) newGardenerResourceManager(seed *gardencorev1beta1.Seed, se }, }, }, + // TODO(vitanovs): Remove the VPAInPlaceUpdates webhook once the + // VPAInPlaceUpdates feature gates is deprecated. + VPAInPlaceUpdatesEnabled: features.DefaultFeatureGate.Enabled(features.VPAInPlaceUpdates), }) } diff --git a/pkg/gardenlet/features/features.go b/pkg/gardenlet/features/features.go index ccea06cf5a1..8ace7dac960 100644 --- a/pkg/gardenlet/features/features.go +++ b/pkg/gardenlet/features/features.go @@ -25,5 +25,6 @@ func GetFeatures() []featuregate.Feature { features.DoNotCopyBackupCredentials, features.OpenTelemetryCollector, features.UseUnifiedHTTPProxyPort, + features.VPAInPlaceUpdates, } } diff --git a/pkg/gardenlet/operation/botanist/resource_manager.go b/pkg/gardenlet/operation/botanist/resource_manager.go index 05e4ca7a5aa..da61222ee3a 100644 --- a/pkg/gardenlet/operation/botanist/resource_manager.go +++ b/pkg/gardenlet/operation/botanist/resource_manager.go @@ -16,6 +16,7 @@ import ( v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" "github.com/gardener/gardener/pkg/component/gardener/resourcemanager" "github.com/gardener/gardener/pkg/component/shared" + "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/logger" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" @@ -55,6 +56,9 @@ func (b *Botanist) DefaultResourceManager() (resourcemanager.Interface, error) { SystemComponentTolerations: gardenerutils.ExtractSystemComponentsTolerations(b.Shoot.GetInfo().Spec.Provider.Workers), TargetNamespaces: []string{metav1.NamespaceSystem, v1beta1constants.KubernetesDashboardNamespace, corev1.NamespaceNodeLease}, TopologyAwareRoutingEnabled: b.Shoot.TopologyAwareRoutingEnabled, + // TODO(vitanovs): Remove the VPAInPlaceUpdates webhook once the + // VPAInPlaceUpdates feature gates is deprecated. + VPAInPlaceUpdatesEnabled: b.isVPAInPlaceUpdatesEnabled(), } ) @@ -92,3 +96,19 @@ func (b *Botanist) DeployGardenerResourceManager(ctx context.Context) error { func (b *Botanist) ScaleGardenerResourceManagerToOne(ctx context.Context) error { return kubernetesutils.ScaleDeployment(ctx, b.SeedClientSet.Client(), client.ObjectKey{Namespace: b.Shoot.ControlPlaneNamespace, Name: v1beta1constants.DeploymentNameGardenerResourceManager}, 1) } + +func (b *Botanist) isVPAInPlaceUpdatesEnabled() bool { + var ( + isGardenletFeatureGateEnabled = features.DefaultFeatureGate.Enabled(features.VPAInPlaceUpdates) + isShootVPAFeatureGateEnabled = true + ) + + shootVerticalPodAutoscaler := b.Shoot.GetInfo().Spec.Kubernetes.VerticalPodAutoscaler + if shootVerticalPodAutoscaler != nil { + if value, ok := shootVerticalPodAutoscaler.FeatureGates["InPlaceOrRecreate"]; ok { + isShootVPAFeatureGateEnabled = value + } + } + + return isGardenletFeatureGateEnabled && isShootVPAFeatureGateEnabled +} diff --git a/pkg/gardenlet/operation/botanist/resource_manager_test.go b/pkg/gardenlet/operation/botanist/resource_manager_test.go index 1fe6d87539a..d29aa832014 100644 --- a/pkg/gardenlet/operation/botanist/resource_manager_test.go +++ b/pkg/gardenlet/operation/botanist/resource_manager_test.go @@ -32,6 +32,7 @@ import ( mockresourcemanager "github.com/gardener/gardener/pkg/component/gardener/resourcemanager/mock" mockkubeapiserver "github.com/gardener/gardener/pkg/component/kubernetes/apiserver/mock" "github.com/gardener/gardener/pkg/component/shared" + "github.com/gardener/gardener/pkg/features" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/gardenlet/operation" . "github.com/gardener/gardener/pkg/gardenlet/operation/botanist" @@ -144,6 +145,54 @@ var _ = Describe("ResourceManager", func() { Expect(err).NotTo(HaveOccurred()) Expect(resourceManager.GetValues().NodeAgentAuthorizerAuthorizeWithSelectors).To(PointTo(Equal(true))) }) + + When("VPAInPlaceUpdates feature gate is enabled", func() { + BeforeEach(func() { + DeferCleanup(test.WithFeatureGate(features.DefaultFeatureGate, features.VPAInPlaceUpdates, true)) + }) + + Context("with Shoot Vertical Pod Autoscaler InPlaceOrRecreate feature gate enabled", func() { + BeforeEach(func() { + botanist.Shoot.SetInfo(&gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Kubernetes: gardencorev1beta1.Kubernetes{ + VerticalPodAutoscaler: &gardencorev1beta1.VerticalPodAutoscaler{ + FeatureGates: map[string]bool{"InPlaceOrRecreate": true}, + }, + }, + }, + }) + }) + + It("should set VPAInPlaceUpdatesEnabled=true", func() { + resourceManager, err := botanist.DefaultResourceManager() + Expect(resourceManager).NotTo(BeNil()) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceManager.GetValues().VPAInPlaceUpdatesEnabled).To(BeTrue()) + }) + }) + + Context("with Shoot Vertical Pod Autoscaler InPlaceOrRecreate feature gate disabled", func() { + BeforeEach(func() { + botanist.Shoot.SetInfo(&gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Kubernetes: gardencorev1beta1.Kubernetes{ + VerticalPodAutoscaler: &gardencorev1beta1.VerticalPodAutoscaler{ + FeatureGates: map[string]bool{"InPlaceOrRecreate": false}, + }, + }, + }, + }) + }) + + It("should set VPAInPlaceUpdatesEnabled=false", func() { + resourceManager, err := botanist.DefaultResourceManager() + Expect(resourceManager).NotTo(BeNil()) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceManager.GetValues().VPAInPlaceUpdatesEnabled).To(BeFalse()) + }) + }) + }) }) Describe("#DeployGardenerResourceManager", func() { diff --git a/pkg/operator/controller/garden/garden/components.go b/pkg/operator/controller/garden/garden/components.go index a4f1fd71e9d..05f5aa7f139 100644 --- a/pkg/operator/controller/garden/garden/components.go +++ b/pkg/operator/controller/garden/garden/components.go @@ -428,6 +428,7 @@ func (r *Reconciler) newGardenerResourceManager(garden *operatorv1alpha1.Garden, }, }, }, + VPAInPlaceUpdatesEnabled: features.DefaultFeatureGate.Enabled(features.VPAInPlaceUpdates), }) } diff --git a/pkg/operator/features/features.go b/pkg/operator/features/features.go index 9e20bae71b6..7359fc33bbe 100644 --- a/pkg/operator/features/features.go +++ b/pkg/operator/features/features.go @@ -17,5 +17,6 @@ func RegisterFeatureGates() { features.IstioTLSTermination, features.DoNotCopyBackupCredentials, features.UseUnifiedHTTPProxyPort, + features.VPAInPlaceUpdates, ))) } diff --git a/pkg/resourcemanager/apis/config/v1alpha1/types.go b/pkg/resourcemanager/apis/config/v1alpha1/types.go index a7fe58cbbaf..717f3967458 100644 --- a/pkg/resourcemanager/apis/config/v1alpha1/types.go +++ b/pkg/resourcemanager/apis/config/v1alpha1/types.go @@ -251,6 +251,8 @@ type ResourceManagerWebhookConfiguration struct { NodeAgentAuthorizer NodeAgentAuthorizerWebhookConfig `json:"nodeAgentAuthorizer"` // SeccompProfile is the configuration for the seccomp-profile webhook. SeccompProfile SeccompProfileWebhookConfig `json:"seccompProfile"` + // VPAInPlaceUpdates is the configuration for the vpa-in-place-updates webhook. + VPAInPlaceUpdates VPAInPlaceUpdatesConfig `json:"vpaInPlaceUpdates"` } // CRDDeletionProtection is the configuration for the crd-deletion-protection webhook. @@ -356,6 +358,12 @@ type SeccompProfileWebhookConfig struct { Enabled bool `json:"enabled"` } +// VPAInPlaceUpdatesConfig is the configuration for the vpa-in-place-updates webhook. +type VPAInPlaceUpdatesConfig struct { + // Enabled defines whether this webhook is enabled. + Enabled bool `json:"enabled"` +} + const ( // DefaultResourceClass is used as resource class if no class is specified on the command line. DefaultResourceClass = "resources" diff --git a/pkg/resourcemanager/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/resourcemanager/apis/config/v1alpha1/zz_generated.deepcopy.go index 54698bc5fc6..4175bce615f 100644 --- a/pkg/resourcemanager/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/resourcemanager/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -551,6 +551,7 @@ func (in *ResourceManagerWebhookConfiguration) DeepCopyInto(out *ResourceManager in.ProjectedTokenMount.DeepCopyInto(&out.ProjectedTokenMount) in.NodeAgentAuthorizer.DeepCopyInto(&out.NodeAgentAuthorizer) out.SeccompProfile = in.SeccompProfile + out.VPAInPlaceUpdates = in.VPAInPlaceUpdates return } @@ -696,3 +697,19 @@ func (in *TokenRequestorControllerConfig) DeepCopy() *TokenRequestorControllerCo in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPAInPlaceUpdatesConfig) DeepCopyInto(out *VPAInPlaceUpdatesConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPAInPlaceUpdatesConfig. +func (in *VPAInPlaceUpdatesConfig) DeepCopy() *VPAInPlaceUpdatesConfig { + if in == nil { + return nil + } + out := new(VPAInPlaceUpdatesConfig) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/resourcemanager/webhook/add.go b/pkg/resourcemanager/webhook/add.go index 5325ec2bf5a..41e714916cd 100644 --- a/pkg/resourcemanager/webhook/add.go +++ b/pkg/resourcemanager/webhook/add.go @@ -24,6 +24,7 @@ import ( "github.com/gardener/gardener/pkg/resourcemanager/webhook/projectedtokenmount" "github.com/gardener/gardener/pkg/resourcemanager/webhook/seccompprofile" "github.com/gardener/gardener/pkg/resourcemanager/webhook/systemcomponentsconfig" + "github.com/gardener/gardener/pkg/resourcemanager/webhook/vpainplaceupdates" ) // AddToManager adds all webhook handlers to the given manager. @@ -136,5 +137,13 @@ func AddToManager(mgr manager.Manager, sourceCluster, targetCluster cluster.Clus } } + if cfg.Webhooks.VPAInPlaceUpdates.Enabled { + if err := (&vpainplaceupdates.Handler{ + Logger: mgr.GetLogger().WithName("webhook").WithName(vpainplaceupdates.HandlerName), + }).AddToManager(mgr); err != nil { + return fmt.Errorf("failed adding %s webhook handler: %w", vpainplaceupdates.HandlerName, err) + } + } + return nil } diff --git a/pkg/resourcemanager/webhook/vpainplaceupdates/add.go b/pkg/resourcemanager/webhook/vpainplaceupdates/add.go new file mode 100644 index 00000000000..210a83e573a --- /dev/null +++ b/pkg/resourcemanager/webhook/vpainplaceupdates/add.go @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package vpainplaceupdates + +import ( + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + // HandlerName is the name of the webhook handler. + HandlerName = "vpa-in-place-updates" + // WebhookPath is the path at which the handler should be registered. + WebhookPath = "/webhooks/vpa-in-place-updates" +) + +// AddToManager adds Handler to the given manager. +func (h *Handler) AddToManager(mgr manager.Manager) error { + webhook := admission. + WithCustomDefaulter(mgr.GetScheme(), &vpaautoscalingv1.VerticalPodAutoscaler{}, h). + WithRecoverPanic(true) + + mgr.GetWebhookServer().Register(WebhookPath, webhook) + return nil +} diff --git a/pkg/resourcemanager/webhook/vpainplaceupdates/handler.go b/pkg/resourcemanager/webhook/vpainplaceupdates/handler.go new file mode 100644 index 00000000000..6b10e488334 --- /dev/null +++ b/pkg/resourcemanager/webhook/vpainplaceupdates/handler.go @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package vpainplaceupdates + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" +) + +// Handler handles admission requests and sets the vpa.spec.updatePolicy.updateMode field in VerticalPodAutoscaler resources. +type Handler struct { + Logger logr.Logger +} + +// Default defaults the update mode of the provided VerticalPodAutoscaler. +func (h *Handler) Default(ctx context.Context, obj runtime.Object) error { + vpa, ok := obj.(*vpaautoscalingv1.VerticalPodAutoscaler) + if !ok { + return fmt.Errorf("expected *vpaautoscalingv1.VerticalPodAutoscaler but got %T", obj) + } + + req, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + + log := h.Logger.WithValues("vpa", kubernetesutils.ObjectKeyForCreateWebhooks(vpa, req)) + + if vpa.Spec.UpdatePolicy == nil { + vpa.Spec.UpdatePolicy = &vpaautoscalingv1.PodUpdatePolicy{} + } + + updateMode := ptr.Deref(vpa.Spec.UpdatePolicy.UpdateMode, vpaautoscalingv1.UpdateModeAuto) + if updateMode == vpaautoscalingv1.UpdateModeAuto || updateMode == vpaautoscalingv1.UpdateModeRecreate { + log.Info("Mutating VerticalPodAutoscaler with InPlaceOrRecreate update mode") + vpa.Spec.UpdatePolicy.UpdateMode = ptr.To(vpaautoscalingv1.UpdateModeInPlaceOrRecreate) + } + + return nil +} diff --git a/skaffold-gardenadm.yaml b/skaffold-gardenadm.yaml index aee74d8838d..d56c50679d6 100644 --- a/skaffold-gardenadm.yaml +++ b/skaffold-gardenadm.yaml @@ -318,6 +318,7 @@ build: - pkg/resourcemanager/webhook/projectedtokenmount - pkg/resourcemanager/webhook/seccompprofile - pkg/resourcemanager/webhook/systemcomponentsconfig + - pkg/resourcemanager/webhook/vpainplaceupdates - pkg/utils - pkg/utils/chart - pkg/utils/context @@ -578,6 +579,7 @@ build: - pkg/resourcemanager/webhook/projectedtokenmount - pkg/resourcemanager/webhook/seccompprofile - pkg/resourcemanager/webhook/systemcomponentsconfig + - pkg/resourcemanager/webhook/vpainplaceupdates - pkg/utils - pkg/utils/context - pkg/utils/errors @@ -931,6 +933,7 @@ build: - pkg/resourcemanager/webhook/projectedtokenmount - pkg/resourcemanager/webhook/seccompprofile - pkg/resourcemanager/webhook/systemcomponentsconfig + - pkg/resourcemanager/webhook/vpainplaceupdates - pkg/utils - pkg/utils/chart - pkg/utils/context diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml index 4a100372750..8b2a07f3199 100644 --- a/skaffold-operator.yaml +++ b/skaffold-operator.yaml @@ -288,6 +288,7 @@ build: - pkg/resourcemanager/webhook/projectedtokenmount - pkg/resourcemanager/webhook/seccompprofile - pkg/resourcemanager/webhook/systemcomponentsconfig + - pkg/resourcemanager/webhook/vpainplaceupdates - pkg/scheduler/apis/config/v1alpha1 - pkg/utils - pkg/utils/chart @@ -415,6 +416,7 @@ build: - pkg/resourcemanager/webhook/projectedtokenmount - pkg/resourcemanager/webhook/seccompprofile - pkg/resourcemanager/webhook/systemcomponentsconfig + - pkg/resourcemanager/webhook/vpainplaceupdates - pkg/utils - pkg/utils/context - pkg/utils/errors diff --git a/skaffold-seed.yaml b/skaffold-seed.yaml index 9f60d0474b4..9041f287d38 100644 --- a/skaffold-seed.yaml +++ b/skaffold-seed.yaml @@ -351,6 +351,7 @@ build: - pkg/resourcemanager/webhook/projectedtokenmount - pkg/resourcemanager/webhook/seccompprofile - pkg/resourcemanager/webhook/systemcomponentsconfig + - pkg/resourcemanager/webhook/vpainplaceupdates - pkg/utils - pkg/utils/chart - pkg/utils/context @@ -502,6 +503,7 @@ build: - pkg/resourcemanager/webhook/projectedtokenmount - pkg/resourcemanager/webhook/seccompprofile - pkg/resourcemanager/webhook/systemcomponentsconfig + - pkg/resourcemanager/webhook/vpainplaceupdates - pkg/utils - pkg/utils/context - pkg/utils/errors diff --git a/skaffold.yaml b/skaffold.yaml index 4ffe908bb22..2c4bdc7756f 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -1267,6 +1267,7 @@ build: - pkg/resourcemanager/webhook/projectedtokenmount - pkg/resourcemanager/webhook/seccompprofile - pkg/resourcemanager/webhook/systemcomponentsconfig + - pkg/resourcemanager/webhook/vpainplaceupdates - pkg/utils - pkg/utils/chart - pkg/utils/context @@ -1418,6 +1419,7 @@ build: - pkg/resourcemanager/webhook/projectedtokenmount - pkg/resourcemanager/webhook/seccompprofile - pkg/resourcemanager/webhook/systemcomponentsconfig + - pkg/resourcemanager/webhook/vpainplaceupdates - pkg/utils - pkg/utils/context - pkg/utils/errors diff --git a/test/integration/resourcemanager/vpainplaceupdates/vpainplaceupdates_suite_test.go b/test/integration/resourcemanager/vpainplaceupdates/vpainplaceupdates_suite_test.go new file mode 100644 index 00000000000..f877ca4ce36 --- /dev/null +++ b/test/integration/resourcemanager/vpainplaceupdates/vpainplaceupdates_suite_test.go @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package vpainplaceupdates_test + +import ( + "context" + "net/http" + "path/filepath" + "testing" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + controllerconfig "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/gardener/gardener/pkg/component/gardener/resourcemanager" + "github.com/gardener/gardener/pkg/logger" + resourcemanagerclient "github.com/gardener/gardener/pkg/resourcemanager/client" + "github.com/gardener/gardener/pkg/resourcemanager/webhook/vpainplaceupdates" + "github.com/gardener/gardener/pkg/utils" + . "github.com/gardener/gardener/pkg/utils/test/matchers" +) + +func TestVPAInPlaceUpdatesProfile(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Test Integration ResourceManager VPAInPlaceUpdates Suite") +} + +const testID = "vpa-in-place-updates-webhook-test" + +var ( + ctx = context.Background() + log logr.Logger + + testEnv *envtest.Environment + restConfig *rest.Config + testClient client.Client + + testNamespace *corev1.Namespace +) + +var _ = BeforeSuite(func() { + logf.SetLogger(logger.MustNewZapLogger(logger.DebugLevel, logger.FormatJSON, zap.WriteTo(GinkgoWriter))) + log = logf.Log.WithName(testID) + + testNamespaceName := testID + "-" + utils.ComputeSHA256Hex([]byte(uuid.NewUUID()))[:8] + + By("Start test environment") + testEnv = &envtest.Environment{ + WebhookInstallOptions: envtest.WebhookInstallOptions{ + MutatingWebhooks: getMutatingWebhookConfigurations(testNamespaceName), + }, + CRDInstallOptions: envtest.CRDInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "..", "pkg", "component", "autoscaling", "vpa", "templates", "crd-autoscaling.k8s.io_verticalpodautoscalers.yaml")}, + }, + ErrorIfCRDPathMissing: true, + } + + var err error + restConfig, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(restConfig).NotTo(BeNil()) + + DeferCleanup(func() { + By("Stop test environment") + Expect(testEnv.Stop()).To(Succeed()) + }) + + By("Create test client") + testClient, err = client.New(restConfig, client.Options{Scheme: resourcemanagerclient.CombinedScheme}) + Expect(err).NotTo(HaveOccurred()) + + By("Create test Namespace") + testNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespaceName, + }, + } + Expect(testClient.Create(ctx, testNamespace)).To(Succeed()) + log.Info("Created Namespace for test", "namespaceName", testNamespace.Name) + + DeferCleanup(func() { + By("Delete test Namespace") + Expect(testClient.Delete(ctx, testNamespace)).To(Or(Succeed(), BeNotFoundError())) + }) + + By("Setup manager") + mgr, err := manager.New(restConfig, manager.Options{ + WebhookServer: webhook.NewServer(webhook.Options{ + Port: testEnv.WebhookInstallOptions.LocalServingPort, + Host: testEnv.WebhookInstallOptions.LocalServingHost, + CertDir: testEnv.WebhookInstallOptions.LocalServingCertDir, + }), + Metrics: metricsserver.Options{BindAddress: "0"}, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespace.Name: {}}, + }, + Controller: controllerconfig.Controller{ + SkipNameValidation: ptr.To(true), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Register webhook") + Expect((&vpainplaceupdates.Handler{ + Logger: log, + }).AddToManager(mgr)).To(Succeed()) + + By("Start manager") + mgrContext, mgrCancel := context.WithCancel(ctx) + + go func() { + defer GinkgoRecover() + Expect(mgr.Start(mgrContext)).To(Succeed()) + }() + + Eventually(func() error { + checker := mgr.GetWebhookServer().StartedChecker() + return checker(&http.Request{}) + }).Should(Succeed()) + + DeferCleanup(func() { + By("Stop manager") + mgrCancel() + }) +}) + +func getMutatingWebhookConfigurations(namespaceName string) []*admissionregistrationv1.MutatingWebhookConfiguration { + return []*admissionregistrationv1.MutatingWebhookConfiguration{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: admissionregistrationv1.SchemeGroupVersion.String(), + Kind: "MutatingWebhookConfiguration", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-resource-manager", + }, + Webhooks: []admissionregistrationv1.MutatingWebhook{ + resourcemanager.NewInPlaceUpdatesWebhook(&metav1.LabelSelector{ + MatchLabels: map[string]string{corev1.LabelMetadataName: namespaceName}, + }, nil, func(_ *corev1.Secret, path string) admissionregistrationv1.WebhookClientConfig { + return admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Path: &path, + }, + } + }), + }, + }, + } +} diff --git a/test/integration/resourcemanager/vpainplaceupdates/vpainplaceupdates_test.go b/test/integration/resourcemanager/vpainplaceupdates/vpainplaceupdates_test.go new file mode 100644 index 00000000000..22a82992231 --- /dev/null +++ b/test/integration/resourcemanager/vpainplaceupdates/vpainplaceupdates_test.go @@ -0,0 +1,111 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package vpainplaceupdates_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/utils/ptr" +) + +var _ = Describe("VPAInPlaceUpdates tests", func() { + var ( + vpa *vpaautoscalingv1.VerticalPodAutoscaler + ) + + BeforeEach(func() { + vpa = &vpaautoscalingv1.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + Namespace: testNamespace.Name, + }, + Spec: vpaautoscalingv1.VerticalPodAutoscalerSpec{ + TargetRef: &autoscalingv1.CrossVersionObjectReference{ + Name: "test-deployment", + APIVersion: "apps/v1", + Kind: "Deployment", + }, + }, + } + }) + + Context("when skip label is specified", func() { + BeforeEach(func() { + vpa.Spec.UpdatePolicy = &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeRecreate), + } + }) + + It("should not mutate vertical pod autoscaler", func() { + metav1.SetMetaDataLabel(&vpa.ObjectMeta, "vpa-in-place-updates.resources.gardener.cloud/skip", "") + + Expect(testClient.Create(ctx, vpa)).To(Succeed()) + Expect(vpa.Spec.UpdatePolicy.UpdateMode).To(Equal(ptr.To(vpaautoscalingv1.UpdateModeRecreate))) + }) + + }) + + Context("when update mode is Auto", func() { + BeforeEach(func() { + vpa.Spec.UpdatePolicy = &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto), + } + }) + + It("should mutate vertical pod autoscaler", func() { + Expect(testClient.Create(ctx, vpa)).To(Succeed()) + Expect(vpa.Spec.UpdatePolicy.UpdateMode).To(Equal(ptr.To(vpaautoscalingv1.UpdateModeInPlaceOrRecreate))) + }) + }) + + Context("when update mode is Recreate", func() { + BeforeEach(func() { + vpa.Spec.UpdatePolicy = &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeRecreate), + } + }) + + It("should mutate vertical pod autoscaler", func() { + Expect(testClient.Create(ctx, vpa)).To(Succeed()) + Expect(vpa.Spec.UpdatePolicy.UpdateMode).To(Equal(ptr.To(vpaautoscalingv1.UpdateModeInPlaceOrRecreate))) + }) + }) + + Context("when update mode is Initial", func() { + BeforeEach(func() { + vpa.Spec.UpdatePolicy = &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeInitial), + } + }) + + It("should not mutate vertical pod autoscaler", func() { + Expect(testClient.Create(ctx, vpa)).To(Succeed()) + Expect(vpa.Spec.UpdatePolicy.UpdateMode).To(Equal(ptr.To(vpaautoscalingv1.UpdateModeInitial))) + }) + }) + + Context("when update mode is Off", func() { + BeforeEach(func() { + vpa.Spec.UpdatePolicy = &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeOff), + } + }) + + It("should not mutate vertical pod autoscaler", func() { + Expect(testClient.Create(ctx, vpa)).To(Succeed()) + Expect(vpa.Spec.UpdatePolicy.UpdateMode).To(Equal(ptr.To(vpaautoscalingv1.UpdateModeOff))) + }) + }) + + Context("when update mode is not specified", func() { + It("should mutate vertical pod autoscaler", func() { + Expect(testClient.Create(ctx, vpa)).To(Succeed()) + Expect(vpa.Spec.UpdatePolicy.UpdateMode).To(Equal(ptr.To(vpaautoscalingv1.UpdateModeInPlaceOrRecreate))) + }) + }) +}) From 6f033e1d5f87be3cc0f26ec4be44dbfe1be3f2a3 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 12 Nov 2025 19:11:10 +0100 Subject: [PATCH 100/176] Update helm to v3.19.2 (#13434) --- go.mod | 2 +- go.sum | 4 ++-- hack/tools.mk | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 1b8e79e6c60..1c08fa71d47 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( gomodules.xyz/jsonpatch/v2 v2.5.0 gonum.org/v1/gonum v0.16.0 google.golang.org/protobuf v1.36.10 - helm.sh/helm/v3 v3.19.1 + helm.sh/helm/v3 v3.19.2 istio.io/api v1.27.3 istio.io/client-go v1.27.2 k8s.io/api v0.34.1 diff --git a/go.sum b/go.sum index 348dd24bcbf..28751314f53 100644 --- a/go.sum +++ b/go.sum @@ -1074,8 +1074,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -helm.sh/helm/v3 v3.19.1 h1:QVMzHbanyurO8oynx0drDOfG02XxSvrHqaFrf9yrMf0= -helm.sh/helm/v3 v3.19.1/go.mod h1:gX10tB5ErM+8fr7bglUUS/UfTOO8UUTYWIBH1IYNnpE= +helm.sh/helm/v3 v3.19.2 h1:psQjaM8aIWrSVEly6PgYtLu/y6MRSmok4ERiGhZmtUY= +helm.sh/helm/v3 v3.19.2/go.mod h1:gX10tB5ErM+8fr7bglUUS/UfTOO8UUTYWIBH1IYNnpE= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/tools.mk b/hack/tools.mk index 01d8d73cd2c..468a8d956a7 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -65,7 +65,7 @@ GO_ADD_LICENSE_VERSION ?= v1.2.0 GOIMPORTSREVISER_VERSION ?= v3.10.0 GO_VULN_CHECK_VERSION ?= latest # renovate: datasource=github-releases depName=helm/helm -HELM_VERSION ?= v3.19.1 +HELM_VERSION ?= v3.19.2 # renovate: datasource=github-releases depName=kubernetes-sigs/kind KIND_VERSION ?= v0.30.0 # renovate: datasource=github-releases depName=kubernetes/kubernetes From 2c7531706c882bf310d6d955006489786ef2edbc Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 13 Nov 2025 04:56:00 +0100 Subject: [PATCH 101/176] Update dependency crate-ci/typos to v1.39.1 (#13441) --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 468a8d956a7..09166f366d0 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -83,7 +83,7 @@ YQ_VERSION ?= v4.48.1 # renovate: datasource=github-releases depName=ironcore-dev/vgopath VGOPATH_VERSION ?= v0.1.9 # renovate: datasource=github-releases depName=crate-ci/typos -TYPOS_VERSION ?= v1.39.0 +TYPOS_VERSION ?= v1.39.1 # tool versions from go.mod CONTROLLER_GEN_VERSION ?= $(call version_gomod,sigs.k8s.io/controller-tools) From 38e2692c22052d9bcc6b1092595eab96a769d79c Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Thu, 13 Nov 2025 06:31:59 +0200 Subject: [PATCH 102/176] Update the list of known extension implementations (#13424) --- extensions/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extensions/README.md b/extensions/README.md index 9ff935d5505..b5a781db595 100644 --- a/extensions/README.md +++ b/extensions/README.md @@ -21,6 +21,7 @@ Check out these repositories for implementations of the Gardener Extension contr - [Equinix Metal](https://github.com/gardener/gardener-extension-provider-equinix-metal) - [GCP](https://github.com/gardener/gardener-extension-provider-gcp) - [Hetzner Cloud](https://github.com/23technologies/gardener-extension-provider-hcloud) +- [IronCore](https://github.com/ironcore-dev/gardener-extension-provider-ironcore) - [Kubevirt](https://github.com/gardener/gardener-extension-provider-kubevirt) - [MetalStack](https://github.com/metal-stack/gardener-extension-provider-metal) - [OpenStack](https://github.com/gardener/gardener-extension-provider-openstack) @@ -61,6 +62,8 @@ The primary DNS provider manages `DNSRecord` resources (mandatory for Gardener r ### Generic Extensions - [Minimal Working Example](https://github.com/23technologies/gardener-extension-mwe) +- [ACL](https://github.com/stackitcloud/gardener-extension-acl) +- [Image Rewriter](https://github.com/gardener/gardener-extension-image-rewriter) - [Registry Cache](https://github.com/gardener/gardener-extension-registry-cache) - [Shoot Certificate Service](https://github.com/gardener/gardener-extension-shoot-cert-service) - [Shoot DNS Service](https://github.com/gardener/gardener-extension-shoot-dns-service) @@ -72,6 +75,10 @@ The primary DNS provider manages `DNSRecord` resources (mandatory for Gardener r - [Shoot OpenID Connect Service](https://github.com/gardener/gardener-extension-shoot-oidc-service) - [Shoot Rsyslog Relp](https://github.com/gardener/gardener-extension-shoot-rsyslog-relp) +### Others + +- [OIDC Apps Controller](https://github.com/gardener/oidc-apps-controller) + If you implemented a new extension, please feel free to add it to this list! ## Feedback and Support From ebb45fa40509999bdac8d526ab4091fbe9bba452 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 13 Nov 2025 10:09:00 +0100 Subject: [PATCH 103/176] Update kubernetes packages to v0.34.2 (#13437) --- go.mod | 32 ++++++++++++++--------------- go.sum | 64 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 1c08fa71d47..54103ec0d4c 100644 --- a/go.mod +++ b/go.mod @@ -63,25 +63,25 @@ require ( helm.sh/helm/v3 v3.19.2 istio.io/api v1.27.3 istio.io/client-go v1.27.2 - k8s.io/api v0.34.1 - k8s.io/apiextensions-apiserver v0.34.1 - k8s.io/apimachinery v0.34.1 - k8s.io/apiserver v0.34.1 + k8s.io/api v0.34.2 + k8s.io/apiextensions-apiserver v0.34.2 + k8s.io/apimachinery v0.34.2 + k8s.io/apiserver v0.34.2 k8s.io/autoscaler/vertical-pod-autoscaler v1.5.1 - k8s.io/cli-runtime v0.34.1 - k8s.io/client-go v0.34.1 - k8s.io/cluster-bootstrap v0.34.1 - k8s.io/code-generator v0.34.1 - k8s.io/component-base v0.34.1 - k8s.io/component-helpers v0.34.1 + k8s.io/cli-runtime v0.34.2 + k8s.io/client-go v0.34.2 + k8s.io/cluster-bootstrap v0.34.2 + k8s.io/code-generator v0.34.2 + k8s.io/component-base v0.34.2 + k8s.io/component-helpers v0.34.2 k8s.io/klog/v2 v2.130.1 - k8s.io/kube-aggregator v0.34.1 + k8s.io/kube-aggregator v0.34.2 k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 - k8s.io/kube-proxy v0.34.1 + k8s.io/kube-proxy v0.34.2 k8s.io/kube-state-metrics/v2 v2.13.0 - k8s.io/kubelet v0.34.1 - k8s.io/metrics v0.34.1 - k8s.io/pod-security-admission v0.34.1 + k8s.io/kubelet v0.34.2 + k8s.io/metrics v0.34.2 + k8s.io/pod-security-admission v0.34.2 k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 sigs.k8s.io/controller-runtime v0.22.4 sigs.k8s.io/controller-tools v0.19.0 @@ -279,7 +279,7 @@ require ( k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect k8s.io/gengo/v2 v2.0.0-20250820003526-c297c0c1eb9d // indirect k8s.io/klog v1.0.0 // indirect - k8s.io/kms v0.34.1 // indirect + k8s.io/kms v0.34.2 // indirect k8s.io/sample-controller v0.30.3 // indirect oras.land/oras-go/v2 v2.6.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect diff --git a/go.sum b/go.sum index 28751314f53..3abf619c064 100644 --- a/go.sum +++ b/go.sum @@ -1086,31 +1086,31 @@ istio.io/api v1.27.3/go.mod h1:DTVGH6CLXj5W8FF9JUD3Tis78iRgT1WeuAnxfTz21Wg= istio.io/client-go v1.27.2 h1:4IsF7UAdV5Yg0iq6ONyWZpjFr3z2ahkIbLWyzOHCAwA= istio.io/client-go v1.27.2/go.mod h1:zgT5R1USl6rwYK1eb2kisPuiji05TQJE7CQHU253iAg= k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= -k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= -k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= -k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= -k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= +k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= +k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= +k8s.io/apiextensions-apiserver v0.34.2 h1:WStKftnGeoKP4AZRz/BaAAEJvYp4mlZGN0UCv+uvsqo= +k8s.io/apiextensions-apiserver v0.34.2/go.mod h1:398CJrsgXF1wytdaanynDpJ67zG4Xq7yj91GrmYN2SE= k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= -k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -k8s.io/apiserver v0.34.1 h1:U3JBGdgANK3dfFcyknWde1G6X1F4bg7PXuvlqt8lITA= -k8s.io/apiserver v0.34.1/go.mod h1:eOOc9nrVqlBI1AFCvVzsob0OxtPZUCPiUJL45JOTBG0= +k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= +k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apiserver v0.34.2 h1:2/yu8suwkmES7IzwlehAovo8dDE07cFRC7KMDb1+MAE= +k8s.io/apiserver v0.34.2/go.mod h1:gqJQy2yDOB50R3JUReHSFr+cwJnL8G1dzTA0YLEqAPI= k8s.io/autoscaler/vertical-pod-autoscaler v1.5.1 h1:LlVtM3IKqIVHz1ZXC3ahe/mAtDWb7Eob0tyTzqFULqg= k8s.io/autoscaler/vertical-pod-autoscaler v1.5.1/go.mod h1:znhUnV0Yn+CkZu3TZ2HVqd8GFRMkPj/CXszX1gdBjTU= -k8s.io/cli-runtime v0.34.1 h1:btlgAgTrYd4sk8vJTRG6zVtqBKt9ZMDeQZo2PIzbL7M= -k8s.io/cli-runtime v0.34.1/go.mod h1:aVA65c+f0MZiMUPbseU/M9l1Wo2byeaGwUuQEQVVveE= +k8s.io/cli-runtime v0.34.2 h1:cct1GEuWc3IyVT8MSCoIWzRGw9HJ/C5rgP32H60H6aE= +k8s.io/cli-runtime v0.34.2/go.mod h1:X13tsrYexYUCIq8MarCBy8lrm0k0weFPTpcaNo7lms4= k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= -k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= -k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= -k8s.io/cluster-bootstrap v0.34.1 h1:lyCwJKoeYzGI93vk5Sn/Gz2rzfTRXkRuZYOk2rUsHfA= -k8s.io/cluster-bootstrap v0.34.1/go.mod h1:9EJfkp7Fu4YBU0F6ysvrI5TndWLo8zufmDSjIWBNd94= +k8s.io/client-go v0.34.2 h1:Co6XiknN+uUZqiddlfAjT68184/37PS4QAzYvQvDR8M= +k8s.io/client-go v0.34.2/go.mod h1:2VYDl1XXJsdcAxw7BenFslRQX28Dxz91U9MWKjX97fE= +k8s.io/cluster-bootstrap v0.34.2 h1:oKckPeunVCns37BntcsxaOesDul32yzGd3DFLjW2fc8= +k8s.io/cluster-bootstrap v0.34.2/go.mod h1:f21byPR7X5nt12ivZi+J3pb4sG4SH6VySX8KAAJA8BY= k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/code-generator v0.34.1 h1:WpphT26E+j7tEgIUfFr5WfbJrktCGzB3JoJH9149xYc= -k8s.io/code-generator v0.34.1/go.mod h1:DeWjekbDnJWRwpw3s0Jat87c+e0TgkxoR4ar608yqvg= -k8s.io/component-base v0.34.1 h1:v7xFgG+ONhytZNFpIz5/kecwD+sUhVE6HU7qQUiRM4A= -k8s.io/component-base v0.34.1/go.mod h1:mknCpLlTSKHzAQJJnnHVKqjxR7gBeHRv0rPXA7gdtQ0= -k8s.io/component-helpers v0.34.1 h1:gWhH3CCdwAx5P3oJqZKb4Lg5FYZTWVbdWtOI8n9U4XY= -k8s.io/component-helpers v0.34.1/go.mod h1:4VgnUH7UA/shuBur+OWoQC0xfb69sy/93ss0ybZqm3c= +k8s.io/code-generator v0.34.2 h1:9bG6jTxmsU3HXE5BNYJTC8AZ1D6hVVfkm8yYSkdkGY0= +k8s.io/code-generator v0.34.2/go.mod h1:dnDDEd6S/z4uZ+PG1aE58ySCi/lR4+qT3a4DddE4/2I= +k8s.io/component-base v0.34.2 h1:HQRqK9x2sSAsd8+R4xxRirlTjowsg6fWCPwWYeSvogQ= +k8s.io/component-base v0.34.2/go.mod h1:9xw2FHJavUHBFpiGkZoKuYZ5pdtLKe97DEByaA+hHbM= +k8s.io/component-helpers v0.34.2 h1:RIUGDdU+QFzeVKLZ9f05sXTNAtJrRJ3bnbMLrogCrvM= +k8s.io/component-helpers v0.34.2/go.mod h1:pLi+GByuRTeFjjcezln8gHL7LcT6HImkwVQ3A2SQaEE= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201203183100-97869a43a9d9/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -1125,23 +1125,23 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kms v0.34.1 h1:iCFOvewDPzWM9fMTfyIPO+4MeuZ0tcZbugxLNSHFG4w= -k8s.io/kms v0.34.1/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= -k8s.io/kube-aggregator v0.34.1 h1:WNLV0dVNoFKmuyvdWLd92iDSyD/TSTjqwaPj0U9XAEU= -k8s.io/kube-aggregator v0.34.1/go.mod h1:RU8j+5ERfp0h+gIvWtxRPfsa5nK7rboDm8RST8BJfYQ= +k8s.io/kms v0.34.2 h1:91rj4MDZLyIT9KxG8J5/CcMH666Z88CF/xJQeuPfJc8= +k8s.io/kms v0.34.2/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= +k8s.io/kube-aggregator v0.34.2 h1:Nn0Vksj67WHBL2x7bJ6vuxL44RbMTK6uRtXX+3vMVJk= +k8s.io/kube-aggregator v0.34.2/go.mod h1:/tp4cc/1p2AvICsS4mjjSJakdrbhcGbRmj0mdHTdR2Q= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 h1:liMHz39T5dJO1aOKHLvwaCjDbf07wVh6yaUlTpunnkE= k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/kube-proxy v0.34.1 h1:cIriNCJY5XmRhXCCyQiazyqi47lbwcBQf0H76fVOpkw= -k8s.io/kube-proxy v0.34.1/go.mod h1:syed9c5+gUVFMo6p24SnlTHzsp+BMd4ACcTw2dbArw0= +k8s.io/kube-proxy v0.34.2 h1:4wDFBTesBtxshw8RcNvwLyoukTVfGaZG7XPo05R5nPw= +k8s.io/kube-proxy v0.34.2/go.mod h1:Hxq2f+IQGwx4plSFaXkqAjkIH6LEhHjNGI0s6hU9G2Y= k8s.io/kube-state-metrics/v2 v2.13.0 h1:g5OV0cwrDKvyrQApYxI7Ny+vFUgU3X6MYrnIzdCAdNU= k8s.io/kube-state-metrics/v2 v2.13.0/go.mod h1:sGt/NFkZkA4hqb4cVd/xG2G17dzZ72TQXqSpHn8rF/U= -k8s.io/kubelet v0.34.1 h1:doAaTA9/Yfzbdq/u/LveZeONp96CwX9giW6b+oHn4m4= -k8s.io/kubelet v0.34.1/go.mod h1:PtV3Ese8iOM19gSooFoQT9iyRisbmJdAPuDImuccbbA= -k8s.io/metrics v0.34.1 h1:374Rexmp1xxgRt64Bi0TsjAM8cA/Y8skwCoPdjtIslE= -k8s.io/metrics v0.34.1/go.mod h1:Drf5kPfk2NJrlpcNdSiAAHn/7Y9KqxpRNagByM7Ei80= -k8s.io/pod-security-admission v0.34.1 h1:XsP5eh8qCj69hK0a5TBMU4Ed7Ckn8JEmmbk/iepj+XM= -k8s.io/pod-security-admission v0.34.1/go.mod h1:87yY36Gxc8Hjx24FxqAD5zMY4k0tP0u7Mu/XuwXEbmg= +k8s.io/kubelet v0.34.2 h1:Dl+1uh7xwJr70r+SHKyIpvu6XvzuoPu0uDIC4cqgJUs= +k8s.io/kubelet v0.34.2/go.mod h1:RfwR03iuKeVV7Z1qD9XKH98c3tlPImJpQ3qHIW40htM= +k8s.io/metrics v0.34.2 h1:zao91FNDVPRGIiHLO2vqqe21zZVPien1goyzn0hsz90= +k8s.io/metrics v0.34.2/go.mod h1:Ydulln+8uZZctUM8yrUQX4rfq/Ay6UzsuXf24QJ37Vc= +k8s.io/pod-security-admission v0.34.2 h1:r77cRPmc2kEPtX2DKh5thmb8zmcFCZhAHUHvVYrjFvA= +k8s.io/pod-security-admission v0.34.2/go.mod h1:lXfDNwD9y0fZM/g1deG7gY/yjED4rcoLrQL2X6BiJgw= k8s.io/sample-controller v0.30.3 h1:oZTxERF8U3gANT2H5VxpkW32asgmW0IYGyUv9Opspvs= k8s.io/sample-controller v0.30.3/go.mod h1:yhy/cWCzevQLa2+7Gvj0J9+xzmNExypunffSNANBy7o= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= From cd8c8499fc6c5015032a53e6ea0e5fbbcf2c2d9d Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 13 Nov 2025 10:09:08 +0100 Subject: [PATCH 104/176] Update dependency protocolbuffers/protobuf to v33.1 (#13440) --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 09166f366d0..14d33f3575d 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -75,7 +75,7 @@ KUSTOMIZE_VERSION ?= v5.3.0 # renovate: datasource=github-releases depName=prometheus/prometheus PROMTOOL_VERSION ?= 3.7.3 # renovate: datasource=github-releases depName=protocolbuffers/protobuf -PROTOC_VERSION ?= v33.0 +PROTOC_VERSION ?= v33.1 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold SKAFFOLD_VERSION ?= v2.16.1 # renovate: datasource=github-releases depName=mikefarah/yq From 76003ccad5769f808c3584f98b6e609c7c7c2675 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Thu, 13 Nov 2025 13:55:25 +0200 Subject: [PATCH 105/176] Move defaulting of the Shoot machine image version from the `ShootValidator` to the `ShootMutator` admission plugin (part 6) (#13351) * Duplicate machine image defaulting in the `ShootMutator` admission plugin * Fix `ShootMutator` unit tests * Duplicate unit tests for machine image defaulting in the `ShootMutator` admission plugin * Clean up machine image defaulting from the `ShootValidator` admission plugin * Add nil check for the machine image when validating API version for RawExtensions * Clean up unit tests for machine image defaulting from the `ShootValidator` admission plugin * Adapt `ShootValidator` unit tests to no longer rely on the machine image defaulting * Update `DescribeTableSubtree` name * Update `ShootMutator` docs * Clean up no longer needed `Context` * Address PR review comments from dimitar-kostadinov --- docs/concepts/apiserver-admission-plugins.md | 4 +- plugin/pkg/shoot/mutator/admission.go | 183 +++++ plugin/pkg/shoot/mutator/admission_test.go | 784 ++++++++++++++++++- plugin/pkg/shoot/validator/admission.go | 192 +---- plugin/pkg/shoot/validator/admission_test.go | 587 +++----------- 5 files changed, 1075 insertions(+), 675 deletions(-) diff --git a/docs/concepts/apiserver-admission-plugins.md b/docs/concepts/apiserver-admission-plugins.md index 24de5b45ef8..b5b10336d62 100644 --- a/docs/concepts/apiserver-admission-plugins.md +++ b/docs/concepts/apiserver-admission-plugins.md @@ -236,7 +236,8 @@ It mutates the `Shoot` in the following way: - It sets the `gardener.cloud/created-by=` annotation for newly created `Shoot` resources. - It maintains annotations used for Shoot lifecycle operations such as `shoot.gardener.cloud/tasks` and `maintenance.shoot.gardener.cloud/needs-retry-operation`. - It defaults Shoot `.spec.networking.pods` and `.spec.networking.services` fields in case they are not provided and the Seed specifies the `.spec.networks.shootDefaults` field. -- It defaults Shoot Kubernetes version (`.spec.kubernetes.version` and `.spec.provider.workers[].kubernetes.version`). +- It defaults the Shoot Kubernetes version (`.spec.kubernetes.version` and `.spec.provider.workers[].kubernetes.version`). +- It defaults the Shoot machine image version (`.spec.provider.workers[].machine.image.{name,version}`). Over time, the `ShootMutator` admission plugin will take over all the mutations that are performed by `ShootValidator`. For more details, see https://github.com/gardener/gardener/issues/2158. @@ -299,7 +300,6 @@ will not be affected by this admission plugin. This admission controller reacts on `CREATE`, `UPDATE` and `DELETE` operations for `Shoot`s. It validates certain configurations in the specification against the referred `CloudProfile` (e.g., machine images, machine types, used Kubernetes version, ...). Generally, it performs validations that cannot be handled by the static API validation due to their dynamic nature (e.g., when something needs to be checked against referred resources). -Additionally, it takes over certain defaulting tasks (e.g., default machine image for worker pools). ## `ValidatingAdmissionPolicy` diff --git a/plugin/pkg/shoot/mutator/admission.go b/plugin/pkg/shoot/mutator/admission.go index e1beb04e5e3..da5bcd75064 100644 --- a/plugin/pkg/shoot/mutator/admission.go +++ b/plugin/pkg/shoot/mutator/admission.go @@ -20,11 +20,13 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/admission" "k8s.io/utils/ptr" + "github.com/gardener/gardener/pkg/api" "github.com/gardener/gardener/pkg/apis/core" "github.com/gardener/gardener/pkg/apis/core/helper" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -181,6 +183,7 @@ func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admissi mutationContext.defaultShootNetworks(helper.IsWorkerless(shoot)) allErrs = append(allErrs, mutationContext.defaultKubernetes()...) allErrs = append(allErrs, mutationContext.defaultKubernetesVersionForWorkers()...) + allErrs = append(allErrs, mutationContext.ensureMachineImages()...) if len(allErrs) > 0 { return admission.NewForbidden(a, allErrs.ToAggregate()) @@ -441,3 +444,183 @@ func (c *mutationContext) defaultKubernetesVersionForWorker(idxPath *field.Path, } return nil } + +func (c *mutationContext) ensureMachineImages() field.ErrorList { + allErrs := field.ErrorList{} + + if c.shoot.DeletionTimestamp == nil { + for idx, worker := range c.shoot.Spec.Provider.Workers { + fldPath := field.NewPath("spec", "provider", "workers").Index(idx) + + image, err := c.ensureMachineImage(c.oldShoot.Spec.Provider.Workers, worker, fldPath) + if err != nil { + allErrs = append(allErrs, err) + continue + } + c.shoot.Spec.Provider.Workers[idx].Machine.Image = image + } + } + + return allErrs +} + +func (c *mutationContext) ensureMachineImage(oldWorkers []core.Worker, worker core.Worker, fldPath *field.Path) (*core.ShootMachineImage, *field.Error) { + // General approach with machine image defaulting in this code: Try to keep the machine image + // from the old shoot object to not accidentally update it to the default machine image. + // This should only happen in the maintenance time window of shoots and is performed by the + // shoot maintenance controller. + machineType := v1beta1helper.FindMachineTypeByName(c.cloudProfileSpec.MachineTypes, worker.Machine.Type) + + oldWorker := helper.FindWorkerByName(oldWorkers, worker.Name) + if oldWorker != nil && oldWorker.Machine.Image != nil { + // worker is already existing -> keep the machine image if name/version is unspecified + if worker.Machine.Image == nil { + // machine image completely unspecified in new worker -> keep the old one + return oldWorker.Machine.Image, nil + } + + if oldWorker.Machine.Image.Name == worker.Machine.Image.Name { + // image name was not changed -> keep version from the new worker if specified, otherwise use the old worker image version + if len(worker.Machine.Image.Version) != 0 { + return getDefaultMachineImage(c.cloudProfileSpec.MachineImages, worker.Machine.Image, worker.Machine.Architecture, machineType, c.cloudProfileSpec.MachineCapabilities, helper.IsUpdateStrategyInPlace(worker.UpdateStrategy), fldPath) + } + return oldWorker.Machine.Image, nil + } else if len(worker.Machine.Image.Version) != 0 { + // image name was changed -> keep version from new worker if specified, otherwise default the image version + return getDefaultMachineImage(c.cloudProfileSpec.MachineImages, worker.Machine.Image, worker.Machine.Architecture, machineType, c.cloudProfileSpec.MachineCapabilities, helper.IsUpdateStrategyInPlace(worker.UpdateStrategy), fldPath) + } + } + + return getDefaultMachineImage(c.cloudProfileSpec.MachineImages, worker.Machine.Image, worker.Machine.Architecture, machineType, c.cloudProfileSpec.MachineCapabilities, helper.IsUpdateStrategyInPlace(worker.UpdateStrategy), fldPath) +} + +// getDefaultMachineImage determines the default machine image version for a shoot. +// It selects the latest non-preview version from the first machine image in the CloudProfile +// that supports the desired architecture and capabilities of the machine type. +func getDefaultMachineImage( + machineImages []gardencorev1beta1.MachineImage, + image *core.ShootMachineImage, + arch *string, + machineType *gardencorev1beta1.MachineType, + capabilityDefinitions []gardencorev1beta1.CapabilityDefinition, + isUpdateStrategyInPlace bool, + fldPath *field.Path, +) (*core.ShootMachineImage, *field.Error) { + var imageReference string + if image != nil { + imageReference = fmt.Sprintf("%s@%s", image.Name, image.Version) + } + + if len(machineImages) == 0 { + return nil, field.Invalid(fldPath, imageReference, "the cloud profile does not contain any machine image - cannot create shoot cluster") + } + + var defaultImage *gardencorev1beta1.MachineImage + + if image != nil && len(image.Name) != 0 { + for _, mi := range machineImages { + machineImage := mi + if machineImage.Name == image.Name { + defaultImage = &machineImage + break + } + } + if defaultImage == nil { + return nil, field.Invalid(fldPath, image.Name, "image is not supported") + } + } else { + // select the first image which supports the required architecture type + for _, mi := range machineImages { + machineImage := mi + for _, version := range machineImage.Versions { + if v1beta1helper.ArchitectureSupportedByImageVersion(version, *arch, capabilityDefinitions) { + defaultImage = &machineImage + break + } + // skip capabilities check if machineType was not found in the cloud profile + if machineType != nil && v1beta1helper.AreCapabilitiesSupportedByImageFlavors(machineType.Capabilities, version.CapabilityFlavors, capabilityDefinitions) { + defaultImage = &machineImage + break + } + } + if defaultImage != nil { + break + } + } + if defaultImage == nil { + return nil, field.Invalid(fldPath, imageReference, fmt.Sprintf("no valid machine image found that supports architecture `%s`", *arch)) + } + } + + var ( + machineImageVersionMajor *uint64 + machineImageVersionMinor *uint64 + ) + + if image != nil { + var err error + versionParts := strings.Split(strings.TrimPrefix(image.Version, "v"), ".") + if len(versionParts) == 3 { + return image, nil + } + if len(versionParts) == 2 && len(versionParts[1]) > 0 { + if machineImageVersionMinor, err = parseSemanticVersionPart(versionParts[1]); err != nil { + return nil, field.Invalid(fldPath, image.Version, err.Error()) + } + } + if len(versionParts) >= 1 && len(versionParts[0]) > 0 { + if machineImageVersionMajor, err = parseSemanticVersionPart(versionParts[0]); err != nil { + return nil, field.Invalid(fldPath, image.Version, err.Error()) + } + } + } + + var validVersions []core.MachineImageVersion + + for _, version := range defaultImage.Versions { + if !v1beta1helper.ArchitectureSupportedByImageVersion(version, *arch, capabilityDefinitions) { + continue + } + + // skip capabilities check if machineType was not found in the cloud profile + if machineType != nil && !v1beta1helper.AreCapabilitiesSupportedByImageFlavors(machineType.Capabilities, version.CapabilityFlavors, capabilityDefinitions) { + continue + } + + // if InPlace update is true, only consider versions that support in-place updates + if isUpdateStrategyInPlace && (version.InPlaceUpdates == nil || !version.InPlaceUpdates.Supported) { + continue + } + + // CloudProfile cannot contain invalid semVer machine image version + parsedVersion := semver.MustParse(version.Version) + if machineImageVersionMajor != nil && parsedVersion.Major() != *machineImageVersionMajor || + machineImageVersionMinor != nil && parsedVersion.Minor() != *machineImageVersionMinor { + continue + } + + var coreVersion core.MachineImageVersion + if err := api.Scheme.Convert(&version, &coreVersion, nil); err != nil { + return nil, field.InternalError(fldPath, fmt.Errorf("failed to convert machine image from cloud profile: %s", err.Error())) + } + validVersions = append(validVersions, coreVersion) + } + + latestMachineImageVersion, err := helper.DetermineLatestMachineImageVersion(validVersions, true) + if err != nil { + return nil, field.Invalid(fldPath, imageReference, fmt.Sprintf("failed to determine latest machine image from cloud profile: %s", err.Error())) + } + var providerConfig *runtime.RawExtension + if image != nil { + providerConfig = image.ProviderConfig + } + return &core.ShootMachineImage{Name: defaultImage.Name, ProviderConfig: providerConfig, Version: latestMachineImageVersion.Version}, nil +} + +func parseSemanticVersionPart(part string) (*uint64, error) { + v, err := strconv.ParseUint(part, 10, 0) + if err != nil { + return nil, fmt.Errorf("%s must be a semantic version: %w", part, err) + } + return ptr.To(v), nil +} diff --git a/plugin/pkg/shoot/mutator/admission_test.go b/plugin/pkg/shoot/mutator/admission_test.go index c26bfe35444..3b10fb2db60 100644 --- a/plugin/pkg/shoot/mutator/admission_test.go +++ b/plugin/pkg/shoot/mutator/admission_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/types" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/admission" @@ -73,7 +74,9 @@ var _ = Describe("mutator", func() { var ( ctx context.Context - userInfo = &user.DefaultInfo{Name: "foo"} + userInfo = &user.DefaultInfo{Name: "foo"} + validMachineImageName = "some-machine-image" + cloudProfile gardencorev1beta1.CloudProfile seed gardencorev1beta1.Seed shoot core.Shoot @@ -90,6 +93,39 @@ var _ = Describe("mutator", func() { ObjectMeta: metav1.ObjectMeta{ Name: "profile", }, + Spec: gardencorev1beta1.CloudProfileSpec{ + MachineImages: []gardencorev1beta1.MachineImage{ + { + Name: validMachineImageName, + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: "0.0.1", + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + Architectures: []string{"amd64", "arm64"}, + }, + }, + }, + }, + MachineTypes: []gardencorev1beta1.MachineType{ + { + Name: "machine-type-1", + CPU: resource.MustParse("2"), + GPU: resource.MustParse("0"), + Memory: resource.MustParse("100Gi"), + Architecture: ptr.To("amd64"), + Usable: ptr.To(true), + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{v1beta1constants.ArchitectureAMD64}, + "someCapability": []string{"value2"}, + }, + }, + }, + }, } seed = gardencorev1beta1.Seed{ ObjectMeta: metav1.ObjectMeta{ @@ -110,6 +146,14 @@ var _ = Describe("mutator", func() { Workers: []core.Worker{ { Name: "worker-name", + Machine: core.Machine{ + Type: "machine-type-1", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + Version: "0.0.1", + }, + Architecture: ptr.To("amd64"), + }, }, }, }, @@ -652,5 +696,743 @@ var _ = Describe("mutator", func() { }) }) }) + + DescribeTableSubtree("machine image", func(isCapabilityCloudProfile bool) { + var ( + classificationPreview = gardencorev1beta1.ClassificationPreview + + imageName1 = validMachineImageName + imageName2 = "other-image" + + expiredVersion = "1.1.1" + expiringVersion = "1.2.1" + nonExpiredVersion = "2.0.0" + latestNonExpiredVersionThatSupportsCapabilities = "2.0.1" + latestNonExpiredVersion = "2.1.0" + previewVersion = "3.0.0" + + cloudProfileMachineImages []gardencorev1beta1.MachineImage + ) + + BeforeEach(func() { + machineCapabilities := []gardencorev1beta1.CapabilityDefinition{ + {Name: "architecture", Values: []string{v1beta1constants.ArchitectureAMD64}}, + {Name: "someCapability", Values: []string{"value1", "value2"}}, + } + cloudProfileMachineImages = []gardencorev1beta1.MachineImage{ + { + Name: validMachineImageName, + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: previewVersion, + Classification: &classificationPreview, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: latestNonExpiredVersion, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{v1beta1constants.ArchitectureAMD64}, + "someCapability": []string{"value1"}, + }, + }, + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{v1beta1constants.ArchitectureARM64}, + "someCapability": []string{"value1"}, + }, + }, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: latestNonExpiredVersionThatSupportsCapabilities, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: nonExpiredVersion, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: expiringVersion, + ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * 1000)}, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: expiredVersion, + ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * -1000)}, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + }, + }, + { + Name: imageName2, + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: previewVersion, + Classification: &classificationPreview, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: latestNonExpiredVersion, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{v1beta1constants.ArchitectureARM64}, + "someCapability": []string{"value1"}, + }, + }, + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{v1beta1constants.ArchitectureAMD64}, + "someCapability": []string{"value1"}, + }, + }, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: latestNonExpiredVersionThatSupportsCapabilities, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: nonExpiredVersion, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: expiringVersion, + ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * 1000)}, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: expiredVersion, + ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * -1000)}, + }, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }, + }, + }, + }, + } + + if !isCapabilityCloudProfile { + machineCapabilities = nil + cloudProfileMachineImages = []gardencorev1beta1.MachineImage{ + { + Name: validMachineImageName, + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: previewVersion, + Classification: &classificationPreview, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: latestNonExpiredVersion, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: nonExpiredVersion, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: latestNonExpiredVersionThatSupportsCapabilities, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: expiringVersion, + ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * 1000)}, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: expiredVersion, + ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * -1000)}, + }, + Architectures: []string{"amd64", "arm64"}, + }, + }, + }, + { + Name: imageName2, + Versions: []gardencorev1beta1.MachineImageVersion{ + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: previewVersion, + Classification: &classificationPreview, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: latestNonExpiredVersion, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: nonExpiredVersion, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: latestNonExpiredVersionThatSupportsCapabilities, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: expiringVersion, + ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * 1000)}, + }, + Architectures: []string{"amd64", "arm64"}, + }, + { + ExpirableVersion: gardencorev1beta1.ExpirableVersion{ + Version: expiredVersion, + ExpirationDate: &metav1.Time{Time: metav1.Now().Add(time.Second * -1000)}, + }, + Architectures: []string{"amd64", "arm64"}, + }, + }, + }, + } + } + + cloudProfile.Spec.MachineCapabilities = machineCapabilities + cloudProfile.Spec.MachineImages = cloudProfileMachineImages + }) + + Context("create Shoot", func() { + BeforeEach(func() { + shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, core.Worker{ + Name: "worker-name-1", + Machine: core.Machine{ + Type: "machine-type-3", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + }, + Architecture: ptr.To("arm64"), + }, + Minimum: 1, + Maximum: 1, + Zones: []string{"europe-a"}, + }) + + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) + }) + + It("should reject due to an invalid machine image (not present in cloudprofile)", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: "not-supported", + Version: "not-supported", + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(BeForbiddenError()) + Expect(err.Error()).To(ContainSubstring("image is not supported")) + }) + + It("should reject due to an invalid machine image (version unset)", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: "not-supported", + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("spec.provider.workers[0]: Invalid value: %q: image is not supported", "not-supported")) + }) + + It("should default version to latest supported non-preview version as shoot does not specify one", func() { + expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities + if !isCapabilityCloudProfile { + expectedImageVersion = latestNonExpiredVersion + } + shoot.Spec.Provider.Workers[0].Machine.Image = nil + shoot.Spec.Provider.Workers[1].Machine.Image = nil + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName1, + Version: expectedImageVersion, + })) + Expect(shoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName1, + Version: latestNonExpiredVersion, + })) + }) + + It("should default version to latest supported non-preview version as shoot only specifies name", func() { + expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities + if !isCapabilityCloudProfile { + expectedImageVersion = latestNonExpiredVersion + } + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + } + shoot.Spec.Provider.Workers[1].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(shoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName1, + Version: expectedImageVersion, + })) + Expect(shoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName1, + Version: latestNonExpiredVersion, + })) + }) + + Context("default machine image version", func() { + var ( + suffixedVersion = "2.1.1-suffixed" + ) + + BeforeEach(func() { + cloudProfile.Spec.MachineImages[0].Versions = append(cloudProfile.Spec.MachineImages[0].Versions, + gardencorev1beta1.MachineImageVersion{ + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: suffixedVersion}, + CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, + {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, + }}, + ) + + if !isCapabilityCloudProfile { + cloudProfile.Spec.MachineImages[0].Versions = append(cloudProfile.Spec.MachineImages[0].Versions, + gardencorev1beta1.MachineImageVersion{ + ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: suffixedVersion}, + Architectures: []string{"amd64", "arm64"}, + }, + ) + } + }) + + It("should throw an error because of an invalid major version", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: "foo", + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("must be a semantic version"))) + }) + + It("should throw an error because of an invalid minor version", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: "1.bar", + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("must be a semantic version"))) + }) + + It("should default a machine image version to latest major.minor.patch version", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + Expect(shoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(suffixedVersion)) + }) + + It("should default a major machine image version to latest minor.patch version", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: "1", + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + Expect(shoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(expiringVersion)) + }) + + It("should default a major.minor machine image version to latest supported patch version", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: "2.0", + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + Expect(shoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(latestNonExpiredVersionThatSupportsCapabilities)) + }) + + It("should reject defaulting a major.minor machine image version if there is no higher non-preview version available for defaulting", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: "3", + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("failed to determine latest machine image from cloud profile"))) + }) + + It("should be able to explicitly pick preview versions", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: "3.0.0", + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + Expect(shoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(previewVersion)) + }) + + It("should reject: default only exactly matching minor machine image version", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: "1.0", + } + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("failed to determine latest machine image from cloud profile"))) + }) + + It("should reject defaulting a machine image version for worker pool with inplace update strategy if there is no machine image available in the cloud profile supporting inplace update", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + } + shoot.Spec.Provider.Workers[0].UpdateStrategy = ptr.To(core.AutoInPlaceUpdate) + + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("failed to determine latest machine image from cloud profile"))) + }) + }) + }) + + Context("update Shoot", func() { + BeforeEach(func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: nonExpiredVersion, + } + shoot.Spec.Provider.Workers[0].Machine.Architecture = ptr.To("amd64") + + Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) + }) + + It("should keep machine image of the old shoot (unset in new shoot)", func() { + newShoot := shoot.DeepCopy() + newShoot.Spec.Provider.Workers[0].Machine.Image = nil + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(shoot.Spec.Provider.Workers[0].Machine.Image)) + }) + + It("should keep machine image of the old shoot (version unset in new shoot)", func() { + newShoot := shoot.DeepCopy() + newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + } + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(shoot.Spec.Provider.Workers[0].Machine.Image)) + }) + + It("should use updated machine image version as specified", func() { + newShoot := shoot.DeepCopy() + newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: latestNonExpiredVersionThatSupportsCapabilities, + } + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName1, + Version: latestNonExpiredVersionThatSupportsCapabilities, + })) + }) + + It("should default a version prefix of an existing worker pool to the latest supported non-preview version", func() { + expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities + if !isCapabilityCloudProfile { + expectedImageVersion = latestNonExpiredVersion + } + newShoot := shoot.DeepCopy() + newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: "2", + } + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + Expect(newShoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(expectedImageVersion)) + }) + + It("should default a version prefix of a new image of an existing worker pool to the latest supported non-preview version", func() { + newShoot := shoot.DeepCopy() + newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName2, + Version: "2.0", + } + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(Not(HaveOccurred())) + Expect(newShoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(latestNonExpiredVersionThatSupportsCapabilities)) + }) + + It("should default version of new worker pool to latest supported non-preview version", func() { + expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities + if !isCapabilityCloudProfile { + expectedImageVersion = latestNonExpiredVersion + } + newShoot := shoot.DeepCopy() + newWorker := newShoot.Spec.Provider.Workers[0].DeepCopy() + newWorker2 := newShoot.Spec.Provider.Workers[0].DeepCopy() + newWorker.Name = "second-worker" + newWorker2.Name = "third-worker" + newWorker.Machine.Image = nil + newWorker2.Machine.Image = nil + newWorker2.Machine.Type = "machine-type-3" + newWorker2.Machine.Architecture = ptr.To("arm64") + newShoot.Spec.Provider.Workers = append(newShoot.Spec.Provider.Workers, *newWorker, *newWorker2) + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(newShoot.Spec.Provider.Workers[0]).To(Equal(shoot.Spec.Provider.Workers[0])) + Expect(newShoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName1, + Version: expectedImageVersion, + })) + Expect(newShoot.Spec.Provider.Workers[2].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName1, + Version: latestNonExpiredVersion, + })) + }) + + It("should default version of new worker pool to latest supported non-preview version (version unset)", func() { + expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities + if !isCapabilityCloudProfile { + expectedImageVersion = latestNonExpiredVersion + } + + newShoot := shoot.DeepCopy() + newWorker := newShoot.Spec.Provider.Workers[0].DeepCopy() + newWorker2 := newShoot.Spec.Provider.Workers[0].DeepCopy() + newWorker.Name = "second-worker" + newWorker2.Name = "third-worker" + newWorker2.Machine.Type = "machine-type-3" + newWorker2.Machine.Architecture = ptr.To("arm64") + newWorker.Machine.Image = &core.ShootMachineImage{ + Name: imageName2, + } + newWorker2.Machine.Image = &core.ShootMachineImage{ + Name: imageName2, + } + newShoot.Spec.Provider.Workers = append(newShoot.Spec.Provider.Workers, *newWorker, *newWorker2) + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(newShoot.Spec.Provider.Workers[0]).To(Equal(shoot.Spec.Provider.Workers[0])) + Expect(newShoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName2, + Version: expectedImageVersion, + })) + Expect(newShoot.Spec.Provider.Workers[2].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName2, + Version: latestNonExpiredVersion, + })) + }) + + It("should default version of worker pool to latest supported non-preview version when machine architecture is changed", func() { + newShoot := shoot.DeepCopy() + newShoot.Spec.Provider.Workers[0].Machine.Type = "machine-type-3" + newShoot.Spec.Provider.Workers[0].Machine.Image = nil + newShoot.Spec.Provider.Workers[0].Machine.Architecture = ptr.To("arm64") + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName1, + Version: nonExpiredVersion, + })) + }) + + It("should use version of new worker pool as specified", func() { + newShoot := shoot.DeepCopy() + newWorker := newShoot.Spec.Provider.Workers[0].DeepCopy() + newWorker.Name = "second-worker" + newWorker.Machine.Image = &core.ShootMachineImage{ + Name: imageName2, + Version: nonExpiredVersion, + } + newShoot.Spec.Provider.Workers = append(newShoot.Spec.Provider.Workers, *newWorker) + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(newShoot.Spec.Provider.Workers[0]).To(Equal(shoot.Spec.Provider.Workers[0])) + Expect(newShoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName2, + Version: nonExpiredVersion, + })) + }) + + It("should default version of new image to latest supported non-preview version (version unset)", func() { + expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities + if !isCapabilityCloudProfile { + expectedImageVersion = latestNonExpiredVersion + } + + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: latestNonExpiredVersionThatSupportsCapabilities, + } + + newShoot := shoot.DeepCopy() + newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName2, + } + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName2, + Version: expectedImageVersion, + })) + }) + + It("should use version of new image as specified", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: latestNonExpiredVersionThatSupportsCapabilities, + } + + newShoot := shoot.DeepCopy() + newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName2, + Version: latestNonExpiredVersionThatSupportsCapabilities, + } + + attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ + Name: imageName2, + Version: latestNonExpiredVersionThatSupportsCapabilities, + })) + }) + }) + + }, + Entry("Cloudprofile is using Capabilities", true), + Entry("Cloudprofile is NOT using Capabilities", false), + ) }) }) diff --git a/plugin/pkg/shoot/validator/admission.go b/plugin/pkg/shoot/validator/admission.go index 584001dc94a..f9d4e80cfcd 100644 --- a/plugin/pkg/shoot/validator/admission.go +++ b/plugin/pkg/shoot/validator/admission.go @@ -11,7 +11,6 @@ import ( "io" "reflect" "slices" - "strconv" "strings" "github.com/Masterminds/semver/v3" @@ -31,7 +30,6 @@ import ( kubecorev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/utils/ptr" - "github.com/gardener/gardener/pkg/api" "github.com/gardener/gardener/pkg/apis/core" "github.com/gardener/gardener/pkg/apis/core/helper" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -331,9 +329,6 @@ func (v *ValidateShoot) Admit(ctx context.Context, a admission.Attributes, _ adm if err := validationContext.validateCredentialsBindingChange(ctx, a, v.authorizer, v.credentialsBindingLister); err != nil { return err } - if allErrs = validationContext.ensureMachineImages(); len(allErrs) > 0 { - return admission.NewForbidden(a, allErrs.ToAggregate()) - } allErrs = append(allErrs, validationContext.validateAPIVersionForRawExtensions()...) allErrs = append(allErrs, validationContext.validateShootNetworks(a, helper.IsWorkerless(shoot))...) @@ -753,25 +748,6 @@ func (c *validationContext) validateCredentialsBindingChange( return nil } -func (c *validationContext) ensureMachineImages() field.ErrorList { - allErrs := field.ErrorList{} - - if c.shoot.DeletionTimestamp == nil { - for idx, worker := range c.shoot.Spec.Provider.Workers { - fldPath := field.NewPath("spec", "provider", "workers").Index(idx) - - image, err := c.ensureMachineImage(c.oldShoot.Spec.Provider.Workers, worker, fldPath) - if err != nil { - allErrs = append(allErrs, err) - continue - } - c.shoot.Spec.Provider.Workers[idx].Machine.Image = image - } - } - - return allErrs -} - func (c *validationContext) validateAdmissionPlugins(a admission.Attributes, secretLister kubecorev1listers.SecretLister) field.ErrorList { var ( allErrs field.ErrorList @@ -1172,9 +1148,10 @@ func (c *validationContext) validateAPIVersionForRawExtensions() field.ErrorList if ok, gvk := usesInternalVersion(worker.ProviderConfig); ok { allErrs = append(allErrs, field.Invalid(workerPath.Child("providerConfig"), gvk, internalVersionErrorMsg)) } - - if ok, gvk := usesInternalVersion(worker.Machine.Image.ProviderConfig); ok { - allErrs = append(allErrs, field.Invalid(workerPath.Child("machine", "image", "providerConfig"), gvk, internalVersionErrorMsg)) + if worker.Machine.Image != nil { + if ok, gvk := usesInternalVersion(worker.Machine.Image.ProviderConfig); ok { + allErrs = append(allErrs, field.Invalid(workerPath.Child("machine", "image", "providerConfig"), gvk, internalVersionErrorMsg)) + } } if worker.CRI != nil && worker.CRI.ContainerRuntimes != nil { @@ -1605,137 +1582,6 @@ func validateZone(constraints []gardencorev1beta1.Region, region, zone string) ( return false, validValues } -// getDefaultMachineImage determines the default machine image version for a shoot. -// It selects the latest non-preview version from the first machine image in the CloudProfile -// that supports the desired architecture and capabilities of the machine type. -func getDefaultMachineImage( - machineImages []gardencorev1beta1.MachineImage, - image *core.ShootMachineImage, - arch *string, - machineType *gardencorev1beta1.MachineType, - capabilityDefinitions []gardencorev1beta1.CapabilityDefinition, - isUpdateStrategyInPlace bool, - fldPath *field.Path, -) (*core.ShootMachineImage, *field.Error) { - var imageReference string - if image != nil { - imageReference = fmt.Sprintf("%s@%s", image.Name, image.Version) - } - - if len(machineImages) == 0 { - return nil, field.Invalid(fldPath, imageReference, "the cloud profile does not contain any machine image - cannot create shoot cluster") - } - - var defaultImage *gardencorev1beta1.MachineImage - - if image != nil && len(image.Name) != 0 { - for _, mi := range machineImages { - machineImage := mi - if machineImage.Name == image.Name { - defaultImage = &machineImage - break - } - } - if defaultImage == nil { - return nil, field.Invalid(fldPath, image.Name, "image is not supported") - } - } else { - // select the first image which supports the required architecture type - for _, mi := range machineImages { - machineImage := mi - for _, version := range machineImage.Versions { - if v1beta1helper.ArchitectureSupportedByImageVersion(version, *arch, capabilityDefinitions) { - defaultImage = &machineImage - break - } - // skip capabilities check if machineType was not found in the cloud profile - if machineType != nil && v1beta1helper.AreCapabilitiesSupportedByImageFlavors(machineType.Capabilities, version.CapabilityFlavors, capabilityDefinitions) { - defaultImage = &machineImage - break - } - } - if defaultImage != nil { - break - } - } - if defaultImage == nil { - return nil, field.Invalid(fldPath, imageReference, fmt.Sprintf("no valid machine image found that supports architecture `%s`", *arch)) - } - } - - var ( - machineImageVersionMajor *uint64 - machineImageVersionMinor *uint64 - ) - - if image != nil { - var err error - versionParts := strings.Split(strings.TrimPrefix(image.Version, "v"), ".") - if len(versionParts) == 3 { - return image, nil - } - if len(versionParts) == 2 && len(versionParts[1]) > 0 { - if machineImageVersionMinor, err = parseSemanticVersionPart(versionParts[1]); err != nil { - return nil, field.Invalid(fldPath, image.Version, err.Error()) - } - } - if len(versionParts) >= 1 && len(versionParts[0]) > 0 { - if machineImageVersionMajor, err = parseSemanticVersionPart(versionParts[0]); err != nil { - return nil, field.Invalid(fldPath, image.Version, err.Error()) - } - } - } - - var validVersions []core.MachineImageVersion - - for _, version := range defaultImage.Versions { - if !v1beta1helper.ArchitectureSupportedByImageVersion(version, *arch, capabilityDefinitions) { - continue - } - - // skip capabilities check if machineType was not found in the cloud profile - if machineType != nil && !v1beta1helper.AreCapabilitiesSupportedByImageFlavors(machineType.Capabilities, version.CapabilityFlavors, capabilityDefinitions) { - continue - } - - // if InPlace update is true, only consider versions that support in-place updates - if isUpdateStrategyInPlace && (version.InPlaceUpdates == nil || !version.InPlaceUpdates.Supported) { - continue - } - - // CloudProfile cannot contain invalid semVer machine image version - parsedVersion := semver.MustParse(version.Version) - if machineImageVersionMajor != nil && parsedVersion.Major() != *machineImageVersionMajor || - machineImageVersionMinor != nil && parsedVersion.Minor() != *machineImageVersionMinor { - continue - } - - var coreVersion core.MachineImageVersion - if err := api.Scheme.Convert(&version, &coreVersion, nil); err != nil { - return nil, field.InternalError(fldPath, fmt.Errorf("failed to convert machine image from cloud profile: %s", err.Error())) - } - validVersions = append(validVersions, coreVersion) - } - - latestMachineImageVersion, err := helper.DetermineLatestMachineImageVersion(validVersions, true) - if err != nil { - return nil, field.Invalid(fldPath, imageReference, fmt.Sprintf("failed to determine latest machine image from cloud profile: %s", err.Error())) - } - var providerConfig *runtime.RawExtension - if image != nil { - providerConfig = image.ProviderConfig - } - return &core.ShootMachineImage{Name: defaultImage.Name, ProviderConfig: providerConfig, Version: latestMachineImageVersion.Version}, nil -} - -func parseSemanticVersionPart(part string) (*uint64, error) { - v, err := strconv.ParseUint(part, 10, 0) - if err != nil { - return nil, fmt.Errorf("%s must be a semantic version: %w", part, err) - } - return ptr.To(v), nil -} - func validateMachineImagesConstraints(a admission.Attributes, constraints []gardencorev1beta1.MachineImage, isNewWorkerPool, isUpdateStrategyInPlace bool, machine, oldMachine core.Machine, capabilityDefinitions []gardencorev1beta1.CapabilityDefinition) (bool, bool, bool, bool, []string) { if apiequality.Semantic.DeepEqual(machine.Image, oldMachine.Image) && ptr.Equal(machine.Architecture, oldMachine.Architecture) { return true, true, true, true, nil @@ -1905,36 +1751,6 @@ func validateKubeletVersion(constraints []gardencorev1beta1.MachineImage, worker return nil } -func (c *validationContext) ensureMachineImage(oldWorkers []core.Worker, worker core.Worker, fldPath *field.Path) (*core.ShootMachineImage, *field.Error) { - // General approach with machine image defaulting in this code: Try to keep the machine image - // from the old shoot object to not accidentally update it to the default machine image. - // This should only happen in the maintenance time window of shoots and is performed by the - // shoot maintenance controller. - machineType := v1beta1helper.FindMachineTypeByName(c.cloudProfileSpec.MachineTypes, worker.Machine.Type) - - oldWorker := helper.FindWorkerByName(oldWorkers, worker.Name) - if oldWorker != nil && oldWorker.Machine.Image != nil { - // worker is already existing -> keep the machine image if name/version is unspecified - if worker.Machine.Image == nil { - // machine image completely unspecified in new worker -> keep the old one - return oldWorker.Machine.Image, nil - } - - if oldWorker.Machine.Image.Name == worker.Machine.Image.Name { - // image name was not changed -> keep version from the new worker if specified, otherwise use the old worker image version - if len(worker.Machine.Image.Version) != 0 { - return getDefaultMachineImage(c.cloudProfileSpec.MachineImages, worker.Machine.Image, worker.Machine.Architecture, machineType, c.cloudProfileSpec.MachineCapabilities, helper.IsUpdateStrategyInPlace(worker.UpdateStrategy), fldPath) - } - return oldWorker.Machine.Image, nil - } else if len(worker.Machine.Image.Version) != 0 { - // image name was changed -> keep version from new worker if specified, otherwise default the image version - return getDefaultMachineImage(c.cloudProfileSpec.MachineImages, worker.Machine.Image, worker.Machine.Architecture, machineType, c.cloudProfileSpec.MachineCapabilities, helper.IsUpdateStrategyInPlace(worker.UpdateStrategy), fldPath) - } - } - - return getDefaultMachineImage(c.cloudProfileSpec.MachineImages, worker.Machine.Image, worker.Machine.Architecture, machineType, c.cloudProfileSpec.MachineCapabilities, helper.IsUpdateStrategyInPlace(worker.UpdateStrategy), fldPath) -} - // wasShootRescheduledToNewSeed returns true if the shoot.Spec.SeedName has been changed, but the migration operation has not started yet. func wasShootRescheduledToNewSeed(shoot *core.Shoot) bool { return shoot.Status.LastOperation != nil && diff --git a/plugin/pkg/shoot/validator/admission_test.go b/plugin/pkg/shoot/validator/admission_test.go index dba039c9128..a317e2fa7f2 100644 --- a/plugin/pkg/shoot/validator/admission_test.go +++ b/plugin/pkg/shoot/validator/admission_test.go @@ -293,7 +293,8 @@ var _ = Describe("validator", func() { Machine: core.Machine{ Type: "machine-type-1", Image: &core.ShootMachineImage{ - Name: validMachineImageName, + Name: validMachineImageName, + Version: "0.0.1", }, Architecture: ptr.To("amd64"), }, @@ -2936,7 +2937,8 @@ var _ = Describe("validator", func() { Machine: core.Machine{ Type: "machine-type-kc", Image: &core.ShootMachineImage{ - Name: validMachineImageName, + Name: validMachineImageName, + Version: "0.0.1", }, Architecture: ptr.To("amd64"), }, @@ -3288,15 +3290,20 @@ var _ = Describe("validator", func() { Version: latestNonExpiredVersion, }, CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ - {Capabilities: gardencorev1beta1.Capabilities{ - "architecture": []string{v1beta1constants.ArchitectureAMD64}, - "someCapability": []string{"value1"}, - }}, - {Capabilities: gardencorev1beta1.Capabilities{ - "architecture": []string{v1beta1constants.ArchitectureARM64}, - "someCapability": []string{"value1"}, - }}, - }}, + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{v1beta1constants.ArchitectureAMD64}, + "someCapability": []string{"value1"}, + }, + }, + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{v1beta1constants.ArchitectureARM64}, + "someCapability": []string{"value1"}, + }, + }, + }, + }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{ Version: latestNonExpiredVersionThatSupportsCapabilities, @@ -3304,7 +3311,8 @@ var _ = Describe("validator", func() { CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, + }, + }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{ Version: nonExpiredVersion, @@ -3312,7 +3320,8 @@ var _ = Describe("validator", func() { CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, + }, + }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{ Version: expiringVersion, @@ -3321,7 +3330,8 @@ var _ = Describe("validator", func() { CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, + }, + }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{ Version: expiredVersion, @@ -3330,9 +3340,11 @@ var _ = Describe("validator", func() { CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, + }, + }, }, - }, { + }, + { Name: imageName2, Versions: []gardencorev1beta1.MachineImageVersion{ { @@ -3343,21 +3355,27 @@ var _ = Describe("validator", func() { CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, + }, + }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{ Version: latestNonExpiredVersion, }, CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ - {Capabilities: gardencorev1beta1.Capabilities{ - "architecture": []string{v1beta1constants.ArchitectureARM64}, - "someCapability": []string{"value1"}, - }}, - {Capabilities: gardencorev1beta1.Capabilities{ - "architecture": []string{v1beta1constants.ArchitectureAMD64}, - "someCapability": []string{"value1"}, - }}, - }}, + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{v1beta1constants.ArchitectureARM64}, + "someCapability": []string{"value1"}, + }, + }, + { + Capabilities: gardencorev1beta1.Capabilities{ + "architecture": []string{v1beta1constants.ArchitectureAMD64}, + "someCapability": []string{"value1"}, + }, + }, + }, + }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{ Version: latestNonExpiredVersionThatSupportsCapabilities, @@ -3365,7 +3383,8 @@ var _ = Describe("validator", func() { CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, + }, + }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{ Version: nonExpiredVersion, @@ -3373,7 +3392,8 @@ var _ = Describe("validator", func() { CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, + }, + }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{ Version: expiringVersion, @@ -3382,7 +3402,8 @@ var _ = Describe("validator", func() { CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, + }, + }, { ExpirableVersion: gardencorev1beta1.ExpirableVersion{ Version: expiredVersion, @@ -3391,7 +3412,8 @@ var _ = Describe("validator", func() { CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, + }, + }, }, }, } @@ -3441,7 +3463,8 @@ var _ = Describe("validator", func() { Architectures: []string{"amd64", "arm64"}, }, }, - }, { + }, + { Name: imageName2, Versions: []gardencorev1beta1.MachineImageVersion{ { @@ -3517,31 +3540,6 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) }) - It("should reject due to an invalid machine image (not present in cloudprofile)", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: "not-supported", - Version: "not-supported", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(BeForbiddenError()) - Expect(err.Error()).To(ContainSubstring("image is not supported")) - }) - - It("should reject due to an invalid machine image (version unset)", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: "not-supported", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("spec.provider.workers[0]: Invalid value: %q: image is not supported", "not-supported")) - }) - It("should reject due to a machine image with expiration date in the past", func() { shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ Name: validMachineImageName, @@ -3967,211 +3965,28 @@ var _ = Describe("validator", func() { Expect(err.Error()).To(ContainSubstring("machine image 'constraint-image-name@1.2.3' does not support kubelet version '1.25.0', supported kubelet versions by this machine image version: '>= 1.26'")) }) - It("should default version to latest supported non-preview version as shoot does not specify one", func() { - expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities - if !isCapabilityCloudProfile { - expectedImageVersion = latestNonExpiredVersion + It("should throw an error because of an invalid patch version", func() { + shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ + Name: imageName1, + Version: "1.2.baz", } - shoot.Spec.Provider.Workers[0].Machine.Image = nil - shoot.Spec.Provider.Workers[1].Machine.Image = nil attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) err := admissionHandler.Admit(ctx, attrs, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(shoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName1, - Version: expectedImageVersion, - })) - Expect(shoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName1, - Version: latestNonExpiredVersion, - })) + Expect(err).To(MatchError(ContainSubstring("Unsupported value: \"1.2.baz\": supported values:"))) }) - It("should default version to latest supported non-preview version as shoot only specifies name", func() { - expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities - if !isCapabilityCloudProfile { - expectedImageVersion = latestNonExpiredVersion - } + It("should reject to create a worker group with an expired machine image version", func() { shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - } - shoot.Spec.Provider.Workers[1].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, + Name: imageName1, + Version: "1.1.1", } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) err := admissionHandler.Admit(ctx, attrs, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(shoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName1, - Version: expectedImageVersion, - })) - Expect(shoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName1, - Version: latestNonExpiredVersion, - })) - }) - - Context("default machine image version", func() { - var ( - suffixedVersion = "2.1.1-suffixed" - ) - - BeforeEach(func() { - cloudProfile.Spec.MachineImages[0].Versions = append(cloudProfile.Spec.MachineImages[0].Versions, - gardencorev1beta1.MachineImageVersion{ - ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: suffixedVersion}, - CapabilityFlavors: []gardencorev1beta1.MachineImageFlavor{ - {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureAMD64}}}, - {Capabilities: gardencorev1beta1.Capabilities{"architecture": []string{v1beta1constants.ArchitectureARM64}}}, - }}, - ) - - if !isCapabilityCloudProfile { - cloudProfile.Spec.MachineImages[0].Versions = append(cloudProfile.Spec.MachineImages[0].Versions, - gardencorev1beta1.MachineImageVersion{ - ExpirableVersion: gardencorev1beta1.ExpirableVersion{Version: suffixedVersion}, - Architectures: []string{"amd64", "arm64"}, - }, - ) - } - }) - - It("should throw an error because of an invalid major version", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "foo", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("must be a semantic version"))) - }) - - It("should throw an error because of an invalid minor version", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "1.bar", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("must be a semantic version"))) - }) - - It("should throw an error because of an invalid patch version", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "1.2.baz", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("Unsupported value: \"1.2.baz\": supported values:"))) - }) - - It("should default a machine image version to latest major.minor.patch version", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - Expect(shoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(suffixedVersion)) - }) - - It("should default a major machine image version to latest minor.patch version", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "1", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - Expect(shoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(expiringVersion)) - }) - - It("should default a major.minor machine image version to latest supported patch version", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "2.0", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - Expect(shoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(latestNonExpiredVersionThatSupportsCapabilities)) - }) - - It("should reject defaulting a major.minor machine image version if there is no higher non-preview version available for defaulting", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "3", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("failed to determine latest machine image from cloud profile"))) - }) - - It("should be able to explicitly pick preview versions", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "3.0.0", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - Expect(shoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(previewVersion)) - }) - - It("should reject: default only exactly matching minor machine image version", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "1.0", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("failed to determine latest machine image from cloud profile"))) - }) - - It("should reject to create a worker group with an expired machine image version", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "1.1.1", - } - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("machine image version 'some-machine-image:%s' is expired", expiredVersion))) - }) - - It("should reject defaulting a machine image version for worker pool with inplace update strategy if there is no machine image available in the cloud profile supporting inplace update", func() { - shoot.Spec.Provider.Workers[0].UpdateStrategy = ptr.To(core.AutoInPlaceUpdate) - - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("failed to determine latest machine image from cloud profile"))) - - }) + Expect(err).To(MatchError(ContainSubstring("machine image version 'some-machine-image:%s' is expired", expiredVersion))) }) It("should allow supported CRI and CRs", func() { @@ -5027,234 +4842,6 @@ var _ = Describe("validator", func() { Expect(err).NotTo(HaveOccurred()) Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{Name: "constraint-image-name", Version: "1.2.4"})) }) - - It("should keep machine image of the old shoot (unset in new shoot)", func() { - newShoot := shoot.DeepCopy() - newShoot.Spec.Provider.Workers[0].Machine.Image = nil - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(shoot.Spec.Provider.Workers[0].Machine.Image)) - }) - - It("should keep machine image of the old shoot (version unset in new shoot)", func() { - newShoot := shoot.DeepCopy() - newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - } - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(shoot.Spec.Provider.Workers[0].Machine.Image)) - }) - - It("should use updated machine image version as specified", func() { - newShoot := shoot.DeepCopy() - newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: latestNonExpiredVersionThatSupportsCapabilities, - } - - Expect(coreInformerFactory.Core().V1beta1().Projects().Informer().GetStore().Add(&project)).To(Succeed()) - Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) - Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName1, - Version: latestNonExpiredVersionThatSupportsCapabilities, - })) - }) - - It("should default a version prefix of an existing worker pool to the latest supported non-preview version", func() { - expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities - if !isCapabilityCloudProfile { - expectedImageVersion = latestNonExpiredVersion - } - newShoot := shoot.DeepCopy() - newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: "2", - } - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - Expect(newShoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(expectedImageVersion)) - }) - - It("should default a version prefix of a new image of an existing worker pool to the latest supported non-preview version", func() { - newShoot := shoot.DeepCopy() - newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName2, - Version: "2.0", - } - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(Not(HaveOccurred())) - Expect(newShoot.Spec.Provider.Workers[0].Machine.Image.Version).To(Equal(latestNonExpiredVersionThatSupportsCapabilities)) - }) - - It("should default version of new worker pool to latest supported non-preview version", func() { - expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities - if !isCapabilityCloudProfile { - expectedImageVersion = latestNonExpiredVersion - } - newShoot := shoot.DeepCopy() - newWorker := newShoot.Spec.Provider.Workers[0].DeepCopy() - newWorker2 := newShoot.Spec.Provider.Workers[0].DeepCopy() - newWorker.Name = "second-worker" - newWorker2.Name = "third-worker" - newWorker.Machine.Image = nil - newWorker2.Machine.Image = nil - newWorker2.Machine.Type = "machine-type-3" - newWorker2.Machine.Architecture = ptr.To("arm64") - newShoot.Spec.Provider.Workers = append(newShoot.Spec.Provider.Workers, *newWorker, *newWorker2) - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(newShoot.Spec.Provider.Workers[0]).To(Equal(shoot.Spec.Provider.Workers[0])) - Expect(newShoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName1, - Version: expectedImageVersion, - })) - Expect(newShoot.Spec.Provider.Workers[2].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName1, - Version: latestNonExpiredVersion, - })) - }) - - It("should default version of new worker pool to latest supported non-preview version (version unset)", func() { - expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities - if !isCapabilityCloudProfile { - expectedImageVersion = latestNonExpiredVersion - } - - newShoot := shoot.DeepCopy() - newWorker := newShoot.Spec.Provider.Workers[0].DeepCopy() - newWorker2 := newShoot.Spec.Provider.Workers[0].DeepCopy() - newWorker.Name = "second-worker" - newWorker2.Name = "third-worker" - newWorker2.Machine.Type = "machine-type-3" - newWorker2.Machine.Architecture = ptr.To("arm64") - newWorker.Machine.Image = &core.ShootMachineImage{ - Name: imageName2, - } - newWorker2.Machine.Image = &core.ShootMachineImage{ - Name: imageName2, - } - newShoot.Spec.Provider.Workers = append(newShoot.Spec.Provider.Workers, *newWorker, *newWorker2) - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(newShoot.Spec.Provider.Workers[0]).To(Equal(shoot.Spec.Provider.Workers[0])) - Expect(newShoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName2, - Version: expectedImageVersion, - })) - Expect(newShoot.Spec.Provider.Workers[2].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName2, - Version: latestNonExpiredVersion, - })) - }) - - It("should default version of worker pool to latest supported non-preview version when machine architecture is changed", func() { - newShoot := shoot.DeepCopy() - newShoot.Spec.Provider.Workers[0].Machine.Type = "machine-type-3" - newShoot.Spec.Provider.Workers[0].Machine.Image = nil - newShoot.Spec.Provider.Workers[0].Machine.Architecture = ptr.To("arm64") - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName1, - Version: nonExpiredVersion, - })) - }) - - It("should use version of new worker pool as specified", func() { - newShoot := shoot.DeepCopy() - newWorker := newShoot.Spec.Provider.Workers[0].DeepCopy() - newWorker.Name = "second-worker" - newWorker.Machine.Image = &core.ShootMachineImage{ - Name: imageName2, - Version: nonExpiredVersion, - } - newShoot.Spec.Provider.Workers = append(newShoot.Spec.Provider.Workers, *newWorker) - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(newShoot.Spec.Provider.Workers[0]).To(Equal(shoot.Spec.Provider.Workers[0])) - Expect(newShoot.Spec.Provider.Workers[1].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName2, - Version: nonExpiredVersion, - })) - }) - - It("should default version of new image to latest supported non-preview version (version unset)", func() { - expectedImageVersion := latestNonExpiredVersionThatSupportsCapabilities - if !isCapabilityCloudProfile { - expectedImageVersion = latestNonExpiredVersion - } - - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: latestNonExpiredVersionThatSupportsCapabilities, - } - - newShoot := shoot.DeepCopy() - newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName2, - } - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName2, - Version: expectedImageVersion, - })) - }) - - It("should use version of new image as specified", func() { - shoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName1, - Version: latestNonExpiredVersionThatSupportsCapabilities, - } - - newShoot := shoot.DeepCopy() - newShoot.Spec.Provider.Workers[0].Machine.Image = &core.ShootMachineImage{ - Name: imageName2, - Version: latestNonExpiredVersionThatSupportsCapabilities, - } - - attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), newShoot.Namespace, newShoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{ - Name: imageName2, - Version: latestNonExpiredVersionThatSupportsCapabilities, - })) - }) }) Context("delete Shoot", func() { @@ -5395,7 +4982,11 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = []core.Worker{ { Machine: core.Machine{ - Type: "machine-type-1", + Type: "machine-type-1", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + Version: "0.0.1", + }, Architecture: ptr.To("amd64"), }, }, @@ -5554,7 +5145,11 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = []core.Worker{ { Machine: core.Machine{ - Type: "machine-type-1", + Type: "machine-type-1", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + Version: "0.0.1", + }, Architecture: ptr.To("amd64"), }, Volume: &core.Volume{ @@ -5601,7 +5196,11 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = []core.Worker{ { Machine: core.Machine{ - Type: "machine-type-1", + Type: "machine-type-1", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + Version: "0.0.1", + }, Architecture: ptr.To("amd64"), }, Volume: &core.Volume{ @@ -5649,7 +5248,11 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = []core.Worker{ { Machine: core.Machine{ - Type: "machine-type-1", + Type: "machine-type-1", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + Version: "0.0.1", + }, Architecture: ptr.To("amd64"), }, Volume: &core.Volume{ @@ -5684,7 +5287,11 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = []core.Worker{ { Machine: core.Machine{ - Type: "machine-type-1", + Type: "machine-type-1", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + Version: "0.0.1", + }, Architecture: ptr.To("amd64"), }, Volume: &core.Volume{ @@ -5721,7 +5328,11 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = []core.Worker{ { Machine: core.Machine{ - Type: "machine-type-1", + Type: "machine-type-1", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + Version: "0.0.1", + }, Architecture: ptr.To("amd64"), }, Volume: &core.Volume{ @@ -5731,7 +5342,11 @@ var _ = Describe("validator", func() { }, { Machine: core.Machine{ - Type: "machine-type-2", + Type: "machine-type-2", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + Version: "0.0.1", + }, Architecture: ptr.To("amd64"), }, Volume: &core.Volume{ @@ -5741,7 +5356,11 @@ var _ = Describe("validator", func() { }, { Machine: core.Machine{ - Type: "machine-type-2", + Type: "machine-type-2", + Image: &core.ShootMachineImage{ + Name: validMachineImageName, + Version: "0.0.1", + }, Architecture: ptr.To("amd64"), }, Volume: &core.Volume{ From bb0fca73e5fa235b134d9f834fd7d06b43a12adc Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 13 Nov 2025 12:55:32 +0100 Subject: [PATCH 106/176] Update module golang.org/x/tools to v0.39.0 (#13442) --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 54103ec0d4c..44167fb70e0 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 golang.org/x/text v0.31.0 golang.org/x/time v0.14.0 - golang.org/x/tools v0.38.0 + golang.org/x/tools v0.39.0 gomodules.xyz/jsonpatch/v2 v2.5.0 gonum.org/v1/gonum v0.16.0 google.golang.org/protobuf v1.36.10 @@ -261,12 +261,12 @@ require ( go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/mod v0.29.0 // indirect - golang.org/x/net v0.46.0 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.32.0 // indirect golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.38.0 // indirect - golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect + golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 // indirect golang.org/x/term v0.37.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect diff --git a/go.sum b/go.sum index 3abf619c064..eee5d4f5dc9 100644 --- a/go.sum +++ b/go.sum @@ -878,8 +878,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -900,8 +900,8 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= -golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -943,8 +943,8 @@ golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU= -golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= +golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 h1:E2/AqCUMZGgd73TQkxUMcMla25GB9i/5HOdLr+uH7Vo= +golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ= golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -984,8 +984,8 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= From 53cbefc05a790851bd816c82abbdf86263551446 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Thu, 13 Nov 2025 15:16:25 +0100 Subject: [PATCH 107/176] Deflake lease controller integration tests (#13445) --- test/integration/gardenlet/seed/lease/lease_test.go | 2 +- test/integration/gardenlet/shoot/lease/lease_test.go | 2 +- .../operator/controllerregistrar/controllerregistrar_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/gardenlet/seed/lease/lease_test.go b/test/integration/gardenlet/seed/lease/lease_test.go index b7a35e1c922..da6e4ea7437 100644 --- a/test/integration/gardenlet/seed/lease/lease_test.go +++ b/test/integration/gardenlet/seed/lease/lease_test.go @@ -75,7 +75,7 @@ var _ = Describe("Seed lease controller tests", func() { Describe("do not update the Lease object and set the internal health status to false", Ordered, func() { var fakeError error - BeforeEach(OncePerOrdered, func() { + BeforeAll(func() { DeferCleanup(test.WithVar(&leasecontroller.CheckConnection, func(context.Context, rest.Interface) error { return fakeError })) }) diff --git a/test/integration/gardenlet/shoot/lease/lease_test.go b/test/integration/gardenlet/shoot/lease/lease_test.go index fcaa7f22192..abaf5d72d8c 100644 --- a/test/integration/gardenlet/shoot/lease/lease_test.go +++ b/test/integration/gardenlet/shoot/lease/lease_test.go @@ -75,7 +75,7 @@ var _ = Describe("Shoot lease controller tests", func() { Describe("do not update the Lease object and set the internal health status to false", Ordered, func() { var fakeError error - BeforeEach(OncePerOrdered, func() { + BeforeAll(func() { DeferCleanup(test.WithVar(&leasecontroller.CheckConnection, func(context.Context, rest.Interface) error { return fakeError })) }) diff --git a/test/integration/operator/controllerregistrar/controllerregistrar_test.go b/test/integration/operator/controllerregistrar/controllerregistrar_test.go index 61e46d6e83d..06055cb2b91 100644 --- a/test/integration/operator/controllerregistrar/controllerregistrar_test.go +++ b/test/integration/operator/controllerregistrar/controllerregistrar_test.go @@ -23,7 +23,7 @@ import ( . "github.com/gardener/gardener/pkg/utils/test/matchers" ) -var _ = Describe("Controller Registrar controller tests", Ordered, func() { +var _ = Describe("Controller Registrar controller tests", func() { var ( operatorCtx, operatorCancel = context.WithCancel(ctx) garden *operatorv1alpha1.Garden From 17a7b6476fed57b40383d43547a22ab5698bc06b Mon Sep 17 00:00:00 2001 From: Thomas Buchner Date: Thu, 13 Nov 2025 17:43:24 +0100 Subject: [PATCH 108/176] Improve validation of `sysctl` parameters to Shoot worker pools (#13435) * validation of sysctl parameters to worker pools Checks sysctl parameters to Shoot worker pools by making sure that sysctl keys match certain criteria (regex conformance and length). Since sysctl values may contain almos anything, they are checked to be not empty but will get written as is to the /etc/sysctl.d/98-k8s-general.conf file on a Shoot worker node. * address review feedback --- pkg/apis/core/validation/shoot.go | 38 ++++++++++++++ pkg/apis/core/validation/shoot_test.go | 69 ++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index 51673d7ca8c..78351172dc6 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -2169,6 +2169,10 @@ func ValidateWorker(worker core.Worker, kubernetes core.Kubernetes, shootNamespa } } + if worker.Sysctls != nil { + allErrs = append(allErrs, ValidateSysctls(worker.Sysctls, fldPath.Child("sysctls"))...) + } + return allErrs } @@ -3373,3 +3377,37 @@ func ValidateControlPlaneAutoscaling(autoscaling *core.ControlPlaneAutoscaling, return allErrs } + +// sysctlKeyRegex is used to validate the key of a sysctl +var sysctlKeyRegex = regexp.MustCompile(`^(?:(?:[a-zA-Z0-9-][a-zA-Z0-9_/-]*|\*)\.)*[a-zA-Z0-9_/-]*[a-zA-Z0-9]$`) + +// ValidateSysctls validates sysctls for valid keys and non-empty values +func ValidateSysctls(sysctls map[string]string, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + for sysctlKey, sysctlValue := range sysctls { + if !sysctlKeyRegex.MatchString(sysctlKey) { + allErrs = append(allErrs, field.Invalid(fldPath.Child(sysctlKey), sysctlKey, fmt.Sprintf("sysctl key must must match regex %q", sysctlKeyRegex))) + } + + // sysctl keys translate to files in /proc/sys and hence, the limits in + // include/linux/limits.h (NAME_MAX = 255 and PATH_MAX = 4096) apply + maxPathLen := 4096 - len("/proc/sys/") + + if len(sysctlKey) > maxPathLen { + allErrs = append(allErrs, field.TooLong(fldPath.Child(sysctlKey), sysctlKey, maxPathLen)) + } + + for subkey := range strings.SplitSeq(sysctlKey, ".") { + if len(subkey) > 255 { + allErrs = append(allErrs, field.Invalid(fldPath.Child(sysctlKey), subkey, "sub key of sysctl must not exceed 255 bytes")) + } + } + + if len(sysctlValue) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child(sysctlKey), "sysctl requires a non-empty value")) + } + } + + return allErrs +} diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index c3a3ed5f07e..8b0826af77a 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -8242,6 +8242,75 @@ var _ = Describe("Shoot Validation Tests", func() { "Detail": Equal("can not be less than -1"), })))) }) + + DescribeTable("sysctl setting validation", func(sysctls map[string]string, matcher gomegatypes.GomegaMatcher) { + errList := ValidateSysctls(sysctls, field.NewPath("sysctls")) + Expect(errList).To(matcher) + }, + Entry("accept valid sysctl keys", + map[string]string{ + "foo.bar": "123", + "fs.aio-nr": "192", + "fs.binfmt_misc.python3/13": "enabled", + "net.ipv4.conf.eth0.forwarding": "1", + "net.ipv4.tcp_mem": "374874 499832 749748", + "-net.ipv4.conf.all.proxy_arp": "0", + "net.ipv4.conf.*.route_localnet": "1", + }, + BeEmpty(), + ), + Entry("reject invalid sysctl keys not matching regex", + map[string]string{ + ".foo.bar": "123", + "foo..bar": "abc", + "foo.bar.": "123", + "foo.bar_": "abc", + "foo.bar/": "123", + "foo.bar-": "abc", + "foo\"bar": "123", + "foo=bar": "abc", + "foo;bar": "123", + "foo.**.bar": "abc", + "foo.bar*": "123", + "foo.bar.*": "abc", + "_foo.bar": "123", + "/foo.bar": "abc", + }, SatisfyAll( + HaveLen(14), + HaveEach(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": ContainSubstring("sysctls."), + "Detail": ContainSubstring("sysctl key must must match regex"), + }))), + ), + ), + Entry("reject sysctl keys with too long subkeys", + map[string]string{ + func(size int) string { + // create a very long string + var b strings.Builder + for range size { + fmt.Fprint(&b, "s") + } + return b.String() + }(256): "abc", + }, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": ContainSubstring("sysctls.sss"), + "Detail": Equal("sub key of sysctl must not exceed 255 bytes"), + }))), + ), + Entry("reject empty sysctl values", + map[string]string{ + "foo": "", + }, ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("sysctls.foo"), + })), + ), + ), + ) }) Describe("#ValidateWorkers", func() { From 356d00c9bd3dcfc29bbd1a5bb915c63593534bec Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Thu, 13 Nov 2025 18:43:32 +0200 Subject: [PATCH 109/176] Correct the version of the `VPAInPlaceUpdates` feature gate (#13443) --- docs/deployment/feature_gates.md | 2 +- pkg/features/features.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deployment/feature_gates.md b/docs/deployment/feature_gates.md index da24f969470..57e42e1a341 100644 --- a/docs/deployment/feature_gates.md +++ b/docs/deployment/feature_gates.md @@ -32,7 +32,7 @@ The following tables are a summary of the feature gates that you can set on diff | DoNotCopyBackupCredentials | `true` | `Beta` | `1.123` | | | OpenTelemetryCollector | `false` | `Alpha` | `1.124` | | | UseUnifiedHTTPProxyPort | `false` | `Alpha` | `1.130` | | -| VPAInPlaceUpdates | `false` | `Alpha` | `1.132` | | +| VPAInPlaceUpdates | `false` | `Alpha` | `1.133` | | ## Feature Gates for Graduated or Deprecated Features diff --git a/pkg/features/features.go b/pkg/features/features.go index ce140bad45f..53d1b36185b 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -78,7 +78,7 @@ const ( // VPAInPlaceUpdates enables the usage of in-place Pod resource updates in the Vertical Pod Autoscaler resources // to perform in-place Pod resource updates. // owner: @vitanovs @ialidzhikov - // alpha: v1.132.0 + // alpha: v1.133.0 VPAInPlaceUpdates featuregate.Feature = "VPAInPlaceUpdates" ) From 13ae41d51b899591df157182c2fff0bf49992e55 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 13 Nov 2025 21:29:25 +0100 Subject: [PATCH 110/176] Update dependency crate-ci/typos to v1.39.2 (#13451) --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 14d33f3575d..5b58af0a1c9 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -83,7 +83,7 @@ YQ_VERSION ?= v4.48.1 # renovate: datasource=github-releases depName=ironcore-dev/vgopath VGOPATH_VERSION ?= v0.1.9 # renovate: datasource=github-releases depName=crate-ci/typos -TYPOS_VERSION ?= v1.39.1 +TYPOS_VERSION ?= v1.39.2 # tool versions from go.mod CONTROLLER_GEN_VERSION ?= $(call version_gomod,sigs.k8s.io/controller-tools) From 24d6670462052b484bd0b16c8344d579efa602e5 Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Thu, 13 Nov 2025 22:52:24 +0100 Subject: [PATCH 111/176] =?UTF-8?q?=F0=9F=93=96=20Gardener=20Upgrade=20Gui?= =?UTF-8?q?de=20(#13401)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Gardener Upgrade Guide * LLM improvements * Improved readiness verification for `Seed`s * Address PR review feedback * Address PR review feedback * Address PR review feedback --- docs/README.md | 1 + docs/deployment/version_skew_policy.md | 9 +- docs/operations/upgrade-gardener.md | 124 +++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 docs/operations/upgrade-gardener.md diff --git a/docs/README.md b/docs/README.md index cd46d8763f8..1deb0bdf692 100644 --- a/docs/README.md +++ b/docs/README.md @@ -254,6 +254,7 @@ ## Operations * [Gardener configuration and usage](operations/configuration.md) +* [Gardener Upgrade Guide](operations/upgrade-gardener.md) * [Control Plane Migration](operations/control_plane_migration.md) * [Enabling In-place Resource Updates](operations/enabling-in-place-resource-updates.md) * [Immutable Backup Buckets](operations/immutable-backup-buckets.md) diff --git a/docs/deployment/version_skew_policy.md b/docs/deployment/version_skew_policy.md index 2925f326201..0e3abc149d7 100644 --- a/docs/deployment/version_skew_policy.md +++ b/docs/deployment/version_skew_policy.md @@ -16,10 +16,13 @@ For more information, see the [Releases document](../development/process.md#rele ### Supported Version Skew Technically, we follow the same [policy](https://kubernetes.io/releases/version-skew-policy/) as the Kubernetes project. -However, given that our release cadence is much more frequent compared to Kubernetes (every `14d` vs. every `120d`), in many cases it might be possible to skip versions, though we do not test these upgrade paths. -Consequently, in general it might not work, and to be on the safe side, it is highly recommended to follow the described policy. +However, given that our release cadence is much more frequent compared to Kubernetes (every `14d` vs. every `120d`), in many cases it might be possible to skip minor versions, though we do not test these upgrade paths. +Consequently, in general it might not work, and to be on the safe side, it is highly recommended to follow the described policy and increment the minor version only one step at a time. -🚨 Note that downgrading Gardener versions is generally not tested during development and should be considered unsupported. +>[!NOTE] +> 🚨 Downgrading Gardener versions is generally not tested during development and should be considered unsupported. + +Please also take a look at the [Gardener upgrade guide](../operations/upgrade-gardener.md). #### gardener-apiserver diff --git a/docs/operations/upgrade-gardener.md b/docs/operations/upgrade-gardener.md new file mode 100644 index 00000000000..2ddad2617b9 --- /dev/null +++ b/docs/operations/upgrade-gardener.md @@ -0,0 +1,124 @@ +# Gardener Upgrade Guide + +Upgrading Gardener is a multi-step process that updates its core components, including the operator, control plane, `gardenlet`s, and the clusters it manages (seeds and shoots). To ensure a smooth and safe upgrade, you should follow a specific sequence of steps. This process works with Gardener's built-in reconciliation and rollout mechanisms, which automatically apply changes across your landscape. + +## 1. Prepare Before Touching Anything + +First, understand Gardener's [version skew policy](../deployment/version_skew_policy.md), which defines which component versions are compatible with each other. + +> [!NOTE] +> Please pay special attention to this part of the version skew policy: +> +> _"However, given that our release cadence is much more frequent compared to Kubernetes (every `14d` vs. every `120d`), in many cases it might be possible to skip minor versions, though we do not test these upgrade paths._ +> _Consequently, in general it might not work, and to be on the safe side, it is highly recommended to follow the described policy and increment the minor version only one step at a time."_ +> +> _"🚨 Note that downgrading Gardener versions is generally not tested during development and should be considered unsupported."_ + +Next, carefully read the [release notes](https://github.com/gardener/gardener/releases) for your target version. The notes will detail any breaking changes, new features, and bug fixes. +Each release note has target audience: + +- `[USER]` (end-users or users of `Shoot` resources and related configuration) +- `[OPERATOR]` (operators deploying Gardener, managing configuration, seed clusters, etc.) +- `[DEVELOPER]` (developers changing or reviewing code in this repository) +- `[DEPENDENCY]` (developers depending on code in this repository (e.g., extension developers)) + +In the context of this guide, you should focus on release notes with `[OPERATOR]` and `[USER]` prefixes. + +### Deprecations and Backwards-Compatibility + +Gardener introduces breaking changes cautiously to ensure stability. You can read the full policy [here](../development/process.md#deprecations-and-backwards-compatibility). The release notes will always highlight when you need to perform manual steps. + +- Changes affecting `Shoot` clusters are typically tied to a Kubernetes minor version upgrade. +- Changes to operator-level APIs (like `Garden` or `Seed`) are deprecated for at least three minor releases before being removed. +- For extensions, the deprecation period is even longer, typically nine minor releases. + +If there are breaking changes, you must apply them to your configuration files (manifests) *before* you update the version numbers. This prevents components from starting with an incompatible configuration. + +> [!NOTE] +> As explained above, changes affecting the `Shoot` API for `[USER]`s are typically connected with a Kubernetes minor version upgrade. +> Hence, they only become relevant when they upgrade Kubernetes - see also [this document](../usage/shoot/shoot_kubernetes_versions.md). +> All `[USER]`-related changes mark as "breaking" must be adapted right now. + +## 2. Upgrade `gardener-operator` and Gardener Control Plane + +After updating your configuration files, you can deploy the new `gardener-operator` using its [Helm chart](../../charts/gardener/operator). Once the new operator is running, it will automatically begin updating the `Garden` resource. This process rolls out the new versions of the Gardener control plane components, such as `gardener-apiserver` and `gardener-controller-manager`. + +### Image Vector and Overwrites + +Gardener components and extensions use an "image vector" to define the specific container images they deploy. If your organization requires using a private container registry, you can replicate the official images and configure Gardener to use them. Follow the instructions [here](../deployment/image_vector.md) to create an image vector overwrite. + +Each Gardener release includes a `component-descriptor.yaml` file as a release asset. This file lists all container images for that version. You can use this list to pull the images, push them to your private registry, and generate the necessary configuration overwrite. + +### Extensions + +With `gardener-operator`, extensions should always be installed via the `operator.gardener.cloud/v1alpha1.Extension` API in the runtime cluster. +In order to upgrade them, update their OCI references in these `Extension` resources. + +In case you also have extensions manually deployed to the (virtual) garden cluster (not recommended), update the version in their `ControllerDeployment` resources. + +In both cases, make sure to always check the extension's release notes to ensure compatibility with the Gardener version. + +### Verify Readiness + +Before moving to the next step, you must verify that the `Garden` resource has been successfully updated. Check the following: + +- The `gardener.cloud/operation` annotation is removed, indicating the operation is complete. +- The `.status.gardener.version` field shows your target version. +- The `.status.observedGeneration` matches the `.metadata.generation`, meaning the latest configuration has been processed. +- The `.status.lastOperation.state` is `Succeeded`. + +At this stage, it's normal for the `gardenlet`s to still be running the old version. This is similar to how Kubernetes upgrades its control plane before the `kubelet`s on worker nodes. + +Finally, check the health conditions (`.status.conditions[]`) in the `Garden` resource to ensure they all report `True`. +Also check the health conditions in the `operator.gardener.cloud/v1alpha1.Extension` resources and ensure they all report `True`. + +## 3. Upgrade Your `gardenlet`s + +Next, upgrade the `gardenlet`s. + +### Unmanaged Seeds + +Start with `gardenlet`s that manage "unmanaged" seeds - seed clusters that are not created via `Shoot`s. These should be configured using `Gardenlet` resources in the `garden` namespace within the (virtual) garden cluster. + +If you've enabled auto-updates (by adding the `operator.gardener.cloud/auto-update-gardenlet-helm-chart-ref=true` label to your `Gardenlet` resources), the `gardener-operator` will automatically trigger the upgrade. The `gardenlet`s will then perform a self-upgrade. + +Alternatively, if you manage your `Gardenlet` resources with a GitOps tool like Flux, you should not use the auto-update label. Instead, update the Helm chart reference (`.spec.deployment.helm.ociRepository.ref`) in your configuration and apply the changes yourself. + +### Managed Seeds + +Once a `gardenlet` is upgraded, it automatically begins upgrading any "managed" seeds it controls. No manual action is needed for this step. Note that a `ManagedSeed` upgrade will wait until any ongoing reconciliation of its underlying `Shoot` cluster is complete. + +To prevent overloading the system, these upgrades are staggered using a "jitter" period, so they won't all start at once. By default, this period is 5 minutes. You can adjust it in the `gardenlet`'s [component configuration](../../example/20-componentconfig-gardenlet.yaml) by setting `.controllers.managedSeed.syncJitterPeriod`. Set it to `0` to start all upgrades immediately, or increase it if you have many seed clusters to manage the load. + +### Verify Readiness + +To confirm that all your seed clusters are updated, check the following for each `Seed` resource: + +- The `gardener.cloud/operation` annotation is removed, indicating the operation is complete. +- The `.status.gardener.version` field shows your target version. +- The `.status.observedGeneration` matches the `.metadata.generation`, meaning the latest configuration has been processed. +- The `.status.lastOperation.state` is `Succeeded`. + +Finally, check the health conditions (`.status.conditions[]`) in the `Seed` resources to ensure they all report `True`. + +## 4. Shoot Reconciliations + +By default, after a `gardenlet` is upgraded, it immediately starts reconciling the `Shoot` clusters it manages. While this is suitable for small setups, it's often better to perform these reconciliations only during a predefined maintenance window. + +To enable this, set `.controllers.shoot.reconcileInMaintenanceOnly=true` in the `gardenlet`'s [component configuration](../../example/20-componentconfig-gardenlet.yaml). When this setting is enabled, all `Shoot`s will be reconciled during their next scheduled maintenance window, which typically occurs within 24 hours. You can learn more about shoot maintenance [here](../usage/shoot/shoot_maintenance.md#cluster-reconciliation). + +> [!IMPORTANT] +> Make sure that all `Shoot`s have been reconciled (even if it failed) with the currently installed Gardener version before upgrading to the next version. +> This is especially relevant if you choose to only reconcile in the shoot clusters' maintenance time windows. +> In this case, you need to wait `24h` before upgrading Gardener again to be on the safe side. +> +> You can verify that all `Shoot`s have been reconciled with the current Gardener version by checking that their `.status.gardener.version` fields show your target version. + +### Operating System Config Updates + +Gardener upgrades also roll out new versions of `gardener-node-agent` which runs on each worker node of all `Shoot`s. +This rollout happens during `Shoot` reconciliation and could also include other operating system configuration updates. +Similar to seed upgrades, such updates to the operating system on `Shoot` cluster worker nodes are also staggered. +This prevents all nodes from being updated simultaneously, which could cause disruptions like `kubelet` restarts across the entire cluster. + +By default, the rollout across all nodes completes within 5 minutes. You can customize this timeframe by adding the `shoot.gardener.cloud/cloud-config-execution-max-delay-seconds` annotation to your `Shoot`s. A value of `0` updates all nodes in parallel, while a higher value spreads the update over a longer period (up to 1800 seconds). From 029a9a0738a189c16045f44c02e3d9cfbe222f01 Mon Sep 17 00:00:00 2001 From: Aleksandar Savchev <57963548+AleksandarSavchev@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:39:25 +0200 Subject: [PATCH 112/176] Create new status field `.status.credentials.encryptionAtRest` to `Shoot` and `Garden` (#12894) * Adapt Shoot API * Make generate * Use new status field for Shoot * Migrate encryptedResources status to new field in Shoot * Adapt Garden API * Make generate * Use new status field for Garden * Migrate encryptedResources status to new field in Garden * Fix unit tests * Change API * Make generate * Add GetShootEncryptedResourcesInStatus utils func * Improve strategy * Add on read decoder for Shoot status field * Default on garden status change as well * Fix gardenlet shoot build * Add suggested changes * Add nit * Make generate * Add nits --- .../operator/templates/crd-gardens.yaml | 16 + docs/api-reference/core.md | 49 + docs/api-reference/operator.md | 49 + ...0-crd-operator.gardener.cloud_gardens.yaml | 16 + pkg/apis/core/types_shoot.go | 13 + pkg/apis/core/v1beta1/generated.pb.go | 2355 +++++++++-------- pkg/apis/core/v1beta1/generated.proto | 16 + pkg/apis/core/v1beta1/helper/shoot.go | 9 + pkg/apis/core/v1beta1/helper/shoot_test.go | 9 + pkg/apis/core/v1beta1/types_shoot.go | 15 + .../core/v1beta1/zz_generated.conversion.go | 32 + .../core/v1beta1/zz_generated.deepcopy.go | 26 + pkg/apis/core/validation/shoot.go | 20 +- pkg/apis/core/validation/shoot_test.go | 93 +- pkg/apis/core/zz_generated.deepcopy.go | 26 + pkg/apis/operator/v1alpha1/helper/garden.go | 9 + .../operator/v1alpha1/helper/garden_test.go | 9 + pkg/apis/operator/v1alpha1/types_garden.go | 15 + .../operator/v1alpha1/validation/garden.go | 6 +- .../v1alpha1/validation/garden_test.go | 73 +- .../v1alpha1/zz_generated.deepcopy.go | 26 + pkg/apiserver/openapi/api_violations.report | 1 + pkg/apiserver/openapi/openapi_generated.go | 39 +- .../registry/core/shoot/storage/storage.go | 27 + pkg/apiserver/registry/core/shoot/strategy.go | 24 + .../registry/core/shoot/strategy_test.go | 87 + .../shoot/shoot/reconciler_reconcile.go | 15 + pkg/gardenlet/operation/shoot/shoot.go | 5 +- .../garden/garden/reconciler_reconcile.go | 22 +- pkg/operator/webhook/add.go | 3 +- .../webhook/defaulting/garden/handler.go | 15 + .../webhook/defaulting/garden/handler_test.go | 42 + 32 files changed, 2024 insertions(+), 1138 deletions(-) diff --git a/charts/gardener/operator/templates/crd-gardens.yaml b/charts/gardener/operator/templates/crd-gardens.yaml index 3d85255cfbe..134597cb6eb 100644 --- a/charts/gardener/operator/templates/crd-gardens.yaml +++ b/charts/gardener/operator/templates/crd-gardens.yaml @@ -2016,6 +2016,19 @@ spec: description: Credentials contains information about the virtual garden cluster credentials. properties: + encryptionAtRest: + description: EncryptionAtRest contains information about garden + data encryption at rest. + properties: + resources: + description: |- + Resources is the list of resources which are currently encrypted in the virtual garden by the virtual kube-apiserver. + Resources which are encrypted by default will not appear here. + See https://github.com/gardener/gardener/blob/master/docs/concepts/operator.md#etcd-encryption-config for more details. + items: + type: string + type: array + type: object rotation: description: Rotation contains information about the credential rotations. @@ -2234,6 +2247,9 @@ spec: EncryptedResources is the list of resources which are currently encrypted in the virtual garden by the virtual kube-apiserver. Resources which are encrypted by default will not appear here. See https://github.com/gardener/gardener/blob/master/docs/concepts/operator.md#etcd-encryption-config for more details. + + Deprecated: This field is deprecated and will be removed with release v1.138. + This field will be removed in favor of `status.credentials.encryptionAtRest.resources`. items: type: string type: array diff --git a/docs/api-reference/core.md b/docs/api-reference/core.md index a3562357d69..be97731bb4e 100644 --- a/docs/api-reference/core.md +++ b/docs/api-reference/core.md @@ -5533,6 +5533,39 @@ TODO(AleksandarSavchev): Remove this after support for Kubernetes v1.33 is dropp +

EncryptionAtRest +

+

+(Appears on: +ShootCredentials) +

+

+

EncryptionAtRest contains information about Shoot data encryption at rest.

+

+ + + + + + + + + + + + + +
FieldDescription
+resources
+ +[]string + +		 +(Optional) +

Resources is the list of resources in the Shoot which are currently encrypted. +Secrets are encrypted by default and are not part of the list. +See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details.

+

EncryptionConfig

@@ -12753,6 +12786,20 @@ ShootCredentialsRotation

Rotation contains information about the credential rotations.

+ + +encryptionAtRest
+ + +EncryptionAtRest + + + + +(Optional) +

EncryptionAtRest contains information about Shoot data encryption at rest.

+ +

ShootCredentialsRotation @@ -13717,6 +13764,8 @@ LastMaintenance

EncryptedResources is the list of resources in the Shoot which are currently encrypted. Secrets are encrypted by default and are not part of the list. See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details.

+

Deprecated: This field is deprecated and will be removed with release v1.138. +This field will be removed in favor of status.credentials.encryptionAtRest.resources.

diff --git a/docs/api-reference/operator.md b/docs/api-reference/operator.md index 27573f77627..26fe312b313 100644 --- a/docs/api-reference/operator.md +++ b/docs/api-reference/operator.md @@ -361,6 +361,20 @@ CredentialsRotation

Rotation contains information about the credential rotations.

+ + +encryptionAtRest
+ + +EncryptionAtRest + + + + +(Optional) +

EncryptionAtRest contains information about garden data encryption at rest.

+ +

CredentialsRotation @@ -1152,6 +1166,39 @@ Storage +

EncryptionAtRest +

+

+(Appears on: +Credentials) +

+

+

EncryptionAtRest contains information about virtual garden data encryption at rest.

+

+ + + + + + + + + + + + + +
FieldDescription
+resources
+ +[]string + +		 +(Optional) +

Resources is the list of resources which are currently encrypted in the virtual garden by the virtual kube-apiserver. +Resources which are encrypted by default will not appear here. +See https://github.com/gardener/gardener/blob/master/docs/concepts/operator.md#etcd-encryption-config for more details.

+

Extension

@@ -1792,6 +1839,8 @@ Credentials

EncryptedResources is the list of resources which are currently encrypted in the virtual garden by the virtual kube-apiserver. Resources which are encrypted by default will not appear here. See https://github.com/gardener/gardener/blob/master/docs/concepts/operator.md#etcd-encryption-config for more details.

+

Deprecated: This field is deprecated and will be removed with release v1.138. +This field will be removed in favor of status.credentials.encryptionAtRest.resources.

diff --git a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml index 3d85255cfbe..134597cb6eb 100644 --- a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml +++ b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml @@ -2016,6 +2016,19 @@ spec: description: Credentials contains information about the virtual garden cluster credentials. properties: + encryptionAtRest: + description: EncryptionAtRest contains information about garden + data encryption at rest. + properties: + resources: + description: |- + Resources is the list of resources which are currently encrypted in the virtual garden by the virtual kube-apiserver. + Resources which are encrypted by default will not appear here. + See https://github.com/gardener/gardener/blob/master/docs/concepts/operator.md#etcd-encryption-config for more details. + items: + type: string + type: array + type: object rotation: description: Rotation contains information about the credential rotations. @@ -2234,6 +2247,9 @@ spec: EncryptedResources is the list of resources which are currently encrypted in the virtual garden by the virtual kube-apiserver. Resources which are encrypted by default will not appear here. See https://github.com/gardener/gardener/blob/master/docs/concepts/operator.md#etcd-encryption-config for more details. + + Deprecated: This field is deprecated and will be removed with release v1.138. + This field will be removed in favor of `status.credentials.encryptionAtRest.resources`. items: type: string type: array diff --git a/pkg/apis/core/types_shoot.go b/pkg/apis/core/types_shoot.go index 9fc077acd0e..8137b497204 100644 --- a/pkg/apis/core/types_shoot.go +++ b/pkg/apis/core/types_shoot.go @@ -164,6 +164,9 @@ type ShootStatus struct { // EncryptedResources is the list of resources in the Shoot which are currently encrypted. // Secrets are encrypted by default and are not part of the list. // See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details. + // + // Deprecated: This field is deprecated and will be removed with release v1.138. + // This field will be removed in favor of `status.credentials.encryptionAtRest.resources`. EncryptedResources []string // Networking contains information about cluster networking such as CIDRs. Networking *NetworkingStatus @@ -217,6 +220,8 @@ type PendingWorkerUpdates struct { type ShootCredentials struct { // Rotation contains information about the credential rotations. Rotation *ShootCredentialsRotation + // EncryptionAtRest contains information about Shoot data encryption at rest. + EncryptionAtRest *EncryptionAtRest } // ShootCredentialsRotation contains information about the rotation of credentials. @@ -233,6 +238,14 @@ type ShootCredentialsRotation struct { ETCDEncryptionKey *ETCDEncryptionKeyRotation } +// EncryptionAtRest contains information about Shoot data encryption at rest. +type EncryptionAtRest struct { + // Resources is the list of resources in the Shoot which are currently encrypted. + // Secrets are encrypted by default and are not part of the list. + // See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details. + Resources []string +} + // CARotation contains information about the certificate authority credential rotation. type CARotation struct { // Phase describes the phase of the certificate authority credential rotation. diff --git a/pkg/apis/core/v1beta1/generated.pb.go b/pkg/apis/core/v1beta1/generated.pb.go index fcf4807b78b..dfa77f4f0c4 100644 --- a/pkg/apis/core/v1beta1/generated.pb.go +++ b/pkg/apis/core/v1beta1/generated.pb.go @@ -1807,10 +1807,38 @@ func (m *ETCDEncryptionKeyRotation) XXX_DiscardUnknown() { var xxx_messageInfo_ETCDEncryptionKeyRotation proto.InternalMessageInfo +func (m *EncryptionAtRest) Reset() { *m = EncryptionAtRest{} } +func (*EncryptionAtRest) ProtoMessage() {} +func (*EncryptionAtRest) Descriptor() ([]byte, []int) { + return fileDescriptor_ca37af0df9a5bbd2, []int{63} +} +func (m *EncryptionAtRest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *EncryptionAtRest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *EncryptionAtRest) XXX_Merge(src proto.Message) { + xxx_messageInfo_EncryptionAtRest.Merge(m, src) +} +func (m *EncryptionAtRest) XXX_Size() int { + return m.Size() +} +func (m *EncryptionAtRest) XXX_DiscardUnknown() { + xxx_messageInfo_EncryptionAtRest.DiscardUnknown(m) +} + +var xxx_messageInfo_EncryptionAtRest proto.InternalMessageInfo + func (m *EncryptionConfig) Reset() { *m = EncryptionConfig{} } func (*EncryptionConfig) ProtoMessage() {} func (*EncryptionConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{63} + return fileDescriptor_ca37af0df9a5bbd2, []int{64} } func (m *EncryptionConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1838,7 +1866,7 @@ var xxx_messageInfo_EncryptionConfig proto.InternalMessageInfo func (m *ExpirableVersion) Reset() { *m = ExpirableVersion{} } func (*ExpirableVersion) ProtoMessage() {} func (*ExpirableVersion) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{64} + return fileDescriptor_ca37af0df9a5bbd2, []int{65} } func (m *ExpirableVersion) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1866,7 +1894,7 @@ var xxx_messageInfo_ExpirableVersion proto.InternalMessageInfo func (m *ExposureClass) Reset() { *m = ExposureClass{} } func (*ExposureClass) ProtoMessage() {} func (*ExposureClass) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{65} + return fileDescriptor_ca37af0df9a5bbd2, []int{66} } func (m *ExposureClass) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1894,7 +1922,7 @@ var xxx_messageInfo_ExposureClass proto.InternalMessageInfo func (m *ExposureClassList) Reset() { *m = ExposureClassList{} } func (*ExposureClassList) ProtoMessage() {} func (*ExposureClassList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{66} + return fileDescriptor_ca37af0df9a5bbd2, []int{67} } func (m *ExposureClassList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1922,7 +1950,7 @@ var xxx_messageInfo_ExposureClassList proto.InternalMessageInfo func (m *ExposureClassScheduling) Reset() { *m = ExposureClassScheduling{} } func (*ExposureClassScheduling) ProtoMessage() {} func (*ExposureClassScheduling) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{67} + return fileDescriptor_ca37af0df9a5bbd2, []int{68} } func (m *ExposureClassScheduling) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1950,7 +1978,7 @@ var xxx_messageInfo_ExposureClassScheduling proto.InternalMessageInfo func (m *Extension) Reset() { *m = Extension{} } func (*Extension) ProtoMessage() {} func (*Extension) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{68} + return fileDescriptor_ca37af0df9a5bbd2, []int{69} } func (m *Extension) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1978,7 +2006,7 @@ var xxx_messageInfo_Extension proto.InternalMessageInfo func (m *ExtensionResourceState) Reset() { *m = ExtensionResourceState{} } func (*ExtensionResourceState) ProtoMessage() {} func (*ExtensionResourceState) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{69} + return fileDescriptor_ca37af0df9a5bbd2, []int{70} } func (m *ExtensionResourceState) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2006,7 +2034,7 @@ var xxx_messageInfo_ExtensionResourceState proto.InternalMessageInfo func (m *FailureTolerance) Reset() { *m = FailureTolerance{} } func (*FailureTolerance) ProtoMessage() {} func (*FailureTolerance) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{70} + return fileDescriptor_ca37af0df9a5bbd2, []int{71} } func (m *FailureTolerance) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2034,7 +2062,7 @@ var xxx_messageInfo_FailureTolerance proto.InternalMessageInfo func (m *Gardener) Reset() { *m = Gardener{} } func (*Gardener) ProtoMessage() {} func (*Gardener) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{71} + return fileDescriptor_ca37af0df9a5bbd2, []int{72} } func (m *Gardener) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2062,7 +2090,7 @@ var xxx_messageInfo_Gardener proto.InternalMessageInfo func (m *GardenerResourceData) Reset() { *m = GardenerResourceData{} } func (*GardenerResourceData) ProtoMessage() {} func (*GardenerResourceData) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{72} + return fileDescriptor_ca37af0df9a5bbd2, []int{73} } func (m *GardenerResourceData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2090,7 +2118,7 @@ var xxx_messageInfo_GardenerResourceData proto.InternalMessageInfo func (m *HelmControllerDeployment) Reset() { *m = HelmControllerDeployment{} } func (*HelmControllerDeployment) ProtoMessage() {} func (*HelmControllerDeployment) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{73} + return fileDescriptor_ca37af0df9a5bbd2, []int{74} } func (m *HelmControllerDeployment) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2118,7 +2146,7 @@ var xxx_messageInfo_HelmControllerDeployment proto.InternalMessageInfo func (m *Hibernation) Reset() { *m = Hibernation{} } func (*Hibernation) ProtoMessage() {} func (*Hibernation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{74} + return fileDescriptor_ca37af0df9a5bbd2, []int{75} } func (m *Hibernation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2146,7 +2174,7 @@ var xxx_messageInfo_Hibernation proto.InternalMessageInfo func (m *HibernationSchedule) Reset() { *m = HibernationSchedule{} } func (*HibernationSchedule) ProtoMessage() {} func (*HibernationSchedule) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{75} + return fileDescriptor_ca37af0df9a5bbd2, []int{76} } func (m *HibernationSchedule) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2174,7 +2202,7 @@ var xxx_messageInfo_HibernationSchedule proto.InternalMessageInfo func (m *HighAvailability) Reset() { *m = HighAvailability{} } func (*HighAvailability) ProtoMessage() {} func (*HighAvailability) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{76} + return fileDescriptor_ca37af0df9a5bbd2, []int{77} } func (m *HighAvailability) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2202,7 +2230,7 @@ var xxx_messageInfo_HighAvailability proto.InternalMessageInfo func (m *HorizontalPodAutoscalerConfig) Reset() { *m = HorizontalPodAutoscalerConfig{} } func (*HorizontalPodAutoscalerConfig) ProtoMessage() {} func (*HorizontalPodAutoscalerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{77} + return fileDescriptor_ca37af0df9a5bbd2, []int{78} } func (m *HorizontalPodAutoscalerConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2230,7 +2258,7 @@ var xxx_messageInfo_HorizontalPodAutoscalerConfig proto.InternalMessageInfo func (m *InPlaceUpdates) Reset() { *m = InPlaceUpdates{} } func (*InPlaceUpdates) ProtoMessage() {} func (*InPlaceUpdates) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{78} + return fileDescriptor_ca37af0df9a5bbd2, []int{79} } func (m *InPlaceUpdates) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2258,7 +2286,7 @@ var xxx_messageInfo_InPlaceUpdates proto.InternalMessageInfo func (m *InPlaceUpdatesStatus) Reset() { *m = InPlaceUpdatesStatus{} } func (*InPlaceUpdatesStatus) ProtoMessage() {} func (*InPlaceUpdatesStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{79} + return fileDescriptor_ca37af0df9a5bbd2, []int{80} } func (m *InPlaceUpdatesStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2286,7 +2314,7 @@ var xxx_messageInfo_InPlaceUpdatesStatus proto.InternalMessageInfo func (m *Ingress) Reset() { *m = Ingress{} } func (*Ingress) ProtoMessage() {} func (*Ingress) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{80} + return fileDescriptor_ca37af0df9a5bbd2, []int{81} } func (m *Ingress) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2314,7 +2342,7 @@ var xxx_messageInfo_Ingress proto.InternalMessageInfo func (m *IngressController) Reset() { *m = IngressController{} } func (*IngressController) ProtoMessage() {} func (*IngressController) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{81} + return fileDescriptor_ca37af0df9a5bbd2, []int{82} } func (m *IngressController) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2342,7 +2370,7 @@ var xxx_messageInfo_IngressController proto.InternalMessageInfo func (m *InternalSecret) Reset() { *m = InternalSecret{} } func (*InternalSecret) ProtoMessage() {} func (*InternalSecret) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{82} + return fileDescriptor_ca37af0df9a5bbd2, []int{83} } func (m *InternalSecret) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2370,7 +2398,7 @@ var xxx_messageInfo_InternalSecret proto.InternalMessageInfo func (m *InternalSecretList) Reset() { *m = InternalSecretList{} } func (*InternalSecretList) ProtoMessage() {} func (*InternalSecretList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{83} + return fileDescriptor_ca37af0df9a5bbd2, []int{84} } func (m *InternalSecretList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2398,7 +2426,7 @@ var xxx_messageInfo_InternalSecretList proto.InternalMessageInfo func (m *KubeAPIServerConfig) Reset() { *m = KubeAPIServerConfig{} } func (*KubeAPIServerConfig) ProtoMessage() {} func (*KubeAPIServerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{84} + return fileDescriptor_ca37af0df9a5bbd2, []int{85} } func (m *KubeAPIServerConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2426,7 +2454,7 @@ var xxx_messageInfo_KubeAPIServerConfig proto.InternalMessageInfo func (m *KubeControllerManagerConfig) Reset() { *m = KubeControllerManagerConfig{} } func (*KubeControllerManagerConfig) ProtoMessage() {} func (*KubeControllerManagerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{85} + return fileDescriptor_ca37af0df9a5bbd2, []int{86} } func (m *KubeControllerManagerConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2454,7 +2482,7 @@ var xxx_messageInfo_KubeControllerManagerConfig proto.InternalMessageInfo func (m *KubeProxyConfig) Reset() { *m = KubeProxyConfig{} } func (*KubeProxyConfig) ProtoMessage() {} func (*KubeProxyConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{86} + return fileDescriptor_ca37af0df9a5bbd2, []int{87} } func (m *KubeProxyConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2482,7 +2510,7 @@ var xxx_messageInfo_KubeProxyConfig proto.InternalMessageInfo func (m *KubeSchedulerConfig) Reset() { *m = KubeSchedulerConfig{} } func (*KubeSchedulerConfig) ProtoMessage() {} func (*KubeSchedulerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{87} + return fileDescriptor_ca37af0df9a5bbd2, []int{88} } func (m *KubeSchedulerConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2510,7 +2538,7 @@ var xxx_messageInfo_KubeSchedulerConfig proto.InternalMessageInfo func (m *KubeletConfig) Reset() { *m = KubeletConfig{} } func (*KubeletConfig) ProtoMessage() {} func (*KubeletConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{88} + return fileDescriptor_ca37af0df9a5bbd2, []int{89} } func (m *KubeletConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2538,7 +2566,7 @@ var xxx_messageInfo_KubeletConfig proto.InternalMessageInfo func (m *KubeletConfigEviction) Reset() { *m = KubeletConfigEviction{} } func (*KubeletConfigEviction) ProtoMessage() {} func (*KubeletConfigEviction) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{89} + return fileDescriptor_ca37af0df9a5bbd2, []int{90} } func (m *KubeletConfigEviction) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2566,7 +2594,7 @@ var xxx_messageInfo_KubeletConfigEviction proto.InternalMessageInfo func (m *KubeletConfigEvictionMinimumReclaim) Reset() { *m = KubeletConfigEvictionMinimumReclaim{} } func (*KubeletConfigEvictionMinimumReclaim) ProtoMessage() {} func (*KubeletConfigEvictionMinimumReclaim) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{90} + return fileDescriptor_ca37af0df9a5bbd2, []int{91} } func (m *KubeletConfigEvictionMinimumReclaim) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2594,7 +2622,7 @@ var xxx_messageInfo_KubeletConfigEvictionMinimumReclaim proto.InternalMessageInf func (m *KubeletConfigEvictionSoftGracePeriod) Reset() { *m = KubeletConfigEvictionSoftGracePeriod{} } func (*KubeletConfigEvictionSoftGracePeriod) ProtoMessage() {} func (*KubeletConfigEvictionSoftGracePeriod) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{91} + return fileDescriptor_ca37af0df9a5bbd2, []int{92} } func (m *KubeletConfigEvictionSoftGracePeriod) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2622,7 +2650,7 @@ var xxx_messageInfo_KubeletConfigEvictionSoftGracePeriod proto.InternalMessageIn func (m *KubeletConfigReserved) Reset() { *m = KubeletConfigReserved{} } func (*KubeletConfigReserved) ProtoMessage() {} func (*KubeletConfigReserved) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{92} + return fileDescriptor_ca37af0df9a5bbd2, []int{93} } func (m *KubeletConfigReserved) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2650,7 +2678,7 @@ var xxx_messageInfo_KubeletConfigReserved proto.InternalMessageInfo func (m *Kubernetes) Reset() { *m = Kubernetes{} } func (*Kubernetes) ProtoMessage() {} func (*Kubernetes) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{93} + return fileDescriptor_ca37af0df9a5bbd2, []int{94} } func (m *Kubernetes) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2678,7 +2706,7 @@ var xxx_messageInfo_Kubernetes proto.InternalMessageInfo func (m *KubernetesConfig) Reset() { *m = KubernetesConfig{} } func (*KubernetesConfig) ProtoMessage() {} func (*KubernetesConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{94} + return fileDescriptor_ca37af0df9a5bbd2, []int{95} } func (m *KubernetesConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2706,7 +2734,7 @@ var xxx_messageInfo_KubernetesConfig proto.InternalMessageInfo func (m *KubernetesDashboard) Reset() { *m = KubernetesDashboard{} } func (*KubernetesDashboard) ProtoMessage() {} func (*KubernetesDashboard) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{95} + return fileDescriptor_ca37af0df9a5bbd2, []int{96} } func (m *KubernetesDashboard) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2734,7 +2762,7 @@ var xxx_messageInfo_KubernetesDashboard proto.InternalMessageInfo func (m *KubernetesSettings) Reset() { *m = KubernetesSettings{} } func (*KubernetesSettings) ProtoMessage() {} func (*KubernetesSettings) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{96} + return fileDescriptor_ca37af0df9a5bbd2, []int{97} } func (m *KubernetesSettings) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2762,7 +2790,7 @@ var xxx_messageInfo_KubernetesSettings proto.InternalMessageInfo func (m *LastError) Reset() { *m = LastError{} } func (*LastError) ProtoMessage() {} func (*LastError) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{97} + return fileDescriptor_ca37af0df9a5bbd2, []int{98} } func (m *LastError) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2790,7 +2818,7 @@ var xxx_messageInfo_LastError proto.InternalMessageInfo func (m *LastMaintenance) Reset() { *m = LastMaintenance{} } func (*LastMaintenance) ProtoMessage() {} func (*LastMaintenance) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{98} + return fileDescriptor_ca37af0df9a5bbd2, []int{99} } func (m *LastMaintenance) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2818,7 +2846,7 @@ var xxx_messageInfo_LastMaintenance proto.InternalMessageInfo func (m *LastOperation) Reset() { *m = LastOperation{} } func (*LastOperation) ProtoMessage() {} func (*LastOperation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{99} + return fileDescriptor_ca37af0df9a5bbd2, []int{100} } func (m *LastOperation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2846,7 +2874,7 @@ var xxx_messageInfo_LastOperation proto.InternalMessageInfo func (m *Limits) Reset() { *m = Limits{} } func (*Limits) ProtoMessage() {} func (*Limits) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{100} + return fileDescriptor_ca37af0df9a5bbd2, []int{101} } func (m *Limits) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2874,7 +2902,7 @@ var xxx_messageInfo_Limits proto.InternalMessageInfo func (m *LoadBalancerServicesProxyProtocol) Reset() { *m = LoadBalancerServicesProxyProtocol{} } func (*LoadBalancerServicesProxyProtocol) ProtoMessage() {} func (*LoadBalancerServicesProxyProtocol) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{101} + return fileDescriptor_ca37af0df9a5bbd2, []int{102} } func (m *LoadBalancerServicesProxyProtocol) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2902,7 +2930,7 @@ var xxx_messageInfo_LoadBalancerServicesProxyProtocol proto.InternalMessageInfo func (m *Machine) Reset() { *m = Machine{} } func (*Machine) ProtoMessage() {} func (*Machine) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{102} + return fileDescriptor_ca37af0df9a5bbd2, []int{103} } func (m *Machine) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2930,7 +2958,7 @@ var xxx_messageInfo_Machine proto.InternalMessageInfo func (m *MachineControllerManagerSettings) Reset() { *m = MachineControllerManagerSettings{} } func (*MachineControllerManagerSettings) ProtoMessage() {} func (*MachineControllerManagerSettings) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{103} + return fileDescriptor_ca37af0df9a5bbd2, []int{104} } func (m *MachineControllerManagerSettings) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2958,7 +2986,7 @@ var xxx_messageInfo_MachineControllerManagerSettings proto.InternalMessageInfo func (m *MachineImage) Reset() { *m = MachineImage{} } func (*MachineImage) ProtoMessage() {} func (*MachineImage) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{104} + return fileDescriptor_ca37af0df9a5bbd2, []int{105} } func (m *MachineImage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2986,7 +3014,7 @@ var xxx_messageInfo_MachineImage proto.InternalMessageInfo func (m *MachineImageFlavor) Reset() { *m = MachineImageFlavor{} } func (*MachineImageFlavor) ProtoMessage() {} func (*MachineImageFlavor) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{105} + return fileDescriptor_ca37af0df9a5bbd2, []int{106} } func (m *MachineImageFlavor) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3014,7 +3042,7 @@ var xxx_messageInfo_MachineImageFlavor proto.InternalMessageInfo func (m *MachineImageVersion) Reset() { *m = MachineImageVersion{} } func (*MachineImageVersion) ProtoMessage() {} func (*MachineImageVersion) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{106} + return fileDescriptor_ca37af0df9a5bbd2, []int{107} } func (m *MachineImageVersion) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3042,7 +3070,7 @@ var xxx_messageInfo_MachineImageVersion proto.InternalMessageInfo func (m *MachineType) Reset() { *m = MachineType{} } func (*MachineType) ProtoMessage() {} func (*MachineType) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{107} + return fileDescriptor_ca37af0df9a5bbd2, []int{108} } func (m *MachineType) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3070,7 +3098,7 @@ var xxx_messageInfo_MachineType proto.InternalMessageInfo func (m *MachineTypeStorage) Reset() { *m = MachineTypeStorage{} } func (*MachineTypeStorage) ProtoMessage() {} func (*MachineTypeStorage) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{108} + return fileDescriptor_ca37af0df9a5bbd2, []int{109} } func (m *MachineTypeStorage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3098,7 +3126,7 @@ var xxx_messageInfo_MachineTypeStorage proto.InternalMessageInfo func (m *Maintenance) Reset() { *m = Maintenance{} } func (*Maintenance) ProtoMessage() {} func (*Maintenance) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{109} + return fileDescriptor_ca37af0df9a5bbd2, []int{110} } func (m *Maintenance) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3126,7 +3154,7 @@ var xxx_messageInfo_Maintenance proto.InternalMessageInfo func (m *MaintenanceAutoUpdate) Reset() { *m = MaintenanceAutoUpdate{} } func (*MaintenanceAutoUpdate) ProtoMessage() {} func (*MaintenanceAutoUpdate) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{110} + return fileDescriptor_ca37af0df9a5bbd2, []int{111} } func (m *MaintenanceAutoUpdate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3154,7 +3182,7 @@ var xxx_messageInfo_MaintenanceAutoUpdate proto.InternalMessageInfo func (m *MaintenanceTimeWindow) Reset() { *m = MaintenanceTimeWindow{} } func (*MaintenanceTimeWindow) ProtoMessage() {} func (*MaintenanceTimeWindow) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{111} + return fileDescriptor_ca37af0df9a5bbd2, []int{112} } func (m *MaintenanceTimeWindow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3182,7 +3210,7 @@ var xxx_messageInfo_MaintenanceTimeWindow proto.InternalMessageInfo func (m *ManualWorkerPoolRollout) Reset() { *m = ManualWorkerPoolRollout{} } func (*ManualWorkerPoolRollout) ProtoMessage() {} func (*ManualWorkerPoolRollout) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{112} + return fileDescriptor_ca37af0df9a5bbd2, []int{113} } func (m *ManualWorkerPoolRollout) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3210,7 +3238,7 @@ var xxx_messageInfo_ManualWorkerPoolRollout proto.InternalMessageInfo func (m *MemorySwapConfiguration) Reset() { *m = MemorySwapConfiguration{} } func (*MemorySwapConfiguration) ProtoMessage() {} func (*MemorySwapConfiguration) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{113} + return fileDescriptor_ca37af0df9a5bbd2, []int{114} } func (m *MemorySwapConfiguration) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3238,7 +3266,7 @@ var xxx_messageInfo_MemorySwapConfiguration proto.InternalMessageInfo func (m *Monitoring) Reset() { *m = Monitoring{} } func (*Monitoring) ProtoMessage() {} func (*Monitoring) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{114} + return fileDescriptor_ca37af0df9a5bbd2, []int{115} } func (m *Monitoring) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3266,7 +3294,7 @@ var xxx_messageInfo_Monitoring proto.InternalMessageInfo func (m *NamedResourceReference) Reset() { *m = NamedResourceReference{} } func (*NamedResourceReference) ProtoMessage() {} func (*NamedResourceReference) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{115} + return fileDescriptor_ca37af0df9a5bbd2, []int{116} } func (m *NamedResourceReference) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3294,7 +3322,7 @@ var xxx_messageInfo_NamedResourceReference proto.InternalMessageInfo func (m *NamespacedCloudProfile) Reset() { *m = NamespacedCloudProfile{} } func (*NamespacedCloudProfile) ProtoMessage() {} func (*NamespacedCloudProfile) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{116} + return fileDescriptor_ca37af0df9a5bbd2, []int{117} } func (m *NamespacedCloudProfile) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3322,7 +3350,7 @@ var xxx_messageInfo_NamespacedCloudProfile proto.InternalMessageInfo func (m *NamespacedCloudProfileList) Reset() { *m = NamespacedCloudProfileList{} } func (*NamespacedCloudProfileList) ProtoMessage() {} func (*NamespacedCloudProfileList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{117} + return fileDescriptor_ca37af0df9a5bbd2, []int{118} } func (m *NamespacedCloudProfileList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3350,7 +3378,7 @@ var xxx_messageInfo_NamespacedCloudProfileList proto.InternalMessageInfo func (m *NamespacedCloudProfileSpec) Reset() { *m = NamespacedCloudProfileSpec{} } func (*NamespacedCloudProfileSpec) ProtoMessage() {} func (*NamespacedCloudProfileSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{118} + return fileDescriptor_ca37af0df9a5bbd2, []int{119} } func (m *NamespacedCloudProfileSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3378,7 +3406,7 @@ var xxx_messageInfo_NamespacedCloudProfileSpec proto.InternalMessageInfo func (m *NamespacedCloudProfileStatus) Reset() { *m = NamespacedCloudProfileStatus{} } func (*NamespacedCloudProfileStatus) ProtoMessage() {} func (*NamespacedCloudProfileStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{119} + return fileDescriptor_ca37af0df9a5bbd2, []int{120} } func (m *NamespacedCloudProfileStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3406,7 +3434,7 @@ var xxx_messageInfo_NamespacedCloudProfileStatus proto.InternalMessageInfo func (m *Networking) Reset() { *m = Networking{} } func (*Networking) ProtoMessage() {} func (*Networking) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{120} + return fileDescriptor_ca37af0df9a5bbd2, []int{121} } func (m *Networking) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3434,7 +3462,7 @@ var xxx_messageInfo_Networking proto.InternalMessageInfo func (m *NetworkingStatus) Reset() { *m = NetworkingStatus{} } func (*NetworkingStatus) ProtoMessage() {} func (*NetworkingStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{121} + return fileDescriptor_ca37af0df9a5bbd2, []int{122} } func (m *NetworkingStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3462,7 +3490,7 @@ var xxx_messageInfo_NetworkingStatus proto.InternalMessageInfo func (m *NginxIngress) Reset() { *m = NginxIngress{} } func (*NginxIngress) ProtoMessage() {} func (*NginxIngress) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{122} + return fileDescriptor_ca37af0df9a5bbd2, []int{123} } func (m *NginxIngress) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3490,7 +3518,7 @@ var xxx_messageInfo_NginxIngress proto.InternalMessageInfo func (m *NodeLocalDNS) Reset() { *m = NodeLocalDNS{} } func (*NodeLocalDNS) ProtoMessage() {} func (*NodeLocalDNS) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{123} + return fileDescriptor_ca37af0df9a5bbd2, []int{124} } func (m *NodeLocalDNS) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3518,7 +3546,7 @@ var xxx_messageInfo_NodeLocalDNS proto.InternalMessageInfo func (m *OCIRepository) Reset() { *m = OCIRepository{} } func (*OCIRepository) ProtoMessage() {} func (*OCIRepository) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{124} + return fileDescriptor_ca37af0df9a5bbd2, []int{125} } func (m *OCIRepository) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3546,7 +3574,7 @@ var xxx_messageInfo_OCIRepository proto.InternalMessageInfo func (m *OIDCConfig) Reset() { *m = OIDCConfig{} } func (*OIDCConfig) ProtoMessage() {} func (*OIDCConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{125} + return fileDescriptor_ca37af0df9a5bbd2, []int{126} } func (m *OIDCConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3574,7 +3602,7 @@ var xxx_messageInfo_OIDCConfig proto.InternalMessageInfo func (m *ObservabilityRotation) Reset() { *m = ObservabilityRotation{} } func (*ObservabilityRotation) ProtoMessage() {} func (*ObservabilityRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{126} + return fileDescriptor_ca37af0df9a5bbd2, []int{127} } func (m *ObservabilityRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3602,7 +3630,7 @@ var xxx_messageInfo_ObservabilityRotation proto.InternalMessageInfo func (m *OpenIDConnectClientAuthentication) Reset() { *m = OpenIDConnectClientAuthentication{} } func (*OpenIDConnectClientAuthentication) ProtoMessage() {} func (*OpenIDConnectClientAuthentication) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{127} + return fileDescriptor_ca37af0df9a5bbd2, []int{128} } func (m *OpenIDConnectClientAuthentication) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3630,7 +3658,7 @@ var xxx_messageInfo_OpenIDConnectClientAuthentication proto.InternalMessageInfo func (m *PendingWorkerUpdates) Reset() { *m = PendingWorkerUpdates{} } func (*PendingWorkerUpdates) ProtoMessage() {} func (*PendingWorkerUpdates) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{128} + return fileDescriptor_ca37af0df9a5bbd2, []int{129} } func (m *PendingWorkerUpdates) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3658,7 +3686,7 @@ var xxx_messageInfo_PendingWorkerUpdates proto.InternalMessageInfo func (m *PendingWorkersRollout) Reset() { *m = PendingWorkersRollout{} } func (*PendingWorkersRollout) ProtoMessage() {} func (*PendingWorkersRollout) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{129} + return fileDescriptor_ca37af0df9a5bbd2, []int{130} } func (m *PendingWorkersRollout) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3686,7 +3714,7 @@ var xxx_messageInfo_PendingWorkersRollout proto.InternalMessageInfo func (m *Project) Reset() { *m = Project{} } func (*Project) ProtoMessage() {} func (*Project) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{130} + return fileDescriptor_ca37af0df9a5bbd2, []int{131} } func (m *Project) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3714,7 +3742,7 @@ var xxx_messageInfo_Project proto.InternalMessageInfo func (m *ProjectList) Reset() { *m = ProjectList{} } func (*ProjectList) ProtoMessage() {} func (*ProjectList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{131} + return fileDescriptor_ca37af0df9a5bbd2, []int{132} } func (m *ProjectList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3742,7 +3770,7 @@ var xxx_messageInfo_ProjectList proto.InternalMessageInfo func (m *ProjectMember) Reset() { *m = ProjectMember{} } func (*ProjectMember) ProtoMessage() {} func (*ProjectMember) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{132} + return fileDescriptor_ca37af0df9a5bbd2, []int{133} } func (m *ProjectMember) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3770,7 +3798,7 @@ var xxx_messageInfo_ProjectMember proto.InternalMessageInfo func (m *ProjectSpec) Reset() { *m = ProjectSpec{} } func (*ProjectSpec) ProtoMessage() {} func (*ProjectSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{133} + return fileDescriptor_ca37af0df9a5bbd2, []int{134} } func (m *ProjectSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3798,7 +3826,7 @@ var xxx_messageInfo_ProjectSpec proto.InternalMessageInfo func (m *ProjectStatus) Reset() { *m = ProjectStatus{} } func (*ProjectStatus) ProtoMessage() {} func (*ProjectStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{134} + return fileDescriptor_ca37af0df9a5bbd2, []int{135} } func (m *ProjectStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3826,7 +3854,7 @@ var xxx_messageInfo_ProjectStatus proto.InternalMessageInfo func (m *ProjectTolerations) Reset() { *m = ProjectTolerations{} } func (*ProjectTolerations) ProtoMessage() {} func (*ProjectTolerations) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{135} + return fileDescriptor_ca37af0df9a5bbd2, []int{136} } func (m *ProjectTolerations) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3854,7 +3882,7 @@ var xxx_messageInfo_ProjectTolerations proto.InternalMessageInfo func (m *Provider) Reset() { *m = Provider{} } func (*Provider) ProtoMessage() {} func (*Provider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{136} + return fileDescriptor_ca37af0df9a5bbd2, []int{137} } func (m *Provider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3882,7 +3910,7 @@ var xxx_messageInfo_Provider proto.InternalMessageInfo func (m *Quota) Reset() { *m = Quota{} } func (*Quota) ProtoMessage() {} func (*Quota) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{137} + return fileDescriptor_ca37af0df9a5bbd2, []int{138} } func (m *Quota) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3910,7 +3938,7 @@ var xxx_messageInfo_Quota proto.InternalMessageInfo func (m *QuotaList) Reset() { *m = QuotaList{} } func (*QuotaList) ProtoMessage() {} func (*QuotaList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{138} + return fileDescriptor_ca37af0df9a5bbd2, []int{139} } func (m *QuotaList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3938,7 +3966,7 @@ var xxx_messageInfo_QuotaList proto.InternalMessageInfo func (m *QuotaSpec) Reset() { *m = QuotaSpec{} } func (*QuotaSpec) ProtoMessage() {} func (*QuotaSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{139} + return fileDescriptor_ca37af0df9a5bbd2, []int{140} } func (m *QuotaSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3966,7 +3994,7 @@ var xxx_messageInfo_QuotaSpec proto.InternalMessageInfo func (m *Region) Reset() { *m = Region{} } func (*Region) ProtoMessage() {} func (*Region) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{140} + return fileDescriptor_ca37af0df9a5bbd2, []int{141} } func (m *Region) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3994,7 +4022,7 @@ var xxx_messageInfo_Region proto.InternalMessageInfo func (m *ResourceData) Reset() { *m = ResourceData{} } func (*ResourceData) ProtoMessage() {} func (*ResourceData) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{141} + return fileDescriptor_ca37af0df9a5bbd2, []int{142} } func (m *ResourceData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4022,7 +4050,7 @@ var xxx_messageInfo_ResourceData proto.InternalMessageInfo func (m *ResourceWatchCacheSize) Reset() { *m = ResourceWatchCacheSize{} } func (*ResourceWatchCacheSize) ProtoMessage() {} func (*ResourceWatchCacheSize) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{142} + return fileDescriptor_ca37af0df9a5bbd2, []int{143} } func (m *ResourceWatchCacheSize) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4050,7 +4078,7 @@ var xxx_messageInfo_ResourceWatchCacheSize proto.InternalMessageInfo func (m *SSHAccess) Reset() { *m = SSHAccess{} } func (*SSHAccess) ProtoMessage() {} func (*SSHAccess) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{143} + return fileDescriptor_ca37af0df9a5bbd2, []int{144} } func (m *SSHAccess) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4078,7 +4106,7 @@ var xxx_messageInfo_SSHAccess proto.InternalMessageInfo func (m *SecretBinding) Reset() { *m = SecretBinding{} } func (*SecretBinding) ProtoMessage() {} func (*SecretBinding) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{144} + return fileDescriptor_ca37af0df9a5bbd2, []int{145} } func (m *SecretBinding) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4106,7 +4134,7 @@ var xxx_messageInfo_SecretBinding proto.InternalMessageInfo func (m *SecretBindingList) Reset() { *m = SecretBindingList{} } func (*SecretBindingList) ProtoMessage() {} func (*SecretBindingList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{145} + return fileDescriptor_ca37af0df9a5bbd2, []int{146} } func (m *SecretBindingList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4134,7 +4162,7 @@ var xxx_messageInfo_SecretBindingList proto.InternalMessageInfo func (m *SecretBindingProvider) Reset() { *m = SecretBindingProvider{} } func (*SecretBindingProvider) ProtoMessage() {} func (*SecretBindingProvider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{146} + return fileDescriptor_ca37af0df9a5bbd2, []int{147} } func (m *SecretBindingProvider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4162,7 +4190,7 @@ var xxx_messageInfo_SecretBindingProvider proto.InternalMessageInfo func (m *Seed) Reset() { *m = Seed{} } func (*Seed) ProtoMessage() {} func (*Seed) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{147} + return fileDescriptor_ca37af0df9a5bbd2, []int{148} } func (m *Seed) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4190,7 +4218,7 @@ var xxx_messageInfo_Seed proto.InternalMessageInfo func (m *SeedDNS) Reset() { *m = SeedDNS{} } func (*SeedDNS) ProtoMessage() {} func (*SeedDNS) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{148} + return fileDescriptor_ca37af0df9a5bbd2, []int{149} } func (m *SeedDNS) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4218,7 +4246,7 @@ var xxx_messageInfo_SeedDNS proto.InternalMessageInfo func (m *SeedDNSProvider) Reset() { *m = SeedDNSProvider{} } func (*SeedDNSProvider) ProtoMessage() {} func (*SeedDNSProvider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{149} + return fileDescriptor_ca37af0df9a5bbd2, []int{150} } func (m *SeedDNSProvider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4246,7 +4274,7 @@ var xxx_messageInfo_SeedDNSProvider proto.InternalMessageInfo func (m *SeedDNSProviderConfig) Reset() { *m = SeedDNSProviderConfig{} } func (*SeedDNSProviderConfig) ProtoMessage() {} func (*SeedDNSProviderConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{150} + return fileDescriptor_ca37af0df9a5bbd2, []int{151} } func (m *SeedDNSProviderConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4274,7 +4302,7 @@ var xxx_messageInfo_SeedDNSProviderConfig proto.InternalMessageInfo func (m *SeedList) Reset() { *m = SeedList{} } func (*SeedList) ProtoMessage() {} func (*SeedList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{151} + return fileDescriptor_ca37af0df9a5bbd2, []int{152} } func (m *SeedList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4302,7 +4330,7 @@ var xxx_messageInfo_SeedList proto.InternalMessageInfo func (m *SeedNetworks) Reset() { *m = SeedNetworks{} } func (*SeedNetworks) ProtoMessage() {} func (*SeedNetworks) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{152} + return fileDescriptor_ca37af0df9a5bbd2, []int{153} } func (m *SeedNetworks) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4330,7 +4358,7 @@ var xxx_messageInfo_SeedNetworks proto.InternalMessageInfo func (m *SeedProvider) Reset() { *m = SeedProvider{} } func (*SeedProvider) ProtoMessage() {} func (*SeedProvider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{153} + return fileDescriptor_ca37af0df9a5bbd2, []int{154} } func (m *SeedProvider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4358,7 +4386,7 @@ var xxx_messageInfo_SeedProvider proto.InternalMessageInfo func (m *SeedSelector) Reset() { *m = SeedSelector{} } func (*SeedSelector) ProtoMessage() {} func (*SeedSelector) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{154} + return fileDescriptor_ca37af0df9a5bbd2, []int{155} } func (m *SeedSelector) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4386,7 +4414,7 @@ var xxx_messageInfo_SeedSelector proto.InternalMessageInfo func (m *SeedSettingDependencyWatchdog) Reset() { *m = SeedSettingDependencyWatchdog{} } func (*SeedSettingDependencyWatchdog) ProtoMessage() {} func (*SeedSettingDependencyWatchdog) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{155} + return fileDescriptor_ca37af0df9a5bbd2, []int{156} } func (m *SeedSettingDependencyWatchdog) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4414,7 +4442,7 @@ var xxx_messageInfo_SeedSettingDependencyWatchdog proto.InternalMessageInfo func (m *SeedSettingDependencyWatchdogProber) Reset() { *m = SeedSettingDependencyWatchdogProber{} } func (*SeedSettingDependencyWatchdogProber) ProtoMessage() {} func (*SeedSettingDependencyWatchdogProber) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{156} + return fileDescriptor_ca37af0df9a5bbd2, []int{157} } func (m *SeedSettingDependencyWatchdogProber) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4442,7 +4470,7 @@ var xxx_messageInfo_SeedSettingDependencyWatchdogProber proto.InternalMessageInf func (m *SeedSettingDependencyWatchdogWeeder) Reset() { *m = SeedSettingDependencyWatchdogWeeder{} } func (*SeedSettingDependencyWatchdogWeeder) ProtoMessage() {} func (*SeedSettingDependencyWatchdogWeeder) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{157} + return fileDescriptor_ca37af0df9a5bbd2, []int{158} } func (m *SeedSettingDependencyWatchdogWeeder) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4470,7 +4498,7 @@ var xxx_messageInfo_SeedSettingDependencyWatchdogWeeder proto.InternalMessageInf func (m *SeedSettingExcessCapacityReservation) Reset() { *m = SeedSettingExcessCapacityReservation{} } func (*SeedSettingExcessCapacityReservation) ProtoMessage() {} func (*SeedSettingExcessCapacityReservation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{158} + return fileDescriptor_ca37af0df9a5bbd2, []int{159} } func (m *SeedSettingExcessCapacityReservation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4500,7 +4528,7 @@ func (m *SeedSettingExcessCapacityReservationConfig) Reset() { } func (*SeedSettingExcessCapacityReservationConfig) ProtoMessage() {} func (*SeedSettingExcessCapacityReservationConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{159} + return fileDescriptor_ca37af0df9a5bbd2, []int{160} } func (m *SeedSettingExcessCapacityReservationConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4528,7 +4556,7 @@ var xxx_messageInfo_SeedSettingExcessCapacityReservationConfig proto.InternalMes func (m *SeedSettingLoadBalancerServices) Reset() { *m = SeedSettingLoadBalancerServices{} } func (*SeedSettingLoadBalancerServices) ProtoMessage() {} func (*SeedSettingLoadBalancerServices) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{160} + return fileDescriptor_ca37af0df9a5bbd2, []int{161} } func (m *SeedSettingLoadBalancerServices) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4558,7 +4586,7 @@ func (m *SeedSettingLoadBalancerServicesZonalIngress) Reset() { } func (*SeedSettingLoadBalancerServicesZonalIngress) ProtoMessage() {} func (*SeedSettingLoadBalancerServicesZonalIngress) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{161} + return fileDescriptor_ca37af0df9a5bbd2, []int{162} } func (m *SeedSettingLoadBalancerServicesZonalIngress) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4586,7 +4614,7 @@ var xxx_messageInfo_SeedSettingLoadBalancerServicesZonalIngress proto.InternalMe func (m *SeedSettingLoadBalancerServicesZones) Reset() { *m = SeedSettingLoadBalancerServicesZones{} } func (*SeedSettingLoadBalancerServicesZones) ProtoMessage() {} func (*SeedSettingLoadBalancerServicesZones) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{162} + return fileDescriptor_ca37af0df9a5bbd2, []int{163} } func (m *SeedSettingLoadBalancerServicesZones) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4614,7 +4642,7 @@ var xxx_messageInfo_SeedSettingLoadBalancerServicesZones proto.InternalMessageIn func (m *SeedSettingScheduling) Reset() { *m = SeedSettingScheduling{} } func (*SeedSettingScheduling) ProtoMessage() {} func (*SeedSettingScheduling) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{163} + return fileDescriptor_ca37af0df9a5bbd2, []int{164} } func (m *SeedSettingScheduling) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4642,7 +4670,7 @@ var xxx_messageInfo_SeedSettingScheduling proto.InternalMessageInfo func (m *SeedSettingTopologyAwareRouting) Reset() { *m = SeedSettingTopologyAwareRouting{} } func (*SeedSettingTopologyAwareRouting) ProtoMessage() {} func (*SeedSettingTopologyAwareRouting) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{164} + return fileDescriptor_ca37af0df9a5bbd2, []int{165} } func (m *SeedSettingTopologyAwareRouting) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4670,7 +4698,7 @@ var xxx_messageInfo_SeedSettingTopologyAwareRouting proto.InternalMessageInfo func (m *SeedSettingVerticalPodAutoscaler) Reset() { *m = SeedSettingVerticalPodAutoscaler{} } func (*SeedSettingVerticalPodAutoscaler) ProtoMessage() {} func (*SeedSettingVerticalPodAutoscaler) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{165} + return fileDescriptor_ca37af0df9a5bbd2, []int{166} } func (m *SeedSettingVerticalPodAutoscaler) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4698,7 +4726,7 @@ var xxx_messageInfo_SeedSettingVerticalPodAutoscaler proto.InternalMessageInfo func (m *SeedSettings) Reset() { *m = SeedSettings{} } func (*SeedSettings) ProtoMessage() {} func (*SeedSettings) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{166} + return fileDescriptor_ca37af0df9a5bbd2, []int{167} } func (m *SeedSettings) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4726,7 +4754,7 @@ var xxx_messageInfo_SeedSettings proto.InternalMessageInfo func (m *SeedSpec) Reset() { *m = SeedSpec{} } func (*SeedSpec) ProtoMessage() {} func (*SeedSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{167} + return fileDescriptor_ca37af0df9a5bbd2, []int{168} } func (m *SeedSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4754,7 +4782,7 @@ var xxx_messageInfo_SeedSpec proto.InternalMessageInfo func (m *SeedStatus) Reset() { *m = SeedStatus{} } func (*SeedStatus) ProtoMessage() {} func (*SeedStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{168} + return fileDescriptor_ca37af0df9a5bbd2, []int{169} } func (m *SeedStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4782,7 +4810,7 @@ var xxx_messageInfo_SeedStatus proto.InternalMessageInfo func (m *SeedTaint) Reset() { *m = SeedTaint{} } func (*SeedTaint) ProtoMessage() {} func (*SeedTaint) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{169} + return fileDescriptor_ca37af0df9a5bbd2, []int{170} } func (m *SeedTaint) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4810,7 +4838,7 @@ var xxx_messageInfo_SeedTaint proto.InternalMessageInfo func (m *SeedTemplate) Reset() { *m = SeedTemplate{} } func (*SeedTemplate) ProtoMessage() {} func (*SeedTemplate) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{170} + return fileDescriptor_ca37af0df9a5bbd2, []int{171} } func (m *SeedTemplate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4838,7 +4866,7 @@ var xxx_messageInfo_SeedTemplate proto.InternalMessageInfo func (m *SeedVolume) Reset() { *m = SeedVolume{} } func (*SeedVolume) ProtoMessage() {} func (*SeedVolume) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{171} + return fileDescriptor_ca37af0df9a5bbd2, []int{172} } func (m *SeedVolume) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4866,7 +4894,7 @@ var xxx_messageInfo_SeedVolume proto.InternalMessageInfo func (m *SeedVolumeProvider) Reset() { *m = SeedVolumeProvider{} } func (*SeedVolumeProvider) ProtoMessage() {} func (*SeedVolumeProvider) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{172} + return fileDescriptor_ca37af0df9a5bbd2, []int{173} } func (m *SeedVolumeProvider) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4894,7 +4922,7 @@ var xxx_messageInfo_SeedVolumeProvider proto.InternalMessageInfo func (m *ServiceAccountConfig) Reset() { *m = ServiceAccountConfig{} } func (*ServiceAccountConfig) ProtoMessage() {} func (*ServiceAccountConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{173} + return fileDescriptor_ca37af0df9a5bbd2, []int{174} } func (m *ServiceAccountConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4922,7 +4950,7 @@ var xxx_messageInfo_ServiceAccountConfig proto.InternalMessageInfo func (m *ServiceAccountKeyRotation) Reset() { *m = ServiceAccountKeyRotation{} } func (*ServiceAccountKeyRotation) ProtoMessage() {} func (*ServiceAccountKeyRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{174} + return fileDescriptor_ca37af0df9a5bbd2, []int{175} } func (m *ServiceAccountKeyRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4950,7 +4978,7 @@ var xxx_messageInfo_ServiceAccountKeyRotation proto.InternalMessageInfo func (m *Shoot) Reset() { *m = Shoot{} } func (*Shoot) ProtoMessage() {} func (*Shoot) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{175} + return fileDescriptor_ca37af0df9a5bbd2, []int{176} } func (m *Shoot) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4978,7 +5006,7 @@ var xxx_messageInfo_Shoot proto.InternalMessageInfo func (m *ShootAdvertisedAddress) Reset() { *m = ShootAdvertisedAddress{} } func (*ShootAdvertisedAddress) ProtoMessage() {} func (*ShootAdvertisedAddress) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{176} + return fileDescriptor_ca37af0df9a5bbd2, []int{177} } func (m *ShootAdvertisedAddress) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5006,7 +5034,7 @@ var xxx_messageInfo_ShootAdvertisedAddress proto.InternalMessageInfo func (m *ShootCredentials) Reset() { *m = ShootCredentials{} } func (*ShootCredentials) ProtoMessage() {} func (*ShootCredentials) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{177} + return fileDescriptor_ca37af0df9a5bbd2, []int{178} } func (m *ShootCredentials) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5034,7 +5062,7 @@ var xxx_messageInfo_ShootCredentials proto.InternalMessageInfo func (m *ShootCredentialsRotation) Reset() { *m = ShootCredentialsRotation{} } func (*ShootCredentialsRotation) ProtoMessage() {} func (*ShootCredentialsRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{178} + return fileDescriptor_ca37af0df9a5bbd2, []int{179} } func (m *ShootCredentialsRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5062,7 +5090,7 @@ var xxx_messageInfo_ShootCredentialsRotation proto.InternalMessageInfo func (m *ShootKubeconfigRotation) Reset() { *m = ShootKubeconfigRotation{} } func (*ShootKubeconfigRotation) ProtoMessage() {} func (*ShootKubeconfigRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{179} + return fileDescriptor_ca37af0df9a5bbd2, []int{180} } func (m *ShootKubeconfigRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5090,7 +5118,7 @@ var xxx_messageInfo_ShootKubeconfigRotation proto.InternalMessageInfo func (m *ShootList) Reset() { *m = ShootList{} } func (*ShootList) ProtoMessage() {} func (*ShootList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{180} + return fileDescriptor_ca37af0df9a5bbd2, []int{181} } func (m *ShootList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5118,7 +5146,7 @@ var xxx_messageInfo_ShootList proto.InternalMessageInfo func (m *ShootMachineImage) Reset() { *m = ShootMachineImage{} } func (*ShootMachineImage) ProtoMessage() {} func (*ShootMachineImage) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{181} + return fileDescriptor_ca37af0df9a5bbd2, []int{182} } func (m *ShootMachineImage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5146,7 +5174,7 @@ var xxx_messageInfo_ShootMachineImage proto.InternalMessageInfo func (m *ShootNetworks) Reset() { *m = ShootNetworks{} } func (*ShootNetworks) ProtoMessage() {} func (*ShootNetworks) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{182} + return fileDescriptor_ca37af0df9a5bbd2, []int{183} } func (m *ShootNetworks) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5174,7 +5202,7 @@ var xxx_messageInfo_ShootNetworks proto.InternalMessageInfo func (m *ShootSSHKeypairRotation) Reset() { *m = ShootSSHKeypairRotation{} } func (*ShootSSHKeypairRotation) ProtoMessage() {} func (*ShootSSHKeypairRotation) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{183} + return fileDescriptor_ca37af0df9a5bbd2, []int{184} } func (m *ShootSSHKeypairRotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5202,7 +5230,7 @@ var xxx_messageInfo_ShootSSHKeypairRotation proto.InternalMessageInfo func (m *ShootSpec) Reset() { *m = ShootSpec{} } func (*ShootSpec) ProtoMessage() {} func (*ShootSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{184} + return fileDescriptor_ca37af0df9a5bbd2, []int{185} } func (m *ShootSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5230,7 +5258,7 @@ var xxx_messageInfo_ShootSpec proto.InternalMessageInfo func (m *ShootState) Reset() { *m = ShootState{} } func (*ShootState) ProtoMessage() {} func (*ShootState) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{185} + return fileDescriptor_ca37af0df9a5bbd2, []int{186} } func (m *ShootState) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5258,7 +5286,7 @@ var xxx_messageInfo_ShootState proto.InternalMessageInfo func (m *ShootStateList) Reset() { *m = ShootStateList{} } func (*ShootStateList) ProtoMessage() {} func (*ShootStateList) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{186} + return fileDescriptor_ca37af0df9a5bbd2, []int{187} } func (m *ShootStateList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5286,7 +5314,7 @@ var xxx_messageInfo_ShootStateList proto.InternalMessageInfo func (m *ShootStateSpec) Reset() { *m = ShootStateSpec{} } func (*ShootStateSpec) ProtoMessage() {} func (*ShootStateSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{187} + return fileDescriptor_ca37af0df9a5bbd2, []int{188} } func (m *ShootStateSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5314,7 +5342,7 @@ var xxx_messageInfo_ShootStateSpec proto.InternalMessageInfo func (m *ShootStatus) Reset() { *m = ShootStatus{} } func (*ShootStatus) ProtoMessage() {} func (*ShootStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{188} + return fileDescriptor_ca37af0df9a5bbd2, []int{189} } func (m *ShootStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5342,7 +5370,7 @@ var xxx_messageInfo_ShootStatus proto.InternalMessageInfo func (m *ShootTemplate) Reset() { *m = ShootTemplate{} } func (*ShootTemplate) ProtoMessage() {} func (*ShootTemplate) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{189} + return fileDescriptor_ca37af0df9a5bbd2, []int{190} } func (m *ShootTemplate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5370,7 +5398,7 @@ var xxx_messageInfo_ShootTemplate proto.InternalMessageInfo func (m *StructuredAuthentication) Reset() { *m = StructuredAuthentication{} } func (*StructuredAuthentication) ProtoMessage() {} func (*StructuredAuthentication) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{190} + return fileDescriptor_ca37af0df9a5bbd2, []int{191} } func (m *StructuredAuthentication) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5398,7 +5426,7 @@ var xxx_messageInfo_StructuredAuthentication proto.InternalMessageInfo func (m *StructuredAuthorization) Reset() { *m = StructuredAuthorization{} } func (*StructuredAuthorization) ProtoMessage() {} func (*StructuredAuthorization) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{191} + return fileDescriptor_ca37af0df9a5bbd2, []int{192} } func (m *StructuredAuthorization) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5426,7 +5454,7 @@ var xxx_messageInfo_StructuredAuthorization proto.InternalMessageInfo func (m *SystemComponents) Reset() { *m = SystemComponents{} } func (*SystemComponents) ProtoMessage() {} func (*SystemComponents) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{192} + return fileDescriptor_ca37af0df9a5bbd2, []int{193} } func (m *SystemComponents) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5454,7 +5482,7 @@ var xxx_messageInfo_SystemComponents proto.InternalMessageInfo func (m *Toleration) Reset() { *m = Toleration{} } func (*Toleration) ProtoMessage() {} func (*Toleration) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{193} + return fileDescriptor_ca37af0df9a5bbd2, []int{194} } func (m *Toleration) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5482,7 +5510,7 @@ var xxx_messageInfo_Toleration proto.InternalMessageInfo func (m *VerticalPodAutoscaler) Reset() { *m = VerticalPodAutoscaler{} } func (*VerticalPodAutoscaler) ProtoMessage() {} func (*VerticalPodAutoscaler) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{194} + return fileDescriptor_ca37af0df9a5bbd2, []int{195} } func (m *VerticalPodAutoscaler) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5510,7 +5538,7 @@ var xxx_messageInfo_VerticalPodAutoscaler proto.InternalMessageInfo func (m *Volume) Reset() { *m = Volume{} } func (*Volume) ProtoMessage() {} func (*Volume) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{195} + return fileDescriptor_ca37af0df9a5bbd2, []int{196} } func (m *Volume) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5538,7 +5566,7 @@ var xxx_messageInfo_Volume proto.InternalMessageInfo func (m *VolumeType) Reset() { *m = VolumeType{} } func (*VolumeType) ProtoMessage() {} func (*VolumeType) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{196} + return fileDescriptor_ca37af0df9a5bbd2, []int{197} } func (m *VolumeType) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5566,7 +5594,7 @@ var xxx_messageInfo_VolumeType proto.InternalMessageInfo func (m *WatchCacheSizes) Reset() { *m = WatchCacheSizes{} } func (*WatchCacheSizes) ProtoMessage() {} func (*WatchCacheSizes) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{197} + return fileDescriptor_ca37af0df9a5bbd2, []int{198} } func (m *WatchCacheSizes) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5594,7 +5622,7 @@ var xxx_messageInfo_WatchCacheSizes proto.InternalMessageInfo func (m *Worker) Reset() { *m = Worker{} } func (*Worker) ProtoMessage() {} func (*Worker) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{198} + return fileDescriptor_ca37af0df9a5bbd2, []int{199} } func (m *Worker) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5622,7 +5650,7 @@ var xxx_messageInfo_Worker proto.InternalMessageInfo func (m *WorkerControlPlane) Reset() { *m = WorkerControlPlane{} } func (*WorkerControlPlane) ProtoMessage() {} func (*WorkerControlPlane) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{199} + return fileDescriptor_ca37af0df9a5bbd2, []int{200} } func (m *WorkerControlPlane) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5650,7 +5678,7 @@ var xxx_messageInfo_WorkerControlPlane proto.InternalMessageInfo func (m *WorkerKubernetes) Reset() { *m = WorkerKubernetes{} } func (*WorkerKubernetes) ProtoMessage() {} func (*WorkerKubernetes) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{200} + return fileDescriptor_ca37af0df9a5bbd2, []int{201} } func (m *WorkerKubernetes) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5678,7 +5706,7 @@ var xxx_messageInfo_WorkerKubernetes proto.InternalMessageInfo func (m *WorkerSystemComponents) Reset() { *m = WorkerSystemComponents{} } func (*WorkerSystemComponents) ProtoMessage() {} func (*WorkerSystemComponents) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{201} + return fileDescriptor_ca37af0df9a5bbd2, []int{202} } func (m *WorkerSystemComponents) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5706,7 +5734,7 @@ var xxx_messageInfo_WorkerSystemComponents proto.InternalMessageInfo func (m *WorkersSettings) Reset() { *m = WorkersSettings{} } func (*WorkersSettings) ProtoMessage() {} func (*WorkersSettings) Descriptor() ([]byte, []int) { - return fileDescriptor_ca37af0df9a5bbd2, []int{202} + return fileDescriptor_ca37af0df9a5bbd2, []int{203} } func (m *WorkersSettings) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -5797,6 +5825,7 @@ func init() { proto.RegisterType((*ETCD)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.ETCD") proto.RegisterType((*ETCDConfig)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.ETCDConfig") proto.RegisterType((*ETCDEncryptionKeyRotation)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.ETCDEncryptionKeyRotation") + proto.RegisterType((*EncryptionAtRest)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.EncryptionAtRest") proto.RegisterType((*EncryptionConfig)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.EncryptionConfig") proto.RegisterType((*ExpirableVersion)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.ExpirableVersion") proto.RegisterType((*ExposureClass)(nil), "github.com.gardener.gardener.pkg.apis.core.v1beta1.ExposureClass") @@ -5969,932 +5998,934 @@ func init() { } var fileDescriptor_ca37af0df9a5bbd2 = []byte{ - // 14794 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x6b, 0x70, 0x24, 0xd9, - 0x55, 0x20, 0xec, 0x2c, 0xbd, 0x8f, 0x1e, 0x2d, 0xdd, 0x56, 0x77, 0xab, 0xd5, 0x8f, 0x6a, 0xe7, - 0xd8, 0xa6, 0x87, 0xb1, 0xd5, 0xcc, 0x78, 0xec, 0xf1, 0x8c, 0x99, 0x87, 0x54, 0x25, 0x75, 0x97, - 0x5b, 0x52, 0x6b, 0x6e, 0x49, 0x33, 0xe3, 0x19, 0x18, 0x93, 0xca, 0xba, 0x2a, 0xe5, 0x74, 0x56, - 0x66, 0x4d, 0x66, 0x96, 0x5a, 0xea, 0xb1, 0xf1, 0x03, 0xcc, 0x67, 0x0f, 0x86, 0x0f, 0xf8, 0x00, - 0x7f, 0xb6, 0x21, 0x18, 0x4c, 0x60, 0xbe, 0x6f, 0xd9, 0x35, 0x1b, 0x10, 0x6c, 0x04, 0x10, 0x10, - 0x0b, 0x11, 0x2c, 0x86, 0xc0, 0x04, 0x81, 0x61, 0xd7, 0x2c, 0x50, 0x60, 0xad, 0x17, 0x36, 0x62, - 0x9f, 0xb1, 0x44, 0x2c, 0x11, 0xbd, 0x04, 0xbb, 0x71, 0x1f, 0x99, 0x79, 0xf3, 0x55, 0x2a, 0x65, - 0x49, 0xf2, 0xcc, 0xe2, 0x5f, 0x52, 0xdd, 0x73, 0xef, 0x39, 0xf7, 0x95, 0xe7, 0x9e, 0x7b, 0xee, - 0x79, 0xc0, 0x42, 0xdd, 0xf0, 0xb6, 0x5b, 0x9b, 0x73, 0xba, 0xdd, 0xb8, 0x56, 0xd7, 0x9c, 0x1a, - 0xb1, 0x88, 0x13, 0xfe, 0xd3, 0xbc, 0x5d, 0xbf, 0xa6, 0x35, 0x0d, 0xf7, 0x9a, 0x6e, 0x3b, 0xe4, - 0xda, 0xce, 0x83, 0x9b, 0xc4, 0xd3, 0x1e, 0xbc, 0x56, 0xa7, 0x30, 0xcd, 0x23, 0xb5, 0xb9, 0xa6, - 0x63, 0x7b, 0x36, 0x7a, 0x28, 0xc4, 0x31, 0xe7, 0x37, 0x0d, 0xff, 0x69, 0xde, 0xae, 0xcf, 0x51, - 0x1c, 0x73, 0x14, 0xc7, 0x9c, 0xc0, 0x31, 0xfb, 0x0e, 0x99, 0xae, 0x5d, 0xb7, 0xaf, 0x31, 0x54, - 0x9b, 0xad, 0x2d, 0xf6, 0x8b, 0xfd, 0x60, 0xff, 0x71, 0x12, 0xb3, 0xf7, 0xdf, 0x7e, 0x8f, 0x3b, - 0x67, 0xd8, 0xb4, 0x33, 0xd7, 0xb4, 0x96, 0x67, 0xbb, 0xba, 0x66, 0x1a, 0x56, 0xfd, 0xda, 0x4e, - 0xa2, 0x37, 0xb3, 0xaa, 0x54, 0x55, 0x74, 0xbb, 0x63, 0x1d, 0x67, 0x53, 0xd3, 0xd3, 0xea, 0xdc, - 0x08, 0xeb, 0x90, 0x5d, 0x8f, 0x58, 0xae, 0x61, 0x5b, 0xee, 0x3b, 0xe8, 0x48, 0x88, 0xb3, 0x23, - 0xcf, 0x4d, 0xa4, 0x42, 0x1a, 0xa6, 0x87, 0x43, 0x4c, 0x0d, 0x4d, 0xdf, 0x36, 0x2c, 0xe2, 0xec, - 0xf9, 0xcd, 0xaf, 0x39, 0xc4, 0xb5, 0x5b, 0x8e, 0x4e, 0x0e, 0xd5, 0xca, 0xbd, 0xd6, 0x20, 0x9e, - 0x96, 0x46, 0xeb, 0x5a, 0x56, 0x2b, 0xa7, 0x65, 0x79, 0x46, 0x23, 0x49, 0xe6, 0xdd, 0x07, 0x35, - 0x70, 0xf5, 0x6d, 0xd2, 0xd0, 0x12, 0xed, 0xde, 0x99, 0xd5, 0xae, 0xe5, 0x19, 0xe6, 0x35, 0xc3, - 0xf2, 0x5c, 0xcf, 0x89, 0x37, 0x52, 0x5f, 0x55, 0x60, 0x72, 0x7e, 0xad, 0x52, 0x65, 0x33, 0xb8, - 0x6c, 0xd7, 0xeb, 0x86, 0x55, 0x47, 0x0f, 0xc0, 0xc8, 0x0e, 0x71, 0x36, 0x6d, 0xd7, 0xf0, 0xf6, - 0x66, 0x94, 0x2b, 0xca, 0xd5, 0x81, 0x85, 0xf1, 0xfd, 0x76, 0x71, 0xe4, 0x19, 0xbf, 0x10, 0x87, - 0x70, 0x54, 0x81, 0xd3, 0xdb, 0x9e, 0xd7, 0x9c, 0xd7, 0x75, 0xe2, 0xba, 0x41, 0x8d, 0x99, 0x02, - 0x6b, 0x76, 0x6e, 0xbf, 0x5d, 0x3c, 0x7d, 0x63, 0x7d, 0x7d, 0x2d, 0x06, 0xc6, 0x69, 0x6d, 0xd4, - 0x5f, 0x54, 0x60, 0x2a, 0xe8, 0x0c, 0x26, 0x2f, 0xb7, 0x88, 0xeb, 0xb9, 0x08, 0xc3, 0xd9, 0x86, - 0xb6, 0xbb, 0x6a, 0x5b, 0x2b, 0x2d, 0x4f, 0xf3, 0x0c, 0xab, 0x5e, 0xb1, 0xb6, 0x4c, 0xa3, 0xbe, - 0xed, 0x89, 0xae, 0xcd, 0xee, 0xb7, 0x8b, 0x67, 0x57, 0x52, 0x6b, 0xe0, 0x8c, 0x96, 0xb4, 0xd3, - 0x0d, 0x6d, 0x37, 0x81, 0x50, 0xea, 0xf4, 0x4a, 0x12, 0x8c, 0xd3, 0xda, 0xa8, 0xef, 0x82, 0x29, - 0x3e, 0x0e, 0x4c, 0x5c, 0xcf, 0x31, 0x74, 0xcf, 0xb0, 0x2d, 0x74, 0x05, 0xfa, 0x2d, 0xad, 0x41, - 0x58, 0x0f, 0x47, 0x16, 0xc6, 0xbe, 0xd4, 0x2e, 0xbe, 0x69, 0xbf, 0x5d, 0xec, 0x5f, 0xd5, 0x1a, - 0x04, 0x33, 0x88, 0xfa, 0x3f, 0x0a, 0x70, 0x31, 0xd1, 0xee, 0x59, 0xc3, 0xdb, 0xbe, 0xd5, 0xa4, - 0xff, 0xb9, 0xe8, 0x07, 0x15, 0x98, 0xd2, 0xe2, 0x15, 0x18, 0xc2, 0xd1, 0x87, 0x16, 0xe7, 0x0e, - 0xff, 0x81, 0xcf, 0x25, 0xa8, 0x2d, 0x9c, 0x17, 0xfd, 0x4a, 0x0e, 0x00, 0x27, 0x49, 0xa3, 0x4f, - 0x28, 0x30, 0x64, 0xf3, 0xce, 0xcd, 0x14, 0xae, 0xf4, 0x5d, 0x1d, 0x7d, 0xe8, 0x3b, 0x8f, 0xa4, - 0x1b, 0xd2, 0xa0, 0xe7, 0xc4, 0xdf, 0x45, 0xcb, 0x73, 0xf6, 0x16, 0x4e, 0x89, 0xee, 0x0d, 0x89, - 0x52, 0xec, 0x93, 0x9f, 0x7d, 0x0c, 0xc6, 0xe4, 0x9a, 0x68, 0x12, 0xfa, 0x6e, 0x13, 0xbe, 0x55, - 0x47, 0x30, 0xfd, 0x17, 0x4d, 0xc3, 0xc0, 0x8e, 0x66, 0xb6, 0x08, 0x5b, 0xd2, 0x11, 0xcc, 0x7f, - 0x3c, 0x56, 0x78, 0x8f, 0xa2, 0x3e, 0x04, 0x03, 0xf3, 0xb5, 0x9a, 0x6d, 0xa1, 0xfb, 0x61, 0x88, - 0x58, 0xda, 0xa6, 0x49, 0x6a, 0xac, 0xe1, 0x70, 0x48, 0x6f, 0x91, 0x17, 0x63, 0x1f, 0xae, 0xfe, - 0x58, 0x01, 0x06, 0x59, 0x23, 0x17, 0xfd, 0x88, 0x02, 0xa7, 0x6f, 0xb7, 0x36, 0x89, 0x63, 0x11, - 0x8f, 0xb8, 0x65, 0xcd, 0xdd, 0xde, 0xb4, 0x35, 0xa7, 0x26, 0x16, 0xe6, 0x7a, 0x9e, 0x19, 0xb9, - 0x99, 0x44, 0xc7, 0xf7, 0x60, 0x0a, 0x00, 0xa7, 0x11, 0x47, 0x3b, 0x30, 0x66, 0xd5, 0x0d, 0x6b, - 0xb7, 0x62, 0xd5, 0x1d, 0xe2, 0xba, 0x6c, 0xd0, 0xa3, 0x0f, 0x3d, 0x95, 0xa7, 0x33, 0xab, 0x12, - 0x9e, 0x85, 0xc9, 0xfd, 0x76, 0x71, 0x4c, 0x2e, 0xc1, 0x11, 0x3a, 0xea, 0x3f, 0x28, 0x70, 0x6a, - 0xbe, 0xd6, 0x30, 0x5c, 0xca, 0x69, 0xd7, 0xcc, 0x56, 0xdd, 0xe8, 0x62, 0xeb, 0xa3, 0xa7, 0x61, - 0x50, 0xb7, 0xad, 0x2d, 0xa3, 0x2e, 0xfa, 0xf9, 0x8e, 0x39, 0xce, 0xb9, 0xe6, 0x64, 0xce, 0xc5, - 0xba, 0x27, 0x38, 0xde, 0x1c, 0xd6, 0xee, 0x2c, 0xfa, 0x0c, 0x7d, 0x01, 0xf6, 0xdb, 0xc5, 0xc1, - 0x12, 0x43, 0x80, 0x05, 0x22, 0x74, 0x15, 0x86, 0x6b, 0x86, 0xcb, 0x17, 0xb3, 0x8f, 0x2d, 0xe6, - 0xd8, 0x7e, 0xbb, 0x38, 0x5c, 0x16, 0x65, 0x38, 0x80, 0xa2, 0x65, 0x98, 0xa6, 0x33, 0xc8, 0xdb, - 0x55, 0x89, 0xee, 0x10, 0x8f, 0x76, 0x6d, 0xa6, 0x9f, 0x75, 0x77, 0x66, 0xbf, 0x5d, 0x9c, 0xbe, - 0x99, 0x02, 0xc7, 0xa9, 0xad, 0xd4, 0x25, 0x18, 0x9e, 0x37, 0x89, 0x43, 0x19, 0x02, 0x7a, 0x0c, - 0x26, 0x48, 0x43, 0x33, 0x4c, 0x4c, 0x74, 0x62, 0xec, 0x10, 0xc7, 0x9d, 0x51, 0xae, 0xf4, 0x5d, - 0x1d, 0x59, 0x40, 0xfb, 0xed, 0xe2, 0xc4, 0x62, 0x04, 0x82, 0x63, 0x35, 0xd5, 0x8f, 0x2a, 0x30, - 0x3a, 0xdf, 0xaa, 0x19, 0x1e, 0x1f, 0x17, 0x72, 0x60, 0x54, 0xa3, 0x3f, 0xd7, 0x6c, 0xd3, 0xd0, - 0xf7, 0xc4, 0xe6, 0x7a, 0x32, 0xd7, 0xe7, 0x16, 0xa2, 0x59, 0x38, 0xb5, 0xdf, 0x2e, 0x8e, 0x4a, - 0x05, 0x58, 0x26, 0xa2, 0x6e, 0x83, 0x0c, 0x43, 0xef, 0x87, 0x31, 0x3e, 0xdc, 0x15, 0xad, 0x89, - 0xc9, 0x96, 0xe8, 0xc3, 0x7d, 0xd2, 0x5a, 0xf9, 0x84, 0xe6, 0x6e, 0x6d, 0xbe, 0x44, 0x74, 0x0f, - 0x93, 0x2d, 0xe2, 0x10, 0x4b, 0x27, 0x7c, 0xdb, 0x94, 0xa4, 0xc6, 0x38, 0x82, 0x4a, 0xfd, 0x7f, - 0x14, 0xb8, 0x34, 0xdf, 0xf2, 0xb6, 0x6d, 0xc7, 0xb8, 0x4b, 0x9c, 0x70, 0xba, 0x03, 0x0c, 0xe8, - 0x09, 0x98, 0xd0, 0x82, 0x0a, 0xab, 0xe1, 0x76, 0x3a, 0x2b, 0xb6, 0xd3, 0xc4, 0x7c, 0x04, 0x8a, - 0x63, 0xb5, 0xd1, 0x43, 0x00, 0x6e, 0xb8, 0xb6, 0x8c, 0x07, 0x2c, 0x20, 0xd1, 0x16, 0xa4, 0x55, - 0x95, 0x6a, 0xa9, 0x7f, 0x49, 0x8f, 0xc2, 0x1d, 0xcd, 0x30, 0xb5, 0x4d, 0xc3, 0x34, 0xbc, 0xbd, - 0xe7, 0x6d, 0x8b, 0x74, 0xb1, 0x9b, 0x37, 0xe0, 0x5c, 0xcb, 0xd2, 0x78, 0x3b, 0x93, 0xac, 0xf0, - 0xfd, 0xbb, 0xbe, 0xd7, 0x24, 0x9c, 0x4b, 0x8e, 0x2c, 0x5c, 0xd8, 0x6f, 0x17, 0xcf, 0x6d, 0xa4, - 0x57, 0xc1, 0x59, 0x6d, 0xe9, 0xa9, 0x27, 0x81, 0x9e, 0xb1, 0xcd, 0x56, 0x43, 0x60, 0xed, 0x63, - 0x58, 0xd9, 0xa9, 0xb7, 0x91, 0x5a, 0x03, 0x67, 0xb4, 0x54, 0xbf, 0x50, 0x80, 0xc1, 0x05, 0x4d, - 0xbf, 0xdd, 0x6a, 0xa2, 0xb7, 0xc3, 0x70, 0xd3, 0xb1, 0x77, 0x8c, 0x1a, 0x71, 0xc4, 0xd8, 0x26, - 0xc5, 0xd8, 0x86, 0xd7, 0x44, 0x39, 0x0e, 0x6a, 0x20, 0x03, 0x26, 0xfc, 0xff, 0x4b, 0x3d, 0x7c, - 0xb9, 0xec, 0x4b, 0x58, 0x8b, 0x20, 0xc2, 0x31, 0xc4, 0x48, 0x85, 0x41, 0x87, 0xd4, 0xe9, 0x51, - 0xd7, 0xc7, 0xba, 0xc5, 0xbe, 0x76, 0xcc, 0x4a, 0xb0, 0x80, 0xa0, 0x0f, 0xc0, 0x84, 0xee, 0x90, - 0x1a, 0xb1, 0x3c, 0x43, 0x33, 0x5d, 0xba, 0x39, 0x07, 0xba, 0xdf, 0x9c, 0xac, 0x13, 0xa5, 0x48, - 0x73, 0x1c, 0x43, 0xa7, 0x7e, 0xa9, 0x00, 0x63, 0x7c, 0xa2, 0x16, 0x5a, 0xfa, 0x6d, 0xe2, 0xa1, - 0xef, 0x82, 0x61, 0x2a, 0xdf, 0xd5, 0x34, 0x4f, 0x13, 0x1f, 0xc2, 0xb7, 0x65, 0x0e, 0x9d, 0x7d, - 0x83, 0xb4, 0x76, 0x48, 0x7d, 0x85, 0x78, 0x5a, 0xb8, 0xff, 0xc2, 0x32, 0x1c, 0x60, 0x45, 0x5b, - 0xd0, 0xef, 0x36, 0x89, 0x2e, 0x26, 0xb6, 0x9c, 0xe7, 0x53, 0x97, 0x7b, 0x5c, 0x6d, 0x12, 0x3d, - 0xdc, 0xae, 0xf4, 0x17, 0x66, 0xf8, 0x91, 0x05, 0x83, 0xae, 0xa7, 0x79, 0x2d, 0x97, 0xcd, 0xef, - 0xe8, 0x43, 0x4b, 0x3d, 0x53, 0x62, 0xd8, 0x16, 0x26, 0x04, 0xad, 0x41, 0xfe, 0x1b, 0x0b, 0x2a, - 0xea, 0xbf, 0x51, 0x60, 0x52, 0xae, 0xbe, 0x6c, 0xb8, 0x1e, 0xfa, 0x8e, 0xc4, 0x74, 0xce, 0x75, - 0x37, 0x9d, 0xb4, 0x35, 0x9b, 0xcc, 0x60, 0xb7, 0xfa, 0x25, 0xd2, 0x54, 0x12, 0x18, 0x30, 0x3c, - 0xd2, 0xf0, 0xa5, 0x94, 0xa7, 0x7a, 0x1d, 0xe1, 0xc2, 0xb8, 0x20, 0x36, 0x50, 0xa1, 0x68, 0x31, - 0xc7, 0xae, 0x7e, 0x17, 0x4c, 0xcb, 0xb5, 0xfc, 0x7d, 0x4d, 0x59, 0x86, 0xb7, 0xd7, 0x4c, 0xb0, - 0x0c, 0xfa, 0x09, 0x62, 0x06, 0x41, 0x6f, 0x0b, 0xf6, 0x38, 0xe7, 0x4c, 0xc1, 0xdc, 0x45, 0xf7, - 0xb9, 0xfa, 0x91, 0xbe, 0xe8, 0xdc, 0xd1, 0x65, 0x44, 0x3b, 0xb1, 0x2f, 0x77, 0xf4, 0xa1, 0x1b, - 0xbd, 0x0e, 0xd0, 0xef, 0xfa, 0xeb, 0x85, 0x07, 0x5c, 0x85, 0x61, 0x97, 0x90, 0x9a, 0x74, 0x2e, - 0xb3, 0xd3, 0xbc, 0x2a, 0xca, 0x70, 0x00, 0x3d, 0x7e, 0x4e, 0xf0, 0x5a, 0x3f, 0xa0, 0xe4, 0x6e, - 0x97, 0x27, 0x83, 0x97, 0x88, 0xa5, 0xe8, 0x65, 0x32, 0xc4, 0x87, 0x13, 0x43, 0x8c, 0xee, 0xc2, - 0xb8, 0xa9, 0xb9, 0xde, 0xad, 0x26, 0xbd, 0xb7, 0xf9, 0x7b, 0x66, 0xf4, 0xa1, 0xf9, 0x3c, 0x8b, - 0xbe, 0x2c, 0x23, 0x5a, 0x98, 0xda, 0x6f, 0x17, 0xc7, 0x23, 0x45, 0x38, 0x4a, 0x0a, 0xbd, 0x04, - 0x23, 0xb4, 0x60, 0xd1, 0x71, 0x6c, 0x47, 0xf0, 0x8b, 0xc7, 0xf3, 0xd2, 0x65, 0x48, 0xf8, 0x3d, - 0x32, 0xf8, 0x89, 0x43, 0xf4, 0xe8, 0x7d, 0x80, 0xec, 0x4d, 0x76, 0x93, 0xaf, 0x5d, 0xe7, 0x97, - 0x54, 0x3a, 0x58, 0xba, 0xfc, 0x7d, 0x0b, 0xb3, 0x62, 0x5f, 0xa2, 0x5b, 0x89, 0x1a, 0x38, 0xa5, - 0x15, 0xba, 0x0d, 0x28, 0xb8, 0xe8, 0xf2, 0xd3, 0xfe, 0x80, 0xad, 0x11, 0x54, 0x12, 0x5b, 0xe3, - 0x2c, 0x25, 0x76, 0x3d, 0x81, 0x02, 0xa7, 0xa0, 0x55, 0x7f, 0xbb, 0x00, 0xa3, 0x7c, 0x8b, 0xf0, - 0xcb, 0xc8, 0xf1, 0x9f, 0x15, 0x24, 0x72, 0x56, 0x94, 0xf2, 0x7f, 0xfe, 0xac, 0xc3, 0x99, 0x47, - 0x45, 0x23, 0x76, 0x54, 0x2c, 0xf6, 0x4a, 0xa8, 0xf3, 0x49, 0xf1, 0x27, 0x0a, 0x9c, 0x92, 0x6a, - 0x9f, 0xc0, 0x41, 0x51, 0x8b, 0x1e, 0x14, 0x4f, 0xf6, 0x38, 0xbe, 0x8c, 0x73, 0xc2, 0x8e, 0x0c, - 0x8b, 0xf1, 0xf0, 0x87, 0x00, 0x36, 0x19, 0x3b, 0x91, 0x44, 0xdb, 0x60, 0xc9, 0x17, 0x02, 0x08, - 0x96, 0x6a, 0x45, 0x98, 0x62, 0xa1, 0x13, 0x53, 0x54, 0xff, 0x7d, 0x1f, 0x4c, 0x25, 0xa6, 0x3d, - 0xc9, 0x47, 0x94, 0x6f, 0x10, 0x1f, 0x29, 0x7c, 0x23, 0xf8, 0x48, 0x5f, 0x2e, 0x3e, 0xd2, 0xfd, - 0x41, 0xe4, 0x00, 0x6a, 0x18, 0x75, 0xde, 0xac, 0xea, 0x69, 0x8e, 0xb7, 0x6e, 0x34, 0x88, 0xe0, - 0x38, 0xdf, 0xda, 0xdd, 0x96, 0xa5, 0x2d, 0x38, 0xe3, 0x59, 0x49, 0x60, 0xc2, 0x29, 0xd8, 0xd5, - 0xef, 0x29, 0xc0, 0xd0, 0x82, 0xe6, 0xb2, 0x9e, 0x7e, 0x08, 0xc6, 0x04, 0xea, 0x4a, 0x43, 0xab, - 0x93, 0x5e, 0xd4, 0x11, 0x02, 0xe5, 0x8a, 0x84, 0x8e, 0xdf, 0xe8, 0xe4, 0x12, 0x1c, 0x21, 0x87, - 0xf6, 0x60, 0xb4, 0x11, 0xde, 0x5e, 0xc4, 0x12, 0x2f, 0xf5, 0x4e, 0x9d, 0x62, 0xe3, 0xd7, 0x56, - 0xa9, 0x00, 0xcb, 0xb4, 0xd4, 0x17, 0xe1, 0x74, 0x4a, 0x8f, 0xbb, 0xb8, 0xb8, 0xbd, 0x15, 0x86, - 0xe8, 0xdd, 0x3b, 0x14, 0xc3, 0x46, 0xf7, 0xdb, 0xc5, 0xa1, 0x67, 0x78, 0x11, 0xf6, 0x61, 0xea, - 0xbb, 0xa9, 0x00, 0x10, 0xef, 0x53, 0x37, 0x0a, 0xbe, 0x01, 0x80, 0xd2, 0x3c, 0xb6, 0x3d, 0xbe, - 0x95, 0x9e, 0x84, 0x81, 0xe6, 0xb6, 0xe6, 0xfa, 0x2d, 0xee, 0xf7, 0x59, 0xc5, 0x1a, 0x2d, 0xbc, - 0xd7, 0x2e, 0xce, 0xc8, 0x82, 0x88, 0x68, 0xc4, 0x60, 0x98, 0xb7, 0xa3, 0x3b, 0x8c, 0x6e, 0xf2, - 0x92, 0xdd, 0x68, 0x9a, 0x84, 0x42, 0xd9, 0x0e, 0x2b, 0xe4, 0xdb, 0x61, 0xcb, 0x09, 0x4c, 0x38, - 0x05, 0xbb, 0x4f, 0xb3, 0x62, 0x19, 0x9e, 0xa1, 0x05, 0x34, 0xfb, 0xf2, 0xd3, 0x8c, 0x62, 0xc2, - 0x29, 0xd8, 0xd1, 0xab, 0x0a, 0xcc, 0x46, 0x8b, 0x97, 0x0c, 0xcb, 0x70, 0xb7, 0x49, 0x8d, 0x11, - 0xef, 0x3f, 0x34, 0xf1, 0xcb, 0xfb, 0xed, 0xe2, 0xec, 0x72, 0x26, 0x46, 0xdc, 0x81, 0x1a, 0xfa, - 0x01, 0x05, 0x2e, 0xc4, 0xe6, 0xc5, 0x31, 0xea, 0x75, 0xe2, 0x88, 0xde, 0x1c, 0xfe, 0x03, 0x2f, - 0xee, 0xb7, 0x8b, 0x17, 0x96, 0xb3, 0x51, 0xe2, 0x4e, 0xf4, 0xd0, 0x4f, 0x2b, 0x70, 0xb6, 0x49, - 0xac, 0x9a, 0x61, 0xd5, 0x9f, 0xb5, 0x9d, 0xdb, 0xc4, 0x71, 0xb1, 0x6d, 0x9a, 0x76, 0xcb, 0x73, - 0x67, 0x06, 0xd9, 0x19, 0x56, 0xc9, 0xf3, 0xcd, 0xad, 0xa5, 0x61, 0x5c, 0xb8, 0x2c, 0xb6, 0xe8, - 0xd9, 0x54, 0xb0, 0x8b, 0x33, 0x3a, 0xa2, 0xfe, 0x96, 0x02, 0x7d, 0x25, 0x5c, 0x41, 0x0f, 0x44, - 0x3e, 0x91, 0x73, 0xf2, 0x27, 0x72, 0xaf, 0x5d, 0x1c, 0x2a, 0xe1, 0x8a, 0xf4, 0x31, 0xfe, 0x80, - 0x02, 0x53, 0xba, 0x6d, 0x79, 0x1a, 0x9d, 0x3b, 0xcc, 0x65, 0x65, 0xff, 0x5c, 0xce, 0x75, 0x19, - 0x2e, 0xc5, 0x90, 0x85, 0xca, 0xee, 0x38, 0xc4, 0xc5, 0x49, 0xca, 0xea, 0xe7, 0x15, 0x98, 0x2e, - 0x69, 0x4d, 0xa1, 0x0a, 0x2a, 0x93, 0x2d, 0x83, 0x6e, 0x90, 0x6e, 0x34, 0xfb, 0x68, 0x1b, 0x06, - 0x99, 0xb6, 0xd9, 0xed, 0xe5, 0x2e, 0x1f, 0xd2, 0x7e, 0x86, 0xe1, 0xe2, 0x7a, 0x10, 0xfe, 0x3f, - 0x16, 0xf8, 0xd5, 0xc7, 0x61, 0x32, 0x5e, 0x0f, 0x15, 0x7d, 0x99, 0x86, 0x2b, 0x1f, 0x47, 0xe2, - 0xe2, 0xc8, 0x63, 0xc3, 0xff, 0xef, 0x6b, 0xc5, 0x37, 0x7d, 0xe4, 0xcf, 0xaf, 0xbc, 0x49, 0xfd, - 0xaa, 0x02, 0x63, 0x25, 0xd3, 0x6e, 0xd5, 0xd6, 0x1c, 0x7b, 0xcb, 0x30, 0xc9, 0x1b, 0x43, 0xcb, - 0x21, 0xf7, 0x38, 0x4b, 0x74, 0x65, 0x5a, 0x07, 0xb9, 0xe2, 0x1b, 0x44, 0xeb, 0x20, 0x77, 0x39, - 0x43, 0x9a, 0x7c, 0x01, 0xce, 0xc8, 0xb5, 0x42, 0x95, 0xe9, 0x15, 0xe8, 0xbf, 0x6d, 0x58, 0xb5, - 0xf8, 0xc6, 0xbc, 0x69, 0x58, 0x35, 0xcc, 0x20, 0xc1, 0xd6, 0x2d, 0x64, 0x9e, 0x59, 0x7f, 0x3b, - 0x12, 0x9d, 0x36, 0x26, 0xac, 0x5e, 0x85, 0x61, 0x5d, 0x5b, 0x68, 0x59, 0x35, 0x33, 0xd8, 0xf5, - 0x74, 0x0a, 0x4a, 0xf3, 0xbc, 0x0c, 0x07, 0x50, 0x74, 0x17, 0x20, 0x7c, 0x9d, 0xe8, 0x45, 0x08, - 0x08, 0x1f, 0x3e, 0xaa, 0xc4, 0xf3, 0x0c, 0xab, 0xee, 0x86, 0xfb, 0x2a, 0x84, 0x61, 0x89, 0x1a, - 0xfa, 0x10, 0x8c, 0xcb, 0x12, 0x09, 0x57, 0x93, 0xe6, 0x5c, 0x86, 0x88, 0xe8, 0x73, 0x46, 0x10, - 0x1e, 0x97, 0x4b, 0x5d, 0x1c, 0xa5, 0x86, 0xf6, 0x02, 0xf9, 0x8b, 0x2b, 0x69, 0xfb, 0xf3, 0xdf, - 0x28, 0x64, 0xd1, 0x67, 0x5a, 0x10, 0x1f, 0x8b, 0x28, 0x8d, 0x23, 0xa4, 0x52, 0x14, 0x33, 0x03, - 0xc7, 0xa5, 0x98, 0x21, 0x30, 0xc4, 0x55, 0x53, 0xfe, 0x71, 0xf3, 0x58, 0x9e, 0x01, 0x72, 0x2d, - 0x57, 0xf8, 0xdc, 0xc6, 0x7f, 0xbb, 0xd8, 0xc7, 0x8d, 0x76, 0x60, 0x8c, 0x0a, 0xd6, 0x55, 0x62, - 0x12, 0xdd, 0xb3, 0x9d, 0x99, 0xa1, 0xfc, 0xcf, 0x59, 0x55, 0x09, 0x0f, 0x97, 0x62, 0xe5, 0x12, - 0x1c, 0xa1, 0x13, 0x68, 0xee, 0x86, 0x33, 0x35, 0x77, 0x2d, 0x18, 0xdd, 0x91, 0x54, 0xf1, 0x23, - 0x6c, 0x12, 0x9e, 0xc8, 0xd3, 0xb1, 0x50, 0x2f, 0xbf, 0x70, 0x5a, 0x10, 0x1a, 0x95, 0x75, 0xf8, - 0x32, 0x1d, 0xb4, 0x09, 0x43, 0x9b, 0x5c, 0x06, 0x9d, 0x01, 0x36, 0x17, 0xef, 0xed, 0x41, 0xb4, - 0xe6, 0x72, 0xae, 0xf8, 0x81, 0x7d, 0xc4, 0xe8, 0x45, 0x18, 0x34, 0x8d, 0x86, 0xe1, 0xb9, 0x33, - 0xa3, 0x8c, 0x44, 0xae, 0xa5, 0x5d, 0x66, 0x18, 0xf8, 0x61, 0xc5, 0xff, 0xc7, 0x02, 0x2b, 0xfa, - 0xb4, 0x02, 0xa7, 0xc5, 0x36, 0x0c, 0x0e, 0x2d, 0x83, 0xb8, 0x33, 0x63, 0x6c, 0x0e, 0x6f, 0xf4, - 0x76, 0x48, 0x86, 0x07, 0xf4, 0xc2, 0x05, 0x31, 0x9b, 0xa7, 0x57, 0x92, 0xc4, 0x70, 0x5a, 0x0f, - 0xd4, 0x57, 0xa7, 0x60, 0xaa, 0x64, 0xb6, 0x5c, 0x8f, 0x38, 0xf3, 0xc2, 0x92, 0x85, 0x38, 0xe8, - 0x63, 0x0a, 0x9c, 0x65, 0xff, 0x96, 0xed, 0x3b, 0x56, 0x99, 0x98, 0xda, 0xde, 0xfc, 0x16, 0xad, - 0x51, 0xab, 0x1d, 0xee, 0xf0, 0x28, 0xb7, 0xc4, 0x35, 0x99, 0xbd, 0xd8, 0x54, 0x53, 0x31, 0xe2, - 0x0c, 0x4a, 0xe8, 0xfb, 0x15, 0x38, 0x9f, 0x02, 0x2a, 0x13, 0x93, 0x78, 0xbe, 0xf0, 0x7f, 0xd8, - 0x7e, 0x5c, 0xda, 0x6f, 0x17, 0xcf, 0x57, 0xb3, 0x90, 0xe2, 0x6c, 0x7a, 0xe8, 0x07, 0x15, 0x98, - 0x4d, 0x81, 0x2e, 0x69, 0x86, 0xd9, 0x72, 0xfc, 0x7b, 0xc1, 0x61, 0xbb, 0xc3, 0xc4, 0xf3, 0x6a, - 0x26, 0x56, 0xdc, 0x81, 0x22, 0xfa, 0x30, 0x9c, 0x09, 0xa0, 0x1b, 0x96, 0x45, 0x48, 0x2d, 0x72, - 0x4b, 0x38, 0x6c, 0x57, 0xce, 0xef, 0xb7, 0x8b, 0x67, 0xaa, 0x69, 0x08, 0x71, 0x3a, 0x1d, 0x54, - 0x87, 0x4b, 0x21, 0xc0, 0x33, 0x4c, 0xe3, 0x2e, 0xbf, 0xc8, 0x6c, 0x3b, 0xc4, 0xdd, 0xb6, 0xcd, - 0x1a, 0x63, 0xc5, 0xca, 0xc2, 0x9b, 0xf7, 0xdb, 0xc5, 0x4b, 0xd5, 0x4e, 0x15, 0x71, 0x67, 0x3c, - 0xa8, 0x06, 0x63, 0xae, 0xae, 0x59, 0x15, 0xcb, 0x23, 0xce, 0x8e, 0x66, 0xce, 0x0c, 0xe6, 0x1a, - 0x20, 0x67, 0x80, 0x12, 0x1e, 0x1c, 0xc1, 0x8a, 0xde, 0x03, 0xc3, 0x64, 0xb7, 0xa9, 0x59, 0x35, - 0xc2, 0x99, 0xee, 0xc8, 0xc2, 0x45, 0x7a, 0xd4, 0x2f, 0x8a, 0xb2, 0x7b, 0xed, 0xe2, 0x98, 0xff, - 0xff, 0x8a, 0x5d, 0x23, 0x38, 0xa8, 0x8d, 0x3e, 0x08, 0xd3, 0xcc, 0xd4, 0xa6, 0x46, 0xd8, 0x11, - 0xe2, 0xfa, 0x77, 0xc5, 0xe1, 0x5c, 0xfd, 0x64, 0xcf, 0xf0, 0x2b, 0x29, 0xf8, 0x70, 0x2a, 0x15, - 0xba, 0x0c, 0x0d, 0x6d, 0xf7, 0xba, 0xa3, 0xe9, 0x64, 0xab, 0x65, 0xae, 0x13, 0xa7, 0x61, 0x58, - 0x5c, 0x59, 0x42, 0x74, 0xdb, 0xaa, 0x51, 0x46, 0xad, 0x5c, 0x1d, 0xe0, 0xcb, 0xb0, 0xd2, 0xa9, - 0x22, 0xee, 0x8c, 0x07, 0x3d, 0x0c, 0x63, 0x46, 0xdd, 0xb2, 0x1d, 0xb2, 0xae, 0x19, 0x96, 0xe7, - 0xce, 0x00, 0x13, 0xb2, 0xd9, 0xb4, 0x56, 0xa4, 0x72, 0x1c, 0xa9, 0x85, 0x76, 0x00, 0x59, 0xe4, - 0xce, 0x9a, 0x5d, 0x63, 0x5b, 0x60, 0xa3, 0xc9, 0x36, 0xb2, 0x60, 0xb3, 0x87, 0x9d, 0x1a, 0x76, - 0x95, 0x5e, 0x4d, 0x60, 0xc3, 0x29, 0x14, 0xd0, 0x12, 0xa0, 0x86, 0xb6, 0xbb, 0xd8, 0x68, 0x7a, - 0x7b, 0x0b, 0x2d, 0xf3, 0xb6, 0xe0, 0x1a, 0x63, 0x6c, 0x2e, 0xb8, 0xa2, 0x29, 0x01, 0xc5, 0x29, - 0x2d, 0x90, 0x06, 0x17, 0xf8, 0x78, 0xca, 0x1a, 0x69, 0xd8, 0x96, 0x4b, 0x3c, 0x57, 0xda, 0xa4, - 0x33, 0xe3, 0xcc, 0xe0, 0x82, 0x5d, 0x6c, 0x2b, 0xd9, 0xd5, 0x70, 0x27, 0x1c, 0x51, 0x93, 0xb3, - 0x89, 0x03, 0x4c, 0xce, 0x1e, 0x81, 0x71, 0xd7, 0xd3, 0x1c, 0xaf, 0xd5, 0x14, 0xcb, 0x70, 0x8a, - 0x2d, 0x03, 0xd3, 0x43, 0x56, 0x65, 0x00, 0x8e, 0xd6, 0xa3, 0xcb, 0xc7, 0x95, 0xcd, 0xa2, 0xdd, - 0x64, 0xb8, 0x7c, 0x55, 0xa9, 0x1c, 0x47, 0x6a, 0xa1, 0x0d, 0x38, 0xd7, 0xd0, 0x76, 0x83, 0xcf, - 0x77, 0x4d, 0x73, 0x34, 0xd3, 0x24, 0xa6, 0xe1, 0x36, 0x66, 0xa6, 0x58, 0x4f, 0xd9, 0x0b, 0xff, - 0x4a, 0x7a, 0x15, 0x9c, 0xd5, 0x56, 0xd8, 0xa0, 0x95, 0x1d, 0xcd, 0x88, 0xa0, 0x44, 0x11, 0x1b, - 0xb4, 0x38, 0x18, 0xa7, 0xb5, 0x41, 0x3f, 0xa9, 0x40, 0x91, 0x9d, 0x7f, 0x9a, 0x49, 0x3f, 0x8e, - 0xeb, 0x8e, 0xdd, 0x6a, 0x2e, 0x68, 0xfa, 0x6d, 0x7b, 0x6b, 0xcb, 0xdf, 0x38, 0x33, 0xa7, 0x73, - 0x6d, 0xb7, 0xfb, 0xf6, 0xdb, 0xc5, 0x62, 0xa5, 0x33, 0x6a, 0x7c, 0x10, 0x6d, 0xf4, 0xc3, 0x0a, - 0x5c, 0x10, 0x1f, 0x6e, 0x6a, 0xdf, 0xa6, 0x73, 0xf5, 0x8d, 0xed, 0xb8, 0x95, 0x6c, 0xb4, 0xb8, - 0x13, 0x4d, 0xf4, 0xa3, 0x0a, 0x5c, 0xb4, 0x62, 0x40, 0x4c, 0x5c, 0xc2, 0x74, 0xab, 0x76, 0xcb, - 0x9b, 0x39, 0x93, 0xab, 0x53, 0x57, 0xf6, 0xdb, 0xc5, 0x8b, 0xab, 0x1d, 0xf0, 0xe2, 0x8e, 0x54, - 0xd5, 0xff, 0xde, 0x0f, 0x33, 0x09, 0x61, 0xc4, 0xb7, 0x09, 0x3c, 0xf0, 0xb8, 0x51, 0x8e, 0xe8, - 0xb8, 0x69, 0xc2, 0x95, 0xa0, 0xc2, 0xf5, 0x66, 0x2b, 0x95, 0x56, 0x81, 0xd1, 0x7a, 0xcb, 0x7e, - 0xbb, 0x78, 0xa5, 0x7a, 0x40, 0x5d, 0x7c, 0x20, 0xb6, 0xec, 0xa3, 0xbc, 0xef, 0x84, 0x8e, 0xf2, - 0x0f, 0xc2, 0xb4, 0x04, 0x70, 0x88, 0x56, 0xdb, 0xeb, 0x41, 0x94, 0x60, 0x27, 0x58, 0x35, 0x05, - 0x1f, 0x4e, 0xa5, 0x92, 0x79, 0x7e, 0x0e, 0x9c, 0xc4, 0xf9, 0xa9, 0xb6, 0xfb, 0x60, 0xa4, 0x64, - 0x5b, 0x35, 0xae, 0xe2, 0x7a, 0x30, 0x62, 0xc0, 0x70, 0x49, 0xbe, 0x06, 0xdd, 0x6b, 0x17, 0xc7, - 0x83, 0x8a, 0xd2, 0xbd, 0xe8, 0xd1, 0xe0, 0xa9, 0x90, 0x2b, 0x17, 0xde, 0x1c, 0x7d, 0xe3, 0xbb, - 0xd7, 0x2e, 0x9e, 0x0a, 0x9a, 0x45, 0x9f, 0xfd, 0xe8, 0xe1, 0x68, 0x6a, 0xae, 0xb7, 0xee, 0x68, - 0x96, 0x6b, 0xf4, 0xa0, 0x63, 0x0e, 0xde, 0x76, 0x96, 0x13, 0xd8, 0x70, 0x0a, 0x05, 0xf4, 0x12, - 0x4c, 0xd0, 0xd2, 0x8d, 0x66, 0x4d, 0xf3, 0x48, 0x4e, 0xd5, 0x72, 0x60, 0x8e, 0xb6, 0x1c, 0xc1, - 0x84, 0x63, 0x98, 0xb9, 0xc1, 0x87, 0xe6, 0xda, 0x16, 0x5b, 0xcf, 0x88, 0xc1, 0x07, 0x2d, 0xc5, - 0x02, 0x8a, 0xee, 0x87, 0xa1, 0x06, 0x71, 0x5d, 0xad, 0x4e, 0x98, 0x80, 0x37, 0x12, 0xde, 0x91, - 0x57, 0x78, 0x31, 0xf6, 0xe1, 0xe8, 0xed, 0x30, 0xa0, 0xdb, 0x35, 0xe2, 0xce, 0x0c, 0xb1, 0x33, - 0x8c, 0x1e, 0xe7, 0x03, 0x25, 0x5a, 0x70, 0xaf, 0x5d, 0x1c, 0x61, 0x2f, 0x61, 0xf4, 0x17, 0xe6, - 0x95, 0xd4, 0x9f, 0x52, 0x60, 0x32, 0xae, 0xf7, 0xec, 0xc2, 0x50, 0xe5, 0xe4, 0x6c, 0x3e, 0xd4, - 0x4f, 0x2b, 0x30, 0x46, 0x7b, 0xe8, 0xd8, 0xe6, 0x9a, 0xa9, 0x59, 0x04, 0x7d, 0x9f, 0x02, 0x93, - 0xdb, 0x46, 0x7d, 0x5b, 0x36, 0xc9, 0x13, 0x37, 0xaf, 0x5c, 0x7a, 0xc3, 0x1b, 0x31, 0x5c, 0x0b, - 0xd3, 0xfb, 0xed, 0xe2, 0x64, 0xbc, 0x14, 0x27, 0x68, 0xaa, 0x7f, 0x55, 0x80, 0x73, 0x72, 0xcf, - 0xe6, 0x43, 0x6f, 0x07, 0xf4, 0x27, 0x0a, 0x40, 0xc3, 0xb0, 0xe6, 0x4d, 0xd3, 0xbe, 0xc3, 0xec, - 0x88, 0xe9, 0x5d, 0xf6, 0x85, 0xbc, 0xfa, 0xea, 0x14, 0x0a, 0x73, 0x2b, 0x01, 0x76, 0xfe, 0xc6, - 0xfc, 0x9c, 0xaf, 0x07, 0x0b, 0x01, 0xf7, 0xda, 0xc5, 0x62, 0xd2, 0xc5, 0x62, 0x0e, 0x0b, 0x3f, - 0x86, 0x65, 0xc3, 0xf5, 0x3e, 0xf6, 0x97, 0x1d, 0xab, 0xf0, 0xc7, 0xe6, 0x70, 0x20, 0xb3, 0x0d, - 0x38, 0x15, 0x23, 0x9c, 0x62, 0x63, 0x5d, 0x96, 0x6d, 0xac, 0x0f, 0x60, 0x52, 0x73, 0xbe, 0x57, - 0xc5, 0xdc, 0xd3, 0x2d, 0xcd, 0xf2, 0xe8, 0x4c, 0x4b, 0x36, 0xd9, 0x7f, 0x51, 0x80, 0x69, 0x31, - 0x01, 0x26, 0xbd, 0x6d, 0x36, 0x4d, 0x7b, 0xaf, 0x41, 0xac, 0x93, 0xb0, 0xbb, 0xf3, 0x3f, 0x82, - 0x42, 0xe6, 0x47, 0xd0, 0x48, 0x7c, 0x04, 0x7d, 0x79, 0x3e, 0x82, 0x80, 0x57, 0x1c, 0xa0, 0x63, - 0xc3, 0x70, 0xd6, 0xb0, 0x68, 0x47, 0xaf, 0xb3, 0x0d, 0x13, 0x5a, 0xc7, 0x32, 0xfe, 0x34, 0xcc, - 0xd5, 0x08, 0x95, 0xd4, 0x1a, 0x38, 0xa3, 0xa5, 0xfa, 0x37, 0x0a, 0xcc, 0xa4, 0xcd, 0xef, 0x09, - 0xa8, 0xc5, 0x1b, 0x51, 0xb5, 0xf8, 0x8d, 0x1e, 0xbe, 0x8d, 0x48, 0xd7, 0x33, 0xd4, 0xe3, 0x7f, - 0x5d, 0x80, 0xb3, 0x61, 0xf5, 0x8a, 0xe5, 0x7a, 0x9a, 0x69, 0x72, 0x81, 0xef, 0xf8, 0xf7, 0x52, - 0x33, 0xf2, 0xba, 0xb1, 0xda, 0xdb, 0x50, 0xe5, 0xbe, 0x67, 0x9a, 0xe8, 0xec, 0xc6, 0x4c, 0x74, - 0xd6, 0x8e, 0x90, 0x66, 0x67, 0x6b, 0x9d, 0xff, 0xa8, 0xc0, 0x6c, 0x7a, 0xc3, 0x13, 0xd8, 0x54, - 0x76, 0x74, 0x53, 0xbd, 0xef, 0xe8, 0x46, 0x9d, 0xb1, 0xad, 0x7e, 0xb1, 0x90, 0x35, 0x5a, 0xf6, - 0x44, 0xb2, 0x05, 0xa7, 0x1c, 0x52, 0x37, 0x5c, 0x4f, 0x5c, 0x3c, 0x0e, 0x67, 0x2e, 0xef, 0x3f, - 0x8d, 0x9e, 0xc2, 0x51, 0x1c, 0x38, 0x8e, 0x14, 0xad, 0xc2, 0x90, 0x4b, 0x48, 0x8d, 0xe2, 0x2f, - 0x74, 0x8f, 0x3f, 0x10, 0x22, 0xaa, 0xbc, 0x2d, 0xf6, 0x91, 0xa0, 0xef, 0x80, 0xf1, 0x5a, 0xf0, - 0x45, 0x51, 0xac, 0x7d, 0xdd, 0x63, 0x65, 0xb7, 0xed, 0xb2, 0xdc, 0x1a, 0x47, 0x91, 0xa9, 0x7f, - 0xaf, 0xc0, 0xc5, 0x4e, 0x7b, 0x0b, 0xbd, 0x0c, 0xa0, 0xfb, 0x52, 0xa1, 0x2b, 0x0e, 0xcf, 0xc7, - 0x73, 0xae, 0x25, 0xc7, 0x12, 0x7e, 0xa0, 0x41, 0x91, 0x8b, 0x25, 0x22, 0x29, 0x86, 0x9b, 0x85, - 0x63, 0x32, 0xdc, 0x54, 0xff, 0x93, 0x22, 0xb3, 0x22, 0x79, 0x6d, 0xdf, 0x68, 0xac, 0x48, 0xee, - 0x7b, 0xe6, 0x93, 0xeb, 0x57, 0x0a, 0x70, 0x25, 0xbd, 0x89, 0x74, 0x9e, 0x3f, 0x05, 0x83, 0x4d, - 0xee, 0xd2, 0xc2, 0xad, 0xfb, 0xaf, 0x52, 0xce, 0xc2, 0x1d, 0x4e, 0xee, 0xb5, 0x8b, 0xb3, 0x69, - 0x8c, 0x5e, 0xb8, 0xaa, 0x88, 0x76, 0xc8, 0x88, 0xbd, 0x0d, 0x71, 0xa1, 0xfd, 0x9d, 0x5d, 0x32, - 0x17, 0x6d, 0x93, 0x98, 0x5d, 0x3f, 0x07, 0x7d, 0x54, 0x81, 0x89, 0xc8, 0x8e, 0x76, 0x67, 0x06, - 0xd8, 0x1e, 0xcd, 0x65, 0x33, 0x17, 0xf9, 0x54, 0x42, 0x69, 0x20, 0x52, 0xec, 0xe2, 0x18, 0xc1, - 0x18, 0x9b, 0x95, 0x67, 0xf5, 0x0d, 0xc7, 0x66, 0xe5, 0xce, 0x67, 0xb0, 0xd9, 0x9f, 0x28, 0x64, - 0x8d, 0x96, 0xb1, 0xd9, 0x3b, 0x30, 0xe2, 0x8b, 0x91, 0x3e, 0xbb, 0x58, 0xea, 0xb5, 0x4f, 0x1c, - 0xdd, 0xc2, 0x94, 0xe8, 0xcf, 0x88, 0x5f, 0xe2, 0xe2, 0x90, 0x16, 0xfa, 0x5e, 0x05, 0x20, 0x5c, - 0x18, 0xf1, 0x51, 0xad, 0x1f, 0xdd, 0x74, 0x48, 0x62, 0xcd, 0x04, 0xfd, 0xa4, 0xa5, 0x4d, 0x21, - 0xd1, 0x55, 0xff, 0x75, 0x3f, 0xa0, 0x64, 0xdf, 0xbb, 0x7b, 0xf9, 0x3f, 0x40, 0xc8, 0x6d, 0xc2, - 0xa4, 0x43, 0xa5, 0x45, 0xdd, 0x30, 0x89, 0xaf, 0x00, 0xcb, 0xa7, 0xf9, 0x60, 0xd7, 0x2a, 0x1c, - 0xc3, 0x85, 0x13, 0xd8, 0xd1, 0x5b, 0x61, 0xa8, 0xe9, 0x18, 0x0d, 0xcd, 0xd9, 0x63, 0x97, 0xe2, - 0x61, 0xfe, 0x2c, 0xb9, 0xc6, 0x8b, 0xb0, 0x0f, 0x43, 0x1f, 0x84, 0x11, 0xd3, 0xd8, 0x22, 0xfa, - 0x9e, 0x6e, 0x12, 0xf1, 0xea, 0x71, 0xeb, 0x68, 0xd6, 0x7c, 0xd9, 0x47, 0x2b, 0x8c, 0x49, 0xfd, - 0x9f, 0x38, 0x24, 0x88, 0x2a, 0x70, 0xfa, 0x0e, 0x33, 0x6f, 0x32, 0x89, 0xeb, 0x56, 0x5b, 0xcd, - 0xa6, 0xed, 0x78, 0xa4, 0xc6, 0xde, 0x46, 0x86, 0xb9, 0x8e, 0xf6, 0xd9, 0x24, 0x18, 0xa7, 0xb5, - 0x41, 0x8f, 0x03, 0x68, 0x2d, 0xcf, 0xe6, 0xbe, 0xa5, 0x33, 0xc3, 0xec, 0xd6, 0x7e, 0x89, 0x2e, - 0xf6, 0x7c, 0x50, 0x7a, 0xaf, 0x5d, 0x1c, 0x15, 0xba, 0x3f, 0xb6, 0x34, 0x52, 0x03, 0xf4, 0x02, - 0x4c, 0xeb, 0x1c, 0x54, 0xb2, 0x1b, 0x4d, 0xcd, 0x33, 0xc4, 0x8d, 0x78, 0x84, 0x21, 0xfa, 0x96, - 0xfd, 0x76, 0x71, 0xba, 0x94, 0x02, 0x8f, 0xa3, 0x4c, 0x45, 0xa2, 0xbe, 0x5a, 0x80, 0x0b, 0x1d, - 0x26, 0x08, 0x61, 0xfa, 0xe1, 0x89, 0xf5, 0x13, 0xdb, 0xec, 0x61, 0xfe, 0xb1, 0x88, 0xc2, 0x7b, - 0xed, 0xe2, 0x7d, 0x1d, 0x10, 0x54, 0xe9, 0x3e, 0x27, 0xf5, 0x3d, 0x1c, 0xa2, 0x41, 0x15, 0x18, - 0xac, 0x85, 0xcf, 0x98, 0x23, 0x0b, 0x0f, 0xd2, 0xa3, 0x80, 0x3f, 0x38, 0x74, 0x8b, 0x4d, 0x20, - 0x40, 0xcb, 0x30, 0xc4, 0xcd, 0x63, 0x89, 0x38, 0x56, 0x1e, 0x62, 0x2a, 0x13, 0x5e, 0xd4, 0x2d, - 0x32, 0x1f, 0x85, 0xfa, 0x77, 0x0a, 0x0c, 0x95, 0x6c, 0x87, 0x94, 0x57, 0xab, 0x68, 0x0f, 0x46, - 0xa5, 0xe0, 0x06, 0x82, 0xc5, 0xe6, 0xe4, 0x39, 0x0c, 0xa3, 0x74, 0xb5, 0xf7, 0xdd, 0x31, 0x83, - 0x02, 0x2c, 0xd3, 0x42, 0x2f, 0xd3, 0x39, 0xbf, 0xe3, 0x18, 0x1e, 0x25, 0xdc, 0x8b, 0xbd, 0x14, - 0x27, 0x8c, 0x7d, 0x5c, 0x7c, 0xb7, 0x07, 0x3f, 0x71, 0x48, 0x45, 0x5d, 0xa3, 0xec, 0x25, 0xde, - 0x4d, 0xf4, 0x18, 0xf4, 0x37, 0xec, 0x9a, 0xbf, 0xee, 0x6f, 0xf3, 0x99, 0xc7, 0x8a, 0x5d, 0xa3, - 0x73, 0x7b, 0x36, 0xd9, 0x82, 0x3d, 0x0d, 0xb2, 0x36, 0xea, 0x2a, 0x4c, 0xc6, 0xe9, 0xa3, 0xc7, - 0x60, 0x42, 0xb7, 0x1b, 0x0d, 0xdb, 0xaa, 0xb6, 0xb6, 0xb6, 0x8c, 0x5d, 0x12, 0xf1, 0x93, 0x2d, - 0x45, 0x20, 0x38, 0x56, 0x53, 0xfd, 0x9c, 0x02, 0x7d, 0x74, 0x5d, 0x54, 0x18, 0xac, 0xd9, 0x0d, - 0xcd, 0xb0, 0x44, 0xaf, 0x98, 0xc1, 0x41, 0x99, 0x95, 0x60, 0x01, 0x41, 0x4d, 0x18, 0xf1, 0x25, - 0xb2, 0x9e, 0x2c, 0xfc, 0xcb, 0xab, 0xd5, 0xc0, 0x41, 0x2a, 0x38, 0x26, 0xfc, 0x12, 0x17, 0x87, - 0x44, 0x54, 0x0d, 0xa6, 0xca, 0xab, 0xd5, 0x8a, 0xa5, 0x9b, 0xad, 0x1a, 0x59, 0xdc, 0x65, 0x7f, - 0x28, 0x9f, 0x33, 0x78, 0x89, 0x18, 0x27, 0xe3, 0x73, 0xa2, 0x12, 0xf6, 0x61, 0xb4, 0x1a, 0xe1, - 0x2d, 0x84, 0xdb, 0x28, 0xab, 0x26, 0x90, 0x60, 0x1f, 0xa6, 0x7e, 0xb5, 0x00, 0xa3, 0x52, 0x87, - 0x90, 0x09, 0x43, 0x7c, 0xb8, 0x6e, 0x2f, 0xa1, 0x01, 0x12, 0xbd, 0xe6, 0xd4, 0xf9, 0x84, 0xba, - 0xd8, 0x27, 0x21, 0xf3, 0xec, 0x42, 0x07, 0x9e, 0x3d, 0x17, 0xf1, 0xbe, 0xe5, 0x9f, 0xe4, 0x44, - 0xb6, 0xe7, 0x2d, 0xba, 0x28, 0x8e, 0x27, 0x6e, 0x62, 0x3f, 0x1c, 0x3b, 0x9a, 0xb6, 0x60, 0xe0, - 0xae, 0x6d, 0x11, 0x57, 0xe8, 0xc2, 0x8f, 0x68, 0x80, 0xcc, 0x30, 0xf2, 0x79, 0x8a, 0x17, 0x73, - 0xf4, 0xea, 0x4f, 0x2b, 0x00, 0x65, 0xcd, 0xd3, 0xb8, 0x15, 0x4e, 0x17, 0x86, 0x9e, 0x17, 0x23, - 0xa7, 0xea, 0x70, 0xc2, 0xc9, 0xaf, 0xdf, 0x35, 0xee, 0xfa, 0xc3, 0x0f, 0xa4, 0x75, 0x8e, 0xbd, - 0x6a, 0xdc, 0x25, 0x98, 0xc1, 0xd1, 0x03, 0x30, 0x42, 0x2c, 0xdd, 0xd9, 0x6b, 0xd2, 0x83, 0x85, - 0xab, 0x78, 0xd8, 0x17, 0xba, 0xe8, 0x17, 0xe2, 0x10, 0xae, 0x3e, 0x08, 0xd1, 0x2b, 0x57, 0x17, - 0x76, 0xe8, 0xff, 0xa0, 0xc0, 0xb9, 0x72, 0x4b, 0x33, 0xe7, 0x9b, 0x74, 0xa3, 0x6a, 0xe6, 0x92, - 0xcd, 0xcd, 0x39, 0xe8, 0x3d, 0xe4, 0xed, 0x30, 0xec, 0x0b, 0x39, 0x71, 0x2f, 0x60, 0x9f, 0x51, - 0xe2, 0xa0, 0x06, 0xd2, 0x60, 0xd8, 0xf5, 0xc5, 0xee, 0x42, 0x0f, 0x62, 0xb7, 0x4f, 0x22, 0x10, - 0xbb, 0x03, 0xb4, 0x5c, 0xf9, 0xc5, 0x16, 0xa8, 0x4a, 0x9c, 0x1d, 0x43, 0x27, 0xf3, 0xba, 0x6e, - 0xb7, 0x2c, 0xcf, 0x15, 0x5e, 0xfd, 0x42, 0xf9, 0x95, 0x56, 0x03, 0x67, 0xb4, 0x54, 0xbf, 0xa4, - 0x40, 0xff, 0xe2, 0x7a, 0xa9, 0x8c, 0xbe, 0x03, 0xfa, 0x03, 0x96, 0x91, 0xd3, 0x6a, 0x8b, 0xe2, - 0xe1, 0x7a, 0x3a, 0xbe, 0xde, 0x2b, 0x94, 0xe1, 0x30, 0xac, 0x68, 0x13, 0x06, 0xc9, 0x0e, 0xa1, - 0x5d, 0x2d, 0x1c, 0x09, 0x7e, 0xc6, 0xd2, 0x16, 0x19, 0x46, 0x2c, 0x30, 0xab, 0x9f, 0x52, 0x00, - 0xc2, 0x2a, 0xe8, 0xbb, 0xd3, 0x4e, 0xa7, 0x9b, 0x47, 0xa8, 0x7d, 0xee, 0x7c, 0x44, 0xa9, 0xbf, - 0x31, 0x00, 0xe7, 0x69, 0x77, 0xc4, 0x56, 0x35, 0x6c, 0xeb, 0x26, 0xd9, 0xfb, 0xa6, 0xc7, 0xc3, - 0x37, 0x3d, 0x1e, 0x8e, 0xd0, 0xe3, 0xe1, 0x05, 0x38, 0x4f, 0xb7, 0x9c, 0x00, 0x13, 0x66, 0xfe, - 0xb5, 0xe6, 0x90, 0xa6, 0xe6, 0x90, 0x1a, 0xbb, 0x0f, 0x0c, 0x73, 0x83, 0xb6, 0xf9, 0xac, 0x4a, - 0x38, 0xbb, 0xbd, 0xfa, 0x24, 0x4c, 0x86, 0x7b, 0x57, 0x7c, 0x54, 0x0f, 0xc4, 0x2f, 0x99, 0x23, - 0xbe, 0xc4, 0x94, 0xbc, 0x18, 0xaa, 0xf7, 0x14, 0x98, 0x5c, 0xdc, 0x6d, 0x1a, 0x0e, 0x0b, 0xb5, - 0xc0, 0x3d, 0x86, 0xd0, 0xfd, 0xa1, 0x63, 0x91, 0x12, 0x7d, 0xc5, 0x8b, 0x3b, 0x17, 0xa1, 0x2d, - 0x98, 0x20, 0xac, 0x39, 0xbb, 0x05, 0x6a, 0x5e, 0x9e, 0xed, 0xcd, 0xe3, 0x8b, 0x44, 0xb0, 0xe0, - 0x18, 0x56, 0x54, 0x85, 0x09, 0xdd, 0xd4, 0x5c, 0xd7, 0xd8, 0x32, 0xf4, 0xd0, 0x21, 0x6e, 0x64, - 0xe1, 0x01, 0x26, 0x73, 0x45, 0x20, 0xf7, 0xda, 0xc5, 0x33, 0xa2, 0x9f, 0x51, 0x00, 0x8e, 0xa1, - 0x50, 0x3f, 0x53, 0x80, 0xf1, 0xc5, 0xdd, 0xa6, 0xed, 0xb6, 0x1c, 0xc2, 0xaa, 0x9e, 0x80, 0x5e, - 0xeb, 0x7e, 0x18, 0xda, 0xd6, 0xac, 0x9a, 0x49, 0x1c, 0x71, 0xec, 0x06, 0x73, 0x7b, 0x83, 0x17, - 0x63, 0x1f, 0x8e, 0x5e, 0x01, 0x70, 0xf5, 0x6d, 0x52, 0x6b, 0x31, 0xe6, 0xd8, 0x97, 0x9f, 0x39, - 0x46, 0xc6, 0x58, 0x0d, 0x50, 0x0a, 0x91, 0x26, 0xf8, 0x8d, 0x25, 0x72, 0xea, 0x9f, 0x2a, 0x30, - 0x15, 0x69, 0x77, 0x02, 0xea, 0x9a, 0xad, 0xa8, 0xba, 0x66, 0xbe, 0xe7, 0xb1, 0x66, 0x68, 0x69, - 0x3e, 0x51, 0x80, 0x73, 0x19, 0x73, 0x92, 0x30, 0xdd, 0x56, 0x4e, 0xc8, 0x74, 0xbb, 0x05, 0xa3, - 0x9e, 0x6d, 0x0a, 0xbf, 0x4d, 0x7f, 0x06, 0x72, 0x1d, 0xc1, 0xeb, 0x01, 0x9a, 0xd0, 0x30, 0x3b, - 0x2c, 0x73, 0xb1, 0x4c, 0x47, 0xfd, 0x2d, 0x05, 0x46, 0x02, 0xad, 0xf0, 0xeb, 0xea, 0x41, 0xbd, - 0xfb, 0x90, 0x48, 0xea, 0xef, 0x17, 0xe0, 0x6c, 0x80, 0xdb, 0x67, 0x73, 0x55, 0x8f, 0xf2, 0x8d, - 0x83, 0x55, 0x4b, 0x17, 0x23, 0x4e, 0x25, 0xc3, 0x49, 0x1f, 0xcb, 0x66, 0xcb, 0x69, 0xda, 0xae, - 0x2f, 0x07, 0xf3, 0x0b, 0x03, 0x2f, 0xc2, 0x3e, 0x0c, 0xad, 0xc2, 0x80, 0x4b, 0xe9, 0x89, 0xb3, - 0xee, 0x90, 0xb3, 0xc1, 0x44, 0x79, 0xd6, 0x5f, 0xcc, 0xd1, 0xa0, 0x57, 0x64, 0x1e, 0x3e, 0x90, - 0x5f, 0x79, 0x49, 0x47, 0x52, 0x0b, 0x24, 0xe1, 0xf0, 0xe9, 0xa3, 0x93, 0xb2, 0x50, 0x5d, 0x86, - 0x49, 0x61, 0x9f, 0xcc, 0xb7, 0x8d, 0xa5, 0x13, 0xf4, 0x9e, 0xc8, 0xce, 0x78, 0x4b, 0xcc, 0xa4, - 0x66, 0x3a, 0x5e, 0x3f, 0xdc, 0x31, 0xaa, 0x0b, 0xc3, 0xd7, 0x45, 0x27, 0xd1, 0x2c, 0x14, 0x0c, - 0x7f, 0x2d, 0x40, 0xe0, 0x28, 0x54, 0xca, 0xb8, 0x60, 0x74, 0xe1, 0xdc, 0x23, 0x1f, 0x4b, 0x7d, - 0x9d, 0x8f, 0x25, 0xf5, 0xeb, 0x05, 0x98, 0xf6, 0xa9, 0xfa, 0x63, 0x2c, 0x8b, 0xd7, 0xf2, 0x03, - 0x2e, 0x45, 0x07, 0xab, 0x1a, 0x6f, 0x41, 0x3f, 0x63, 0x80, 0xb9, 0x5e, 0xd1, 0x03, 0x84, 0xb4, - 0x3b, 0x98, 0x21, 0x42, 0x1f, 0x84, 0x41, 0x93, 0xde, 0x30, 0x7c, 0xaf, 0x9b, 0x5c, 0x8a, 0xd9, - 0xb4, 0xe1, 0xf2, 0x8b, 0x8b, 0x88, 0x46, 0x17, 0x3c, 0x84, 0xf2, 0x42, 0x2c, 0x68, 0xce, 0x3e, - 0x0a, 0xa3, 0x52, 0xb5, 0x43, 0x85, 0xa2, 0xfb, 0x5c, 0x01, 0x66, 0x6e, 0x10, 0xb3, 0x91, 0x6a, - 0xfa, 0x50, 0x84, 0x01, 0x7d, 0x5b, 0x73, 0x78, 0x94, 0xc3, 0x31, 0xbe, 0xc9, 0x4b, 0xb4, 0x00, - 0xf3, 0x72, 0x7a, 0xe1, 0x88, 0xf8, 0x19, 0x3e, 0x21, 0xcd, 0x64, 0x18, 0xfe, 0xf2, 0x03, 0x41, - 0x7c, 0xcc, 0x70, 0xe0, 0x91, 0x0a, 0xf4, 0x78, 0x79, 0x5f, 0xf5, 0xd6, 0x6a, 0x9a, 0x87, 0x21, - 0xba, 0x0b, 0xe3, 0xb6, 0x6e, 0x60, 0xd2, 0xb4, 0x5d, 0xc3, 0xb3, 0x9d, 0x3d, 0xb1, 0x68, 0xb9, - 0x8e, 0x96, 0x5b, 0xa5, 0x4a, 0x88, 0x88, 0x3f, 0x1f, 0x46, 0x8a, 0x70, 0x94, 0x94, 0xfa, 0x45, - 0x05, 0x46, 0x6f, 0x18, 0x9b, 0xc4, 0xe1, 0x26, 0xd8, 0x4c, 0x43, 0x12, 0x89, 0xd7, 0x37, 0x9a, - 0x16, 0xab, 0x0f, 0xed, 0xc2, 0x88, 0x38, 0x87, 0x03, 0x07, 0xd2, 0xeb, 0xf9, 0xec, 0x85, 0x02, - 0xd2, 0xe2, 0x7c, 0x93, 0x3e, 0x7c, 0xbf, 0xc4, 0xc5, 0x21, 0x31, 0xf5, 0x15, 0x38, 0x9d, 0xd2, - 0x88, 0x2e, 0x24, 0xb3, 0x42, 0x16, 0x1f, 0x8d, 0xcf, 0xad, 0xe8, 0x42, 0xb2, 0x72, 0x74, 0x1e, - 0xfa, 0x88, 0x55, 0x13, 0x5f, 0xcc, 0xd0, 0x7e, 0xbb, 0xd8, 0xb7, 0x68, 0xd5, 0x30, 0x2d, 0xa3, - 0x4c, 0xdc, 0xb4, 0x23, 0x12, 0x1b, 0x63, 0xe2, 0xcb, 0xa2, 0x0c, 0x07, 0x50, 0x66, 0xe1, 0x15, - 0x37, 0x66, 0xa2, 0x37, 0x8b, 0xc9, 0xad, 0x18, 0x6f, 0xe9, 0xc5, 0x86, 0x2a, 0xce, 0xa7, 0x16, - 0x66, 0xc4, 0x84, 0x24, 0x38, 0x1e, 0x4e, 0xd0, 0x55, 0x7f, 0xb5, 0x1f, 0x2e, 0xdd, 0xb0, 0x1d, - 0xe3, 0xae, 0x6d, 0x79, 0x9a, 0xb9, 0x66, 0xd7, 0x42, 0xfb, 0x56, 0x71, 0x64, 0x7d, 0x5c, 0x81, - 0x73, 0x7a, 0xb3, 0x25, 0xcc, 0x8d, 0x85, 0x89, 0xe8, 0x1a, 0x71, 0x0c, 0x3b, 0xaf, 0xcf, 0x0d, - 0xb3, 0xcc, 0x2e, 0xad, 0x6d, 0xa4, 0xa1, 0xc4, 0x59, 0xb4, 0x98, 0xeb, 0x4f, 0xcd, 0xbe, 0x63, - 0xb1, 0xce, 0x55, 0x3d, 0x36, 0x9b, 0x77, 0xc3, 0x45, 0xc8, 0xe9, 0xfa, 0x53, 0x4e, 0xc5, 0x88, - 0x33, 0x28, 0xa1, 0x0f, 0xc3, 0x19, 0x61, 0x56, 0x8d, 0x89, 0x56, 0x33, 0x2c, 0xe2, 0xba, 0xdc, - 0x6f, 0xa0, 0x07, 0xdf, 0x96, 0x4a, 0x1a, 0x42, 0x9c, 0x4e, 0x07, 0xbd, 0x08, 0xe0, 0xee, 0x59, - 0xba, 0x98, 0xff, 0x7c, 0x86, 0xa8, 0x5c, 0x44, 0x0e, 0xb0, 0x60, 0x09, 0x23, 0xbd, 0x68, 0x79, - 0xc1, 0xa6, 0x1c, 0x64, 0xc6, 0xc4, 0xec, 0xa2, 0x15, 0xee, 0xa1, 0x10, 0xae, 0x7e, 0x4a, 0x81, - 0x89, 0x8a, 0xb5, 0x66, 0x6a, 0x3a, 0xe1, 0x76, 0x95, 0x2e, 0xba, 0x06, 0x23, 0x6e, 0xf0, 0x22, - 0xc3, 0x39, 0x42, 0xf8, 0x7d, 0x06, 0x6f, 0x31, 0x61, 0x1d, 0x66, 0x70, 0x6f, 0x58, 0xe2, 0xb0, - 0x5b, 0xb2, 0x1d, 0x8e, 0x48, 0x7c, 0x77, 0xdc, 0xe0, 0x3e, 0x09, 0xc6, 0x69, 0x6d, 0xd4, 0x5f, - 0x50, 0x60, 0x3a, 0xda, 0x1d, 0x61, 0xd2, 0xf0, 0xe3, 0x0a, 0x4c, 0x47, 0xfc, 0xe2, 0x05, 0xb8, - 0x97, 0x50, 0x5d, 0x6b, 0x29, 0xf8, 0xb8, 0x89, 0x6f, 0x1a, 0x04, 0xa7, 0xd2, 0x57, 0xff, 0xa9, - 0x02, 0x43, 0x22, 0x6c, 0x27, 0x7a, 0x5b, 0x4c, 0x7b, 0x1e, 0x1c, 0x6d, 0x31, 0x0d, 0xfa, 0x1e, - 0xb3, 0xcf, 0x10, 0x47, 0x93, 0x38, 0x65, 0x72, 0xa9, 0x5f, 0x05, 0xe1, 0xf0, 0x9c, 0x8b, 0xd8, - 0x69, 0xf8, 0x4f, 0x33, 0x12, 0x31, 0xf5, 0x35, 0x05, 0xa6, 0x12, 0xad, 0xba, 0x10, 0x47, 0x4f, - 0xd0, 0x62, 0xf5, 0x2b, 0xfd, 0x74, 0x4b, 0x7a, 0x94, 0xdf, 0x9b, 0x5c, 0xb1, 0x7d, 0x02, 0xf7, - 0xdf, 0x07, 0x60, 0xc4, 0x68, 0x34, 0x5a, 0x1e, 0x7b, 0x44, 0x1c, 0x08, 0xb5, 0xc5, 0x15, 0xbf, - 0x10, 0x87, 0x70, 0x64, 0x09, 0x49, 0x8b, 0x9f, 0x82, 0xcb, 0xf9, 0x56, 0x4e, 0x1e, 0xe0, 0x1c, - 0x95, 0x8a, 0xb8, 0x38, 0x94, 0x26, 0x88, 0x7d, 0x9f, 0x02, 0xe0, 0x7a, 0x8e, 0x61, 0xd5, 0x69, - 0xa1, 0x90, 0xc6, 0xf0, 0x11, 0x90, 0xad, 0x06, 0x48, 0x39, 0xf1, 0x30, 0x94, 0x67, 0x00, 0xc0, - 0x12, 0x65, 0x34, 0x2f, 0x84, 0x50, 0x7e, 0x64, 0xbe, 0x23, 0x26, 0x6e, 0x5f, 0x4a, 0xb1, 0x84, - 0xe5, 0x84, 0x42, 0x29, 0x75, 0xf6, 0x11, 0x18, 0x09, 0xe8, 0x1d, 0x24, 0xd4, 0x8d, 0x49, 0x42, - 0xdd, 0xec, 0xe3, 0x70, 0x2a, 0xd6, 0xdd, 0x43, 0xc9, 0x84, 0x7f, 0xa6, 0x00, 0x8a, 0x8e, 0xfe, - 0x04, 0x34, 0x07, 0xf5, 0xa8, 0xe6, 0x60, 0xa1, 0xf7, 0x25, 0xcb, 0x50, 0x1d, 0xfc, 0x1c, 0x02, - 0x16, 0xd5, 0x38, 0x88, 0xf2, 0x2d, 0x4e, 0x7e, 0x2a, 0xa8, 0x84, 0xce, 0xfc, 0xe2, 0xcb, 0xed, - 0x41, 0x50, 0xb9, 0x19, 0xc3, 0x15, 0x0a, 0x2a, 0x71, 0x08, 0x4e, 0xd0, 0x45, 0x9f, 0x54, 0x60, - 0x52, 0x8b, 0x46, 0x35, 0xf6, 0x67, 0x26, 0x57, 0xac, 0xb5, 0x58, 0x84, 0xe4, 0xb0, 0x2f, 0x31, - 0x80, 0x8b, 0x13, 0x64, 0xd1, 0xc3, 0x30, 0xa6, 0x35, 0x8d, 0xf9, 0x56, 0xcd, 0xa0, 0x37, 0x4f, - 0x3f, 0xf8, 0x2b, 0xd3, 0x86, 0xcc, 0xaf, 0x55, 0x82, 0x72, 0x1c, 0xa9, 0x15, 0x84, 0x0f, 0x2e, - 0x85, 0x86, 0xc3, 0xbd, 0x84, 0x0f, 0x16, 0x73, 0x18, 0x86, 0x0f, 0x16, 0x53, 0x27, 0x13, 0x41, - 0x16, 0x80, 0x6d, 0xd4, 0x74, 0x41, 0x72, 0x30, 0xff, 0x1b, 0xc8, 0xad, 0x4a, 0xb9, 0x24, 0x28, - 0x32, 0xf1, 0x21, 0xfc, 0x8d, 0x25, 0x0a, 0xe8, 0xd3, 0x0a, 0x8c, 0x0b, 0xde, 0x2d, 0x68, 0x0e, - 0xb1, 0x25, 0x7a, 0x3e, 0xef, 0x7e, 0x89, 0xed, 0xc9, 0x39, 0x2c, 0x23, 0xe7, 0x7c, 0x27, 0x88, - 0x05, 0x11, 0x81, 0xe1, 0x68, 0x3f, 0x98, 0x0c, 0xe0, 0x46, 0x1e, 0xa1, 0x44, 0x07, 0x87, 0xf3, - 0xcb, 0x00, 0xd5, 0x14, 0x7c, 0xc2, 0xc9, 0x28, 0x05, 0x82, 0x53, 0xe9, 0x53, 0xb9, 0xf6, 0xd4, - 0x1d, 0xcd, 0xd3, 0xb7, 0x4b, 0x9a, 0xbe, 0xcd, 0xde, 0x20, 0xb9, 0x67, 0x6c, 0xce, 0x7d, 0xfd, - 0x6c, 0x14, 0xd5, 0xc2, 0xe9, 0xfd, 0x76, 0xf1, 0x54, 0xac, 0x10, 0xc7, 0x09, 0x22, 0x1b, 0x86, - 0x1d, 0x11, 0xda, 0x5f, 0x04, 0x33, 0xc8, 0x17, 0xcd, 0x3e, 0x9e, 0x27, 0x80, 0xdf, 0x8c, 0xfc, - 0x5f, 0x38, 0x20, 0x82, 0xea, 0x70, 0x89, 0xdf, 0x0d, 0xe7, 0x2d, 0xdb, 0xda, 0x6b, 0xd8, 0x2d, - 0x77, 0xbe, 0xe5, 0x6d, 0x13, 0xcb, 0xf3, 0x55, 0xe1, 0xa3, 0xec, 0x18, 0x65, 0x4e, 0x73, 0x8b, - 0x9d, 0x2a, 0xe2, 0xce, 0x78, 0xd0, 0x73, 0x30, 0xcc, 0xde, 0xe9, 0xd6, 0xd7, 0x97, 0x99, 0x93, - 0xed, 0xe1, 0xc5, 0x65, 0x36, 0x84, 0x45, 0x81, 0x03, 0x07, 0xd8, 0xd0, 0x6d, 0x18, 0x32, 0x79, - 0x6e, 0x06, 0xe6, 0x6c, 0x9b, 0x93, 0x29, 0xc6, 0xf3, 0x3c, 0xf0, 0x0b, 0xb4, 0xf8, 0x81, 0x7d, - 0x0a, 0xa8, 0x09, 0x57, 0x6a, 0x64, 0x4b, 0x6b, 0x99, 0xde, 0xaa, 0xed, 0x61, 0xe6, 0xa1, 0x16, - 0x68, 0x3c, 0x7d, 0x7f, 0xea, 0x09, 0x16, 0x4e, 0x8f, 0xf9, 0xfe, 0x95, 0x0f, 0xa8, 0x8b, 0x0f, - 0xc4, 0x86, 0xf6, 0xe0, 0x3e, 0x51, 0x87, 0xb9, 0xc4, 0xe9, 0xdb, 0x74, 0x96, 0x93, 0x44, 0x4f, - 0x31, 0xa2, 0xdf, 0xb2, 0xdf, 0x2e, 0xde, 0x57, 0x3e, 0xb8, 0x3a, 0xee, 0x06, 0x27, 0xf3, 0x32, - 0x22, 0xb1, 0x27, 0xa0, 0x99, 0xc9, 0xfc, 0x73, 0x1c, 0x7f, 0x4e, 0xe2, 0xe6, 0x70, 0xf1, 0x52, - 0x9c, 0xa0, 0x89, 0xbe, 0xa0, 0xc0, 0x8c, 0xeb, 0x39, 0x2d, 0xdd, 0x6b, 0x39, 0xa4, 0x16, 0xdb, - 0xa1, 0x53, 0xac, 0x43, 0xb9, 0x04, 0xb8, 0x6a, 0x06, 0x4e, 0xe6, 0xd9, 0x3f, 0x93, 0x05, 0xc5, - 0x99, 0x7d, 0x41, 0x9f, 0x57, 0xe0, 0x5c, 0x14, 0x48, 0xef, 0xf4, 0xbc, 0x9f, 0x28, 0xff, 0x23, - 0x4b, 0x35, 0x1d, 0x25, 0xbf, 0xc1, 0x67, 0x00, 0x71, 0x56, 0x47, 0xe2, 0x2f, 0xe3, 0xa7, 0x4f, - 0xf8, 0x65, 0x7c, 0xf6, 0x29, 0x40, 0xc9, 0xe3, 0xe3, 0x20, 0x39, 0x70, 0x58, 0x96, 0x03, 0x3f, - 0x3b, 0x00, 0x17, 0xe8, 0xa9, 0x14, 0xde, 0x7e, 0x56, 0x34, 0x4b, 0xab, 0xbf, 0x3e, 0x25, 0xa6, - 0x2f, 0x2a, 0x70, 0x6e, 0x3b, 0x5d, 0xb5, 0x23, 0xee, 0x5f, 0x4f, 0xe7, 0x52, 0xc1, 0x75, 0xd2, - 0x16, 0x71, 0x86, 0xdd, 0xb1, 0x0a, 0xce, 0xea, 0x14, 0x7a, 0x0a, 0x26, 0x2d, 0xbb, 0x46, 0x4a, - 0x95, 0x32, 0x5e, 0xd1, 0xdc, 0xdb, 0x55, 0xdf, 0x50, 0x67, 0x80, 0x7f, 0xaf, 0xab, 0x31, 0x18, - 0x4e, 0xd4, 0x46, 0x3b, 0x80, 0x9a, 0x76, 0x6d, 0x71, 0x87, 0xe7, 0x30, 0xe9, 0xcd, 0x64, 0x96, - 0x19, 0x0b, 0xac, 0x25, 0xb0, 0xe1, 0x14, 0x0a, 0x4c, 0x37, 0x45, 0x3b, 0xb3, 0x62, 0x5b, 0x86, - 0x67, 0x3b, 0x2c, 0x56, 0x45, 0x4f, 0x2a, 0x1a, 0xa6, 0x9b, 0x5a, 0x4d, 0xc5, 0x88, 0x33, 0x28, - 0xa9, 0xff, 0x4d, 0x81, 0x53, 0x74, 0x5b, 0xac, 0x39, 0xf6, 0xee, 0xde, 0xeb, 0x71, 0x43, 0xde, - 0x2f, 0x6c, 0x16, 0xb9, 0x6e, 0xe7, 0x8c, 0x64, 0xaf, 0x38, 0xc2, 0xfa, 0x1c, 0x9a, 0x28, 0xca, - 0x6a, 0xe5, 0xbe, 0x6c, 0xb5, 0xb2, 0xfa, 0xe9, 0x02, 0xbf, 0xb9, 0xf8, 0x6a, 0xdd, 0xd7, 0xe5, - 0x77, 0xf8, 0x08, 0x8c, 0xd3, 0xb2, 0x15, 0x6d, 0x77, 0xad, 0xfc, 0x8c, 0x6d, 0xfa, 0xde, 0xd8, - 0x4c, 0xd7, 0x7e, 0x53, 0x06, 0xe0, 0x68, 0x3d, 0xf4, 0x18, 0x0c, 0x35, 0x79, 0xc8, 0x37, 0x71, - 0x67, 0xbe, 0xc2, 0x0d, 0xfb, 0x58, 0xd1, 0xbd, 0x76, 0x71, 0x2a, 0x7c, 0xe2, 0xf5, 0x03, 0xcf, - 0xf9, 0x0d, 0xd4, 0x5f, 0x3d, 0x0b, 0x0c, 0xb9, 0x49, 0xbc, 0xd7, 0xe3, 0x9c, 0x3c, 0x08, 0xa3, - 0x7a, 0xb3, 0x55, 0x5a, 0xaa, 0x3e, 0xdd, 0xb2, 0x99, 0x2e, 0x84, 0xa5, 0xfa, 0xa1, 0xdc, 0xbb, - 0xb4, 0xb6, 0xe1, 0x17, 0x63, 0xb9, 0x0e, 0xe5, 0x0e, 0x7a, 0xb3, 0x25, 0xf8, 0xed, 0x9a, 0xec, - 0xaf, 0xc2, 0xb8, 0x43, 0x69, 0x6d, 0x23, 0x02, 0xc3, 0x89, 0xda, 0xe8, 0xc3, 0x30, 0x46, 0xc4, - 0x87, 0x7b, 0x43, 0x73, 0x6a, 0x82, 0x2f, 0x54, 0xf2, 0x0e, 0x3e, 0x98, 0x5a, 0x9f, 0x1b, 0xf0, - 0x1b, 0xe0, 0xa2, 0x44, 0x02, 0x47, 0x08, 0xa2, 0x17, 0xe0, 0xbc, 0xff, 0x9b, 0xae, 0xb2, 0x5d, - 0x8b, 0x33, 0x8a, 0x01, 0x6e, 0x36, 0xb3, 0x98, 0x55, 0x09, 0x67, 0xb7, 0x47, 0x3f, 0xaf, 0xc0, - 0xd9, 0x00, 0x6a, 0x58, 0x46, 0xa3, 0xd5, 0xc0, 0x44, 0x37, 0x35, 0xa3, 0x21, 0xee, 0x7d, 0xcf, - 0x1e, 0xd9, 0x40, 0xa3, 0xe8, 0x39, 0xb3, 0x4a, 0x87, 0xe1, 0x8c, 0x2e, 0xa1, 0xd7, 0x14, 0xb8, - 0xe2, 0x83, 0xd6, 0x1c, 0xe2, 0xba, 0x2d, 0x87, 0x84, 0xb1, 0x00, 0xc4, 0x94, 0x0c, 0xe5, 0xe2, - 0x9d, 0x4c, 0x00, 0x5e, 0x3c, 0x00, 0x37, 0x3e, 0x90, 0xba, 0xbc, 0x5d, 0xaa, 0xf6, 0x96, 0x27, - 0x2e, 0x8a, 0xc7, 0xb5, 0x5d, 0x28, 0x09, 0x1c, 0x21, 0x88, 0x7e, 0x41, 0x81, 0x73, 0x72, 0x81, - 0xbc, 0x5b, 0xf8, 0x0d, 0xf1, 0xb9, 0x23, 0xeb, 0x4c, 0x0c, 0x3f, 0x97, 0xf0, 0x32, 0x80, 0x38, - 0xab, 0x57, 0x94, 0x6d, 0x37, 0xd8, 0xc6, 0xe4, 0xb7, 0xc8, 0x01, 0xce, 0xb6, 0xf9, 0x5e, 0x75, - 0xb1, 0x0f, 0x43, 0x0f, 0xc3, 0x58, 0xd3, 0xae, 0xad, 0x19, 0x35, 0x97, 0x85, 0xa3, 0x63, 0x77, - 0xbd, 0x3e, 0x3e, 0x1d, 0x6b, 0x76, 0x6d, 0xad, 0x52, 0xe6, 0xe5, 0x38, 0x52, 0x0b, 0xcd, 0x01, - 0x6c, 0x69, 0x86, 0x59, 0xbd, 0xa3, 0x35, 0x6f, 0xf9, 0xf1, 0x8d, 0x98, 0x2e, 0x62, 0x29, 0x28, - 0xc5, 0x52, 0x0d, 0xba, 0x7e, 0x94, 0xef, 0x60, 0xc2, 0x23, 0x88, 0xb3, 0xeb, 0xd1, 0x51, 0xac, - 0x9f, 0x8f, 0x90, 0x77, 0xf8, 0xa6, 0x44, 0x02, 0x47, 0x08, 0xa2, 0x8f, 0x2b, 0x30, 0xe1, 0xee, - 0xb9, 0x1e, 0x69, 0x04, 0x7d, 0x38, 0x75, 0xd4, 0x7d, 0x60, 0x3a, 0xf1, 0x6a, 0x84, 0x08, 0x8e, - 0x11, 0x65, 0x91, 0xa2, 0x1a, 0x5a, 0x9d, 0x5c, 0x2f, 0xdd, 0x30, 0xea, 0xdb, 0x41, 0x74, 0x97, - 0x35, 0xe2, 0xe8, 0xc4, 0xf2, 0xd8, 0xc5, 0x6a, 0x40, 0x44, 0x8a, 0xca, 0xae, 0x86, 0x3b, 0xe1, - 0x40, 0x2f, 0xc2, 0xac, 0x00, 0x2f, 0xdb, 0x77, 0x12, 0x14, 0x78, 0x40, 0x26, 0x66, 0x00, 0x59, - 0xc9, 0xac, 0x85, 0x3b, 0x60, 0x40, 0x15, 0x38, 0xed, 0x12, 0x87, 0xbd, 0x09, 0xf2, 0xe0, 0x9e, - 0x6b, 0x2d, 0xd3, 0x74, 0xd9, 0xd5, 0x46, 0xb8, 0xfc, 0x54, 0x93, 0x60, 0x9c, 0xd6, 0x06, 0x3d, - 0x1e, 0xb8, 0x05, 0xef, 0xd1, 0x82, 0xa7, 0xd7, 0xaa, 0xec, 0x26, 0x32, 0xc0, 0x35, 0x25, 0x38, - 0x0a, 0xc2, 0xf1, 0xba, 0xf4, 0x34, 0xf7, 0x8b, 0x16, 0x5a, 0x8e, 0xeb, 0xb1, 0x30, 0x49, 0x03, - 0xfc, 0x34, 0xc7, 0x32, 0x00, 0x47, 0xeb, 0xa1, 0xc7, 0x60, 0xc2, 0x25, 0xba, 0x6e, 0x37, 0x9a, - 0xe2, 0x9e, 0xcc, 0x62, 0x19, 0x0d, 0x8b, 0x15, 0x8c, 0x40, 0x70, 0xac, 0x26, 0xda, 0x83, 0xd3, - 0x41, 0x34, 0xe4, 0x65, 0xbb, 0xbe, 0xa2, 0xed, 0x32, 0xe1, 0xf8, 0x6c, 0x9e, 0x10, 0x0f, 0x7c, - 0xba, 0x4a, 0x49, 0x74, 0x38, 0x8d, 0x06, 0x5a, 0x86, 0xe9, 0x58, 0xf1, 0x92, 0x61, 0x12, 0x77, - 0xe6, 0x1c, 0x1b, 0x36, 0x53, 0x76, 0x95, 0x52, 0xe0, 0x38, 0xb5, 0x15, 0xba, 0x05, 0x67, 0x9a, - 0x8e, 0xed, 0x11, 0xdd, 0xbb, 0x49, 0x05, 0x02, 0x53, 0x0c, 0xd0, 0x9d, 0x99, 0x61, 0x73, 0xc1, - 0xde, 0x43, 0xd7, 0xd2, 0x2a, 0xe0, 0xf4, 0x76, 0xe8, 0xb3, 0x0a, 0x5c, 0x76, 0x3d, 0x87, 0x68, - 0x0d, 0xc3, 0xaa, 0x97, 0x6c, 0xcb, 0x22, 0x8c, 0x31, 0x55, 0x6a, 0xa1, 0xc7, 0xdc, 0xf9, 0x5c, - 0xa7, 0x88, 0xba, 0xdf, 0x2e, 0x5e, 0xae, 0x76, 0xc4, 0x8c, 0x0f, 0xa0, 0x8c, 0x5e, 0x01, 0x68, - 0x90, 0x86, 0xed, 0xec, 0x51, 0x8e, 0x34, 0x33, 0x9b, 0xff, 0xbe, 0xbb, 0x12, 0x60, 0xe1, 0x9f, - 0x7f, 0xe4, 0x25, 0x37, 0x04, 0x62, 0x89, 0x1c, 0x9d, 0x6a, 0xca, 0x6f, 0x45, 0x40, 0x32, 0xe9, - 0xa3, 0xb9, 0xc0, 0x56, 0x8e, 0x4d, 0xf5, 0x4a, 0x5a, 0x05, 0x9c, 0xde, 0x0e, 0xb9, 0x30, 0xc5, - 0xbe, 0x50, 0x71, 0x92, 0x5f, 0x2f, 0xcd, 0xd7, 0xc9, 0xcc, 0xc5, 0x5c, 0x93, 0x4b, 0x65, 0xff, - 0xa9, 0x4a, 0x1c, 0x19, 0x4e, 0xe2, 0x0f, 0x89, 0x6a, 0xbb, 0x21, 0xd1, 0x4b, 0xbd, 0x12, 0x95, - 0x90, 0xe1, 0x24, 0x7e, 0xb5, 0x5d, 0x80, 0x33, 0xa9, 0xa7, 0x24, 0x65, 0x1e, 0x7c, 0x8a, 0xe7, - 0xfd, 0x54, 0x6e, 0xe2, 0xd9, 0x93, 0x31, 0x8f, 0x95, 0x28, 0x08, 0xc7, 0xeb, 0x52, 0x19, 0x96, - 0x51, 0x5b, 0xaa, 0x86, 0xed, 0x0b, 0xa1, 0x0c, 0x5b, 0x89, 0xc1, 0x70, 0xa2, 0x36, 0x2a, 0x89, - 0xf9, 0x58, 0xaa, 0x56, 0xe8, 0x35, 0xd0, 0x5d, 0x72, 0x88, 0x7f, 0x3b, 0x08, 0xc7, 0x27, 0x03, - 0x71, 0xb2, 0x3e, 0x1d, 0x05, 0xfd, 0x21, 0xf7, 0xa2, 0x3f, 0x1c, 0xc5, 0x6a, 0x14, 0x84, 0xe3, - 0x75, 0xfd, 0x7b, 0x7a, 0xa4, 0x0b, 0x03, 0xe1, 0x28, 0x56, 0x63, 0x30, 0x9c, 0xa8, 0xad, 0xfe, - 0x79, 0x3f, 0xdc, 0xd7, 0x85, 0x64, 0x89, 0x1a, 0xe9, 0xd3, 0x7d, 0x78, 0x9e, 0xd7, 0xdd, 0xf2, - 0x34, 0x33, 0x96, 0xe7, 0xf0, 0xf4, 0xba, 0x5d, 0x4e, 0x37, 0x6b, 0x39, 0x0f, 0x4f, 0xb2, 0xfb, - 0xe5, 0x6f, 0xa4, 0x2f, 0x7f, 0xce, 0x59, 0x3d, 0x70, 0xbb, 0x34, 0x33, 0xb6, 0x4b, 0xce, 0x59, - 0xed, 0x62, 0x7b, 0xfd, 0x45, 0x3f, 0xbc, 0xa5, 0x1b, 0x29, 0x37, 0xe7, 0xfe, 0x4a, 0xe1, 0x2d, - 0xc7, 0xba, 0xbf, 0xb2, 0xfc, 0xb9, 0x8f, 0x71, 0x7f, 0x75, 0x64, 0x9f, 0xc7, 0xb3, 0xbf, 0xb2, - 0x66, 0xf5, 0xb8, 0xf6, 0x57, 0xd6, 0xac, 0x76, 0xb1, 0xbf, 0xfe, 0x36, 0x7e, 0x3e, 0x04, 0xa2, - 0x76, 0x05, 0xfa, 0xf4, 0x66, 0x2b, 0x27, 0x93, 0x62, 0x56, 0x86, 0xa5, 0xb5, 0x0d, 0x4c, 0x71, - 0x20, 0x0c, 0x83, 0x7c, 0xff, 0xe4, 0x64, 0x41, 0xcc, 0x72, 0x94, 0x6f, 0x49, 0x2c, 0x30, 0xd1, - 0xa9, 0x22, 0xcd, 0x6d, 0xd2, 0x20, 0x8e, 0x66, 0x56, 0x3d, 0xdb, 0xd1, 0xea, 0x79, 0xb9, 0x0d, - 0x7f, 0x41, 0x89, 0xe1, 0xc2, 0x09, 0xec, 0x74, 0x42, 0x9a, 0x46, 0x2d, 0x27, 0x7f, 0x61, 0x13, - 0xb2, 0x56, 0x29, 0x63, 0x8a, 0x43, 0xfd, 0x8d, 0x61, 0x90, 0xf2, 0x0c, 0xa0, 0x57, 0x15, 0x98, - 0xd2, 0xe3, 0x31, 0x39, 0x7b, 0xb1, 0x87, 0x4a, 0x04, 0xf8, 0xe4, 0x5b, 0x3e, 0x51, 0x8c, 0x93, - 0x64, 0xd1, 0x47, 0x14, 0xae, 0xe4, 0x0b, 0x5e, 0xf3, 0xc4, 0xb4, 0x5e, 0x3f, 0xa2, 0x77, 0xef, - 0x50, 0x5b, 0x18, 0x3e, 0xb1, 0x46, 0x09, 0xa2, 0xd7, 0x14, 0x38, 0x73, 0x3b, 0xed, 0x6d, 0x42, - 0x4c, 0xfe, 0xad, 0xbc, 0x5d, 0xc9, 0x78, 0xec, 0xe0, 0x12, 0x64, 0x6a, 0x05, 0x9c, 0xde, 0x91, - 0x60, 0x96, 0x02, 0x75, 0xad, 0xf8, 0x4e, 0x73, 0xcf, 0x52, 0x4c, 0xef, 0x1b, 0xce, 0x52, 0x00, - 0xc0, 0x51, 0x82, 0xa8, 0x09, 0x23, 0xb7, 0x7d, 0x1d, 0xb9, 0xd0, 0x8b, 0x95, 0xf2, 0x52, 0x97, - 0x14, 0xed, 0xdc, 0xde, 0x2b, 0x28, 0xc4, 0x21, 0x11, 0xb4, 0x0d, 0x43, 0xb7, 0x39, 0xaf, 0x10, - 0xfa, 0xac, 0xf9, 0x9e, 0x6f, 0xff, 0x5c, 0xad, 0x22, 0x8a, 0xb0, 0x8f, 0x5e, 0xf6, 0x25, 0x18, - 0x3e, 0xc0, 0xc5, 0xed, 0xb3, 0x0a, 0x9c, 0xd9, 0x21, 0x8e, 0x67, 0xe8, 0xf1, 0x97, 0xa1, 0x91, - 0xfc, 0x1a, 0x8a, 0x67, 0xd2, 0x10, 0xf2, 0x6d, 0x92, 0x0a, 0xc2, 0xe9, 0x5d, 0x40, 0xcf, 0x40, - 0x3f, 0xf1, 0xf4, 0x9a, 0x88, 0xc5, 0xfd, 0x9e, 0xbc, 0x2e, 0xbb, 0xdc, 0xef, 0x85, 0xfe, 0x87, - 0x19, 0x3e, 0xf5, 0xaf, 0x15, 0x48, 0xe8, 0xa6, 0xd1, 0x0f, 0x29, 0x30, 0xb6, 0x45, 0x34, 0xaf, - 0xe5, 0x90, 0xeb, 0xc2, 0x26, 0xb4, 0xef, 0xea, 0xe8, 0x43, 0xcf, 0x1c, 0x85, 0x4a, 0x7c, 0x6e, - 0x49, 0x42, 0xcc, 0x8d, 0x55, 0x82, 0xdc, 0x21, 0x32, 0x08, 0x47, 0x7a, 0x30, 0xfb, 0x24, 0x4c, - 0x25, 0x1a, 0x1e, 0xea, 0x99, 0xf2, 0xd7, 0x15, 0x48, 0x4b, 0x53, 0x8f, 0x5e, 0x84, 0x01, 0xad, - 0x56, 0x0b, 0xb2, 0x95, 0x3e, 0x9a, 0xcf, 0x6e, 0xaa, 0x26, 0x47, 0x0a, 0x62, 0x3f, 0x31, 0x47, - 0x8b, 0x96, 0x00, 0x69, 0x91, 0x77, 0xe9, 0x95, 0x30, 0x44, 0x05, 0x7b, 0x4e, 0x9b, 0x4f, 0x40, - 0x71, 0x4a, 0x0b, 0xf5, 0x13, 0x0a, 0xa0, 0x64, 0xb6, 0x19, 0xe4, 0xc0, 0xb0, 0xd8, 0xbf, 0xfe, - 0x2a, 0x95, 0x73, 0x7a, 0xd3, 0x45, 0x5c, 0x43, 0x43, 0x23, 0x3c, 0x51, 0xe0, 0xe2, 0x80, 0x8e, - 0xfa, 0xf7, 0x0a, 0x84, 0x19, 0x0d, 0xd1, 0xbb, 0x60, 0xb4, 0x46, 0x5c, 0xdd, 0x31, 0x9a, 0x5e, - 0xe8, 0x48, 0x1a, 0x38, 0xa4, 0x95, 0x43, 0x10, 0x96, 0xeb, 0x21, 0x15, 0x06, 0x3d, 0xcd, 0xbd, - 0x5d, 0x29, 0x8b, 0xcb, 0x1e, 0x3b, 0x9a, 0xd7, 0x59, 0x09, 0x16, 0x90, 0x30, 0x74, 0x6c, 0x5f, - 0x17, 0xa1, 0x63, 0xd1, 0xd6, 0x11, 0xc4, 0xc9, 0x45, 0x07, 0xc7, 0xc8, 0x55, 0x7f, 0xb6, 0x00, - 0xa7, 0x68, 0x95, 0x15, 0xcd, 0xb0, 0x3c, 0x62, 0x31, 0xb7, 0xa9, 0x9c, 0x93, 0x50, 0x87, 0x71, - 0x2f, 0xe2, 0xb4, 0x7c, 0x78, 0xa7, 0xda, 0xc0, 0xd2, 0x2b, 0xea, 0xaa, 0x1c, 0xc5, 0x8b, 0x1e, - 0xf5, 0xfd, 0xd6, 0xf8, 0xb5, 0xf8, 0x3e, 0x7f, 0xab, 0x32, 0x67, 0xb4, 0x7b, 0xc2, 0x03, 0x3c, - 0x48, 0x83, 0x19, 0x71, 0x51, 0x7b, 0x04, 0xc6, 0x85, 0x87, 0x04, 0x8f, 0x01, 0x2c, 0xae, 0xc5, - 0xec, 0x58, 0x59, 0x92, 0x01, 0x38, 0x5a, 0x4f, 0xfd, 0xa3, 0x02, 0x44, 0x93, 0x6d, 0xe6, 0x9d, - 0xa5, 0x64, 0x00, 0xe4, 0xc2, 0xb1, 0x05, 0x40, 0xe6, 0xe9, 0xe6, 0x99, 0x39, 0xb7, 0x78, 0x67, - 0x97, 0x53, 0x4d, 0xb3, 0x72, 0x1c, 0xd4, 0x08, 0xa7, 0xb5, 0xff, 0xd0, 0xd3, 0xfa, 0x2e, 0x61, - 0xf9, 0x3b, 0x10, 0x09, 0x43, 0xed, 0x5b, 0xfe, 0x4e, 0x45, 0x1a, 0x4a, 0x5e, 0x76, 0xf3, 0x20, - 0xd2, 0xd5, 0xd0, 0x75, 0x11, 0x21, 0xb2, 0xdd, 0x75, 0xdb, 0xd3, 0x4c, 0x36, 0x9d, 0x42, 0xe9, - 0xba, 0x22, 0x03, 0x70, 0xb4, 0x9e, 0xba, 0x0a, 0x6f, 0x5e, 0xb6, 0xb5, 0xda, 0x82, 0x66, 0xd2, - 0xad, 0xeb, 0x08, 0xb3, 0x3c, 0x97, 0x9d, 0xcc, 0x6b, 0x8e, 0xed, 0xd9, 0xba, 0x6d, 0xd2, 0x73, - 0x53, 0x0b, 0x62, 0x05, 0xb3, 0x87, 0x48, 0xff, 0xdc, 0x14, 0xb1, 0x76, 0xb1, 0x0f, 0x57, 0x7f, - 0x57, 0x81, 0x21, 0x91, 0xc2, 0xa6, 0x0b, 0xc7, 0xd2, 0x2d, 0x18, 0x60, 0xb7, 0xa3, 0x5e, 0xa4, - 0xd2, 0xea, 0xb6, 0x6d, 0x7b, 0x91, 0xdc, 0x57, 0x3c, 0x7b, 0x1c, 0xcb, 0xf7, 0xc9, 0xd1, 0x33, - 0x7b, 0x54, 0x47, 0xdf, 0x36, 0x3c, 0xc2, 0xcc, 0x6e, 0xc4, 0xc6, 0xe7, 0xf6, 0xa8, 0x52, 0x39, - 0x8e, 0xd4, 0x52, 0xff, 0xeb, 0x00, 0x5c, 0xf1, 0xd3, 0xf1, 0xc4, 0x45, 0xb5, 0x80, 0xe7, 0xee, - 0x05, 0xf9, 0x81, 0x58, 0x7e, 0x03, 0x5f, 0xa7, 0x9a, 0xef, 0x96, 0x7c, 0x4e, 0xca, 0x00, 0x24, - 0xa3, 0xc3, 0x69, 0x34, 0x78, 0xf4, 0x75, 0x56, 0x7c, 0x83, 0x68, 0xa6, 0xb7, 0xed, 0xd3, 0x2e, - 0xf4, 0x12, 0x7d, 0x3d, 0x89, 0x0f, 0xa7, 0x52, 0x61, 0x26, 0x1d, 0x7e, 0x5e, 0x22, 0x87, 0x68, - 0xb2, 0x3d, 0x49, 0x0f, 0xee, 0x46, 0x2b, 0xa9, 0x18, 0x71, 0x06, 0x25, 0xa6, 0x6e, 0xd4, 0x76, - 0x99, 0xf6, 0x02, 0x13, 0xcf, 0x31, 0x58, 0x0e, 0xb3, 0xe0, 0xad, 0x62, 0x25, 0x0a, 0xc2, 0xf1, - 0xba, 0xe8, 0x31, 0x98, 0x60, 0x26, 0x32, 0x61, 0x38, 0xcf, 0x81, 0x30, 0xa8, 0xd3, 0x6a, 0x04, - 0x82, 0x63, 0x35, 0xd1, 0xf7, 0x28, 0x30, 0x6d, 0xc8, 0xce, 0x34, 0xfe, 0xe8, 0x07, 0x7b, 0x49, - 0x0b, 0xc1, 0xb7, 0x71, 0x0a, 0x5a, 0x9c, 0x4a, 0x0c, 0x2d, 0xc3, 0xb4, 0xf0, 0x88, 0x8e, 0xee, - 0x01, 0x1e, 0xeb, 0x8d, 0xad, 0x69, 0x39, 0x05, 0x8e, 0x53, 0x5b, 0xa9, 0x1f, 0x2d, 0xc0, 0xd8, - 0x21, 0xf3, 0xd1, 0xb6, 0x24, 0x99, 0xa3, 0x07, 0xbf, 0x45, 0x99, 0x6a, 0x17, 0x62, 0x07, 0x7a, - 0x0e, 0x26, 0x5a, 0x6c, 0x22, 0xfc, 0x48, 0x68, 0xe2, 0x9b, 0xfe, 0x36, 0xba, 0x72, 0x1b, 0x11, - 0xc8, 0xbd, 0x76, 0x71, 0x56, 0x46, 0x1f, 0x85, 0xe2, 0x18, 0x1e, 0xf5, 0x37, 0x0b, 0x80, 0xe4, - 0xea, 0x4b, 0xa6, 0xb6, 0x63, 0x3b, 0xe8, 0xf3, 0x0a, 0x8c, 0xe9, 0x72, 0x06, 0x30, 0x2e, 0x60, - 0x3d, 0xd7, 0xeb, 0x60, 0x39, 0xfa, 0x39, 0x39, 0xb5, 0x17, 0x17, 0x84, 0x7d, 0x77, 0xeb, 0x31, - 0x19, 0x74, 0x2f, 0xf6, 0x1b, 0x47, 0xfa, 0x34, 0xfb, 0x71, 0x05, 0xa6, 0x12, 0x98, 0x52, 0x24, - 0xe3, 0xe7, 0xa3, 0x31, 0xd0, 0x8f, 0x24, 0xd7, 0xa7, 0x2c, 0x5f, 0xff, 0xd4, 0x00, 0x9c, 0x4e, - 0x59, 0x51, 0x66, 0x62, 0x43, 0x62, 0xd2, 0x65, 0x2f, 0x26, 0x36, 0x09, 0x49, 0x35, 0x30, 0xb1, - 0x89, 0x43, 0x70, 0x82, 0x2e, 0x7a, 0x06, 0xfa, 0x74, 0xc7, 0x10, 0x9b, 0xf6, 0x91, 0x5c, 0x53, - 0x80, 0x2b, 0x0b, 0xa3, 0x82, 0x62, 0x5f, 0x09, 0x57, 0x30, 0x45, 0x48, 0xcf, 0x62, 0xf9, 0x18, - 0xf1, 0x05, 0x56, 0x76, 0x16, 0xcb, 0xa7, 0x8d, 0x8b, 0xa3, 0xf5, 0xd0, 0x73, 0x30, 0x23, 0x6e, - 0xaa, 0x7e, 0x24, 0x13, 0xdb, 0x72, 0x3d, 0xca, 0xf1, 0x3d, 0x21, 0x53, 0x30, 0xeb, 0xd7, 0x9b, - 0x19, 0x75, 0x70, 0x66, 0x6b, 0xf4, 0xdd, 0x30, 0x11, 0xe1, 0x1e, 0x7e, 0x54, 0xb2, 0x9c, 0x2e, - 0x33, 0x32, 0x26, 0xce, 0x2b, 0xa3, 0x65, 0x38, 0x46, 0x0d, 0xfd, 0xdf, 0x0a, 0x4c, 0x05, 0x1b, - 0x75, 0x8f, 0x6f, 0x79, 0x3f, 0x19, 0xe3, 0xd2, 0xd1, 0x7c, 0x41, 0x52, 0xa6, 0xdc, 0x38, 0x21, - 0x9c, 0xa4, 0xad, 0xfe, 0xe8, 0x20, 0xc8, 0xc9, 0xb9, 0xd1, 0x4a, 0x2f, 0x7a, 0xc9, 0x70, 0x0f, - 0xf8, 0xba, 0xc9, 0x15, 0xe8, 0xab, 0x37, 0x5b, 0x39, 0x15, 0x93, 0x01, 0xba, 0xeb, 0x14, 0x5d, - 0xbd, 0xd9, 0x42, 0xcf, 0x04, 0xaa, 0xce, 0x7c, 0xca, 0xc8, 0xc0, 0x55, 0x32, 0xa6, 0xee, 0xf4, - 0xd9, 0x7b, 0x7f, 0x26, 0x7b, 0x6f, 0xc0, 0x90, 0x2b, 0xf4, 0xa0, 0x03, 0xf9, 0xc3, 0x48, 0x4a, - 0x33, 0x2d, 0xf4, 0x9e, 0x5c, 0x43, 0xe3, 0xab, 0x45, 0x7d, 0x1a, 0xf4, 0x22, 0xd8, 0x62, 0xe7, - 0x92, 0x08, 0x92, 0xc4, 0x2e, 0x82, 0x1b, 0xac, 0x04, 0x0b, 0x48, 0x42, 0x98, 0x1b, 0xea, 0x46, - 0x98, 0x43, 0x9f, 0x8b, 0xf3, 0xef, 0x61, 0xb6, 0xfb, 0x9e, 0xee, 0x71, 0x38, 0xff, 0x87, 0x31, - 0xee, 0xff, 0x2b, 0x3c, 0xfc, 0xa4, 0xc5, 0x42, 0xf7, 0xc1, 0x00, 0x8b, 0xa2, 0x24, 0xe4, 0x80, - 0x40, 0xb9, 0xc1, 0xe2, 0xe8, 0x60, 0x0e, 0x43, 0x55, 0x11, 0x3a, 0x30, 0xdf, 0xa6, 0x67, 0xb6, - 0x8d, 0x82, 0x9e, 0x14, 0x67, 0xf0, 0x4a, 0xc4, 0x27, 0x32, 0xed, 0x0e, 0xb1, 0x01, 0x43, 0x0d, - 0xc3, 0x62, 0xe6, 0x1e, 0xf9, 0x94, 0xe8, 0xdc, 0x04, 0x8b, 0xa3, 0xc0, 0x3e, 0x2e, 0xf5, 0x2f, - 0x0a, 0x94, 0x41, 0x84, 0x97, 0xfa, 0x3d, 0x1e, 0x08, 0x57, 0x78, 0x5f, 0x2b, 0xf9, 0x95, 0x80, - 0x12, 0xd2, 0xf9, 0x00, 0x21, 0x37, 0x54, 0x08, 0x7f, 0x63, 0x89, 0x18, 0x25, 0xed, 0x19, 0x0d, - 0xf2, 0xac, 0x61, 0xd5, 0xec, 0x3b, 0x62, 0x7a, 0x7b, 0x25, 0xbd, 0x1e, 0x20, 0xe4, 0xa4, 0xc3, - 0xdf, 0x58, 0x22, 0x46, 0x8f, 0x24, 0x96, 0x8c, 0xc3, 0x62, 0x49, 0x95, 0x45, 0xdf, 0x78, 0xca, - 0x74, 0x61, 0x77, 0xcc, 0x8e, 0xa4, 0x52, 0x46, 0x1d, 0x9c, 0xd9, 0x5a, 0xfd, 0x79, 0x05, 0xce, - 0xa4, 0x4e, 0x05, 0xba, 0x0e, 0x53, 0xa1, 0x39, 0xac, 0x2c, 0x24, 0x0c, 0x87, 0x3c, 0xfe, 0x66, - 0xbc, 0x02, 0x4e, 0xb6, 0xe1, 0xa9, 0xea, 0x12, 0x42, 0x88, 0xb0, 0xa5, 0x95, 0xaf, 0x5a, 0x32, - 0x18, 0xa7, 0xb5, 0x51, 0x5f, 0x88, 0x74, 0x36, 0x9c, 0x2c, 0xfa, 0x65, 0x6c, 0x92, 0x7a, 0xe0, - 0x93, 0x1e, 0x7c, 0x19, 0x0b, 0xb4, 0x10, 0x73, 0x18, 0xba, 0x24, 0x87, 0xca, 0x08, 0xb8, 0xbb, - 0x1f, 0x2e, 0x43, 0xfd, 0x75, 0x05, 0xce, 0xad, 0x68, 0x56, 0x4b, 0x33, 0xb9, 0xf3, 0xfb, 0x9a, - 0x6d, 0x9b, 0x62, 0x9a, 0x3a, 0xa5, 0xce, 0x57, 0x5e, 0x2f, 0xa9, 0xf3, 0x3f, 0x00, 0xe7, 0x32, - 0xec, 0x6f, 0x50, 0x19, 0xc6, 0xdc, 0x3b, 0x5a, 0x73, 0x81, 0x6c, 0x6b, 0x3b, 0x86, 0x08, 0xac, - 0xc5, 0xcd, 0xb4, 0xc7, 0xaa, 0x52, 0xf9, 0xbd, 0xd8, 0x6f, 0x1c, 0x69, 0xa5, 0x7a, 0x00, 0xc2, - 0x9c, 0xdf, 0xb0, 0xea, 0x68, 0x0b, 0x86, 0x35, 0x93, 0x38, 0x5e, 0x18, 0x3c, 0xf2, 0xdb, 0x73, - 0xe9, 0x69, 0x05, 0x0e, 0xee, 0xbe, 0xe6, 0xff, 0xc2, 0x01, 0x6e, 0xf5, 0xff, 0x57, 0xe0, 0x6c, - 0x7a, 0x28, 0xa5, 0x2e, 0xae, 0x45, 0x0d, 0x18, 0x75, 0xc2, 0x66, 0xe2, 0xa3, 0x7d, 0xb7, 0x9c, - 0xa1, 0x42, 0x72, 0xbc, 0xa1, 0x17, 0xc1, 0x92, 0x63, 0xbb, 0xfe, 0xce, 0x8d, 0x27, 0xad, 0x08, - 0xb4, 0x62, 0x52, 0x4f, 0xb0, 0x8c, 0x9f, 0x25, 0x90, 0xa1, 0xd4, 0xdd, 0xa6, 0xa6, 0x93, 0xda, - 0x09, 0xa7, 0xc7, 0x3f, 0x82, 0xac, 0x0d, 0xe9, 0x7d, 0x3f, 0xde, 0x04, 0x32, 0x19, 0x34, 0x0f, - 0x4e, 0x20, 0x93, 0xde, 0xf0, 0x0d, 0x92, 0xd9, 0x20, 0xbd, 0xf3, 0x19, 0x8e, 0xef, 0xff, 0x79, - 0x30, 0x6b, 0xb4, 0x87, 0xcc, 0xb1, 0xbf, 0x73, 0x8c, 0x39, 0xf6, 0x27, 0xbe, 0x99, 0x5f, 0x3f, - 0x25, 0xbf, 0x7e, 0x2c, 0xe7, 0xfb, 0xe0, 0x09, 0xe5, 0x7c, 0x7f, 0x19, 0x06, 0x9b, 0x9a, 0x43, - 0x2c, 0xff, 0x2d, 0xb7, 0x92, 0xcf, 0xd0, 0x20, 0xdc, 0x68, 0x21, 0x17, 0x0c, 0x3e, 0xc9, 0x35, - 0x46, 0x00, 0x0b, 0x42, 0x29, 0xc1, 0x53, 0x86, 0x8f, 0x2b, 0x3a, 0x61, 0x98, 0x6d, 0x7e, 0xe4, - 0x38, 0xb2, 0xcd, 0xab, 0x7f, 0xa7, 0xc0, 0xc5, 0x4e, 0x6c, 0x89, 0x29, 0x50, 0xf4, 0xd8, 0x67, - 0xd8, 0x8b, 0x02, 0x25, 0xc1, 0x6d, 0x03, 0x05, 0x4a, 0x1c, 0x82, 0x13, 0x74, 0xd1, 0xfb, 0x00, - 0xd9, 0x9b, 0xdc, 0xae, 0xe7, 0x3a, 0xa5, 0xc1, 0xbd, 0x69, 0x0b, 0xcc, 0x57, 0x21, 0xc8, 0x6b, - 0x79, 0x2b, 0x51, 0x03, 0xa7, 0xb4, 0x52, 0x7f, 0xb5, 0x00, 0xb0, 0x4a, 0xbc, 0x3b, 0xb6, 0x73, - 0x9b, 0x9e, 0xf1, 0x17, 0x23, 0x4f, 0x07, 0xc3, 0xdf, 0xb8, 0x78, 0x94, 0x17, 0xa1, 0xbf, 0x69, - 0xd7, 0x5c, 0x71, 0xff, 0x60, 0x1d, 0x61, 0xae, 0x1a, 0xac, 0x14, 0x15, 0x61, 0x80, 0x19, 0x3d, - 0x89, 0x0b, 0x34, 0x7b, 0x78, 0x60, 0x8f, 0x2b, 0x98, 0x97, 0x53, 0x0e, 0x29, 0x62, 0x1a, 0xb8, - 0xe2, 0x71, 0x67, 0x8c, 0xc7, 0x10, 0xe7, 0x65, 0x38, 0x80, 0xa2, 0xc7, 0x00, 0x8c, 0xe6, 0x92, - 0xd6, 0x30, 0x4c, 0x43, 0x7c, 0xae, 0x23, 0x4c, 0x23, 0x0e, 0x95, 0x35, 0xbf, 0xf4, 0x5e, 0xbb, - 0x38, 0x2c, 0x7e, 0xed, 0x61, 0xa9, 0xb6, 0xfa, 0x45, 0x05, 0x26, 0xc3, 0xc9, 0x13, 0x5b, 0xc5, - 0xef, 0x39, 0x0f, 0x06, 0x9c, 0xd9, 0x73, 0x1e, 0xb6, 0xbf, 0x73, 0xcf, 0xb9, 0x02, 0x2b, 0xab, - 0xe7, 0x0f, 0xc2, 0x28, 0xe1, 0x21, 0x8f, 0x2a, 0x65, 0xcc, 0x79, 0xdc, 0x08, 0xbf, 0xce, 0x2d, - 0x86, 0xc5, 0x58, 0xae, 0xa3, 0xfe, 0x43, 0x1f, 0x8c, 0xad, 0xd6, 0x0d, 0x6b, 0xd7, 0x8f, 0xed, - 0x14, 0x3c, 0xbc, 0x2b, 0xc7, 0xf3, 0xf0, 0xfe, 0x1c, 0xcc, 0x98, 0xf2, 0x33, 0x17, 0x17, 0x9c, - 0x34, 0xab, 0x1e, 0xcc, 0x00, 0xbb, 0xc7, 0x2c, 0x67, 0xd4, 0xc1, 0x99, 0xad, 0x91, 0x07, 0x83, - 0xba, 0x9f, 0x5f, 0x31, 0x77, 0xbc, 0x22, 0x79, 0x2e, 0xe6, 0xe4, 0xd0, 0x1d, 0x01, 0xcf, 0x13, - 0xdb, 0x53, 0xd0, 0x42, 0x1f, 0x53, 0xe0, 0x0c, 0xd9, 0xe5, 0xa1, 0x6b, 0xd6, 0x1d, 0x6d, 0x6b, - 0xcb, 0xd0, 0x85, 0xc7, 0x1f, 0xdf, 0x89, 0xcb, 0xfb, 0xed, 0xe2, 0x99, 0xc5, 0xb4, 0x0a, 0xf7, - 0xda, 0xc5, 0x6b, 0xa9, 0x91, 0x84, 0xd8, 0x6a, 0xa6, 0x36, 0xc1, 0xe9, 0xa4, 0x66, 0x1f, 0x85, - 0xd1, 0x43, 0xf8, 0x89, 0x47, 0xe2, 0x05, 0xfd, 0x5a, 0x01, 0xc6, 0xe8, 0x76, 0x5b, 0xb6, 0x75, - 0xcd, 0x2c, 0xaf, 0x56, 0xd1, 0xfd, 0xf1, 0x30, 0x89, 0xc1, 0x13, 0x63, 0x22, 0x54, 0xe2, 0x32, - 0x4c, 0x6f, 0xd9, 0x8e, 0x4e, 0xd6, 0x4b, 0x6b, 0xeb, 0xb6, 0x30, 0x3e, 0x2b, 0xaf, 0x56, 0xc5, - 0xbd, 0x8e, 0x3d, 0x79, 0x2c, 0xa5, 0xc0, 0x71, 0x6a, 0x2b, 0x74, 0x0b, 0xce, 0x84, 0xe5, 0x1b, - 0x4d, 0xee, 0xb0, 0x40, 0xd1, 0xf5, 0x85, 0x0e, 0x17, 0x4b, 0x69, 0x15, 0x70, 0x7a, 0x3b, 0xa4, - 0xc1, 0x05, 0xf1, 0xb6, 0xb2, 0x64, 0x3b, 0x77, 0x34, 0xa7, 0x16, 0x45, 0xdb, 0x1f, 0xa6, 0x9d, - 0x2f, 0x67, 0x57, 0xc3, 0x9d, 0x70, 0xa8, 0x7f, 0xaf, 0x40, 0x34, 0x08, 0x25, 0x3a, 0x0f, 0x7d, - 0x8e, 0x48, 0xdf, 0x27, 0x82, 0x31, 0xd2, 0x2b, 0x02, 0x2d, 0x43, 0x73, 0x00, 0x4e, 0x18, 0x09, - 0xb3, 0x10, 0xa6, 0xb5, 0x90, 0x62, 0x58, 0x4a, 0x35, 0x28, 0x2a, 0x4f, 0xab, 0x0b, 0x86, 0xc7, - 0x50, 0xad, 0x6b, 0x75, 0x4c, 0xcb, 0x58, 0xfe, 0x12, 0xa3, 0x4e, 0x5c, 0x5f, 0x1d, 0xcd, 0xf3, - 0x97, 0xb0, 0x12, 0x2c, 0x20, 0x48, 0x83, 0xf1, 0x66, 0xcb, 0x14, 0xf1, 0x94, 0xe8, 0xd5, 0x87, - 0xab, 0x0d, 0xaf, 0xa6, 0x25, 0xe7, 0x63, 0xab, 0x9f, 0x9a, 0xa1, 0x6f, 0x4d, 0x46, 0x81, 0xa3, - 0x18, 0xd5, 0x9f, 0x18, 0x04, 0x29, 0xbc, 0xce, 0x21, 0xa4, 0xd0, 0x9f, 0x51, 0x60, 0x5a, 0x37, - 0x0d, 0x62, 0x79, 0xb1, 0x48, 0x15, 0xfc, 0xf8, 0xd8, 0xc8, 0x15, 0xf7, 0xa7, 0x49, 0xac, 0x4a, - 0x59, 0xb8, 0xb7, 0x94, 0x52, 0x90, 0x0b, 0x17, 0xa0, 0x14, 0x08, 0x4e, 0xed, 0x0c, 0x1b, 0x0f, - 0x2b, 0xaf, 0x94, 0xe5, 0xe8, 0x99, 0x25, 0x51, 0x86, 0x03, 0x28, 0xe5, 0xbc, 0x75, 0xc7, 0x6e, - 0x35, 0xdd, 0x12, 0xf3, 0x62, 0xe5, 0x8b, 0xc2, 0x38, 0xef, 0xf5, 0xb0, 0x18, 0xcb, 0x75, 0xd0, - 0xc3, 0x30, 0xc6, 0x7f, 0xae, 0x39, 0x64, 0xcb, 0xd8, 0x15, 0x87, 0x12, 0x53, 0x9e, 0x5e, 0x97, - 0xca, 0x71, 0xa4, 0x16, 0x8b, 0xdf, 0xe6, 0xba, 0x2d, 0xe2, 0x6c, 0xe0, 0x65, 0x91, 0xe3, 0x99, - 0xc7, 0x6f, 0xf3, 0x0b, 0x71, 0x08, 0x47, 0x3f, 0xa2, 0xc0, 0x84, 0x43, 0x5e, 0x6e, 0x19, 0x0e, - 0x15, 0x61, 0x34, 0xa3, 0xe1, 0x8a, 0x18, 0x47, 0xb8, 0xb7, 0xb8, 0x4a, 0x73, 0x38, 0x82, 0x94, - 0x33, 0xc8, 0xc0, 0x94, 0x23, 0x0a, 0xc4, 0xb1, 0x1e, 0xd0, 0xa9, 0x72, 0x8d, 0xba, 0x65, 0x58, - 0xf5, 0x79, 0xb3, 0xee, 0x8a, 0x44, 0x56, 0x5c, 0xe7, 0x18, 0x16, 0x63, 0xb9, 0x0e, 0x7a, 0x04, - 0xc6, 0x5b, 0x2e, 0x65, 0x7b, 0x0d, 0xc2, 0xe7, 0x77, 0x24, 0xb4, 0x75, 0xd9, 0x90, 0x01, 0x38, - 0x5a, 0x0f, 0x3d, 0x06, 0x13, 0x7e, 0x81, 0x98, 0x65, 0xe0, 0xd9, 0x54, 0xd8, 0xdb, 0x64, 0x04, - 0x82, 0x63, 0x35, 0x67, 0xe7, 0xe1, 0x74, 0xca, 0x30, 0x0f, 0xc5, 0x5b, 0xff, 0x97, 0x02, 0x67, - 0xb8, 0xd4, 0xe5, 0x67, 0x87, 0xf6, 0x73, 0x5b, 0xa4, 0xa7, 0x89, 0x50, 0x8e, 0x35, 0x4d, 0xc4, - 0x37, 0x20, 0x1d, 0x86, 0xfa, 0x73, 0x05, 0x78, 0xf3, 0x81, 0xdf, 0x25, 0xfa, 0x49, 0x05, 0x46, - 0xc9, 0xae, 0xe7, 0x68, 0x81, 0xab, 0x3f, 0xdd, 0xa4, 0x5b, 0xc7, 0xc2, 0x04, 0xe6, 0x16, 0x43, - 0x42, 0x7c, 0xe3, 0x06, 0x57, 0x29, 0x09, 0x82, 0xe5, 0xfe, 0x50, 0x6e, 0xcb, 0xb3, 0x0d, 0xc9, - 0x46, 0x71, 0x82, 0x0b, 0x0a, 0xc8, 0xec, 0x13, 0x30, 0x19, 0xc7, 0x7c, 0xa8, 0xbd, 0xf2, 0x05, - 0x05, 0x52, 0xc3, 0x71, 0xa2, 0x12, 0x4c, 0x69, 0x2d, 0xcf, 0x8e, 0xbc, 0xeb, 0x09, 0x51, 0x92, - 0x19, 0x7d, 0xcf, 0xc7, 0x81, 0x38, 0x59, 0x9f, 0x2b, 0x60, 0xad, 0x96, 0x66, 0x46, 0xd1, 0x70, - 0x81, 0x4b, 0x28, 0x60, 0x13, 0x60, 0x9c, 0xd6, 0x46, 0xfd, 0x67, 0x0a, 0x9c, 0x49, 0x55, 0x4b, - 0x76, 0xa1, 0x8b, 0x4b, 0xdf, 0xf6, 0x85, 0xe3, 0xdc, 0xf6, 0xea, 0xaf, 0x14, 0x60, 0x68, 0xcd, - 0xb1, 0xe9, 0xd1, 0x76, 0x02, 0x1a, 0x38, 0x2d, 0xa2, 0x81, 0xcb, 0xa5, 0x5f, 0x10, 0x9d, 0xcd, - 0x54, 0xb9, 0x19, 0x31, 0x95, 0xdb, 0x7c, 0x2f, 0x44, 0x3a, 0xeb, 0xd8, 0xbe, 0xac, 0xc0, 0xa8, - 0xa8, 0x79, 0x02, 0x4a, 0xb5, 0xef, 0x8a, 0x2a, 0xd5, 0xde, 0xdb, 0xc3, 0xb8, 0x32, 0xb4, 0x68, - 0x9f, 0x55, 0x60, 0x5c, 0xd4, 0x58, 0x21, 0x8d, 0x4d, 0xe2, 0xa0, 0x25, 0x18, 0x72, 0x5b, 0x6c, - 0x21, 0xc5, 0x80, 0x2e, 0xc8, 0xe2, 0x91, 0xb3, 0xa9, 0xe9, 0xb4, 0xfb, 0x55, 0x5e, 0x45, 0xca, - 0x84, 0xcc, 0x0b, 0xb0, 0xdf, 0x98, 0xee, 0x7d, 0xc7, 0x36, 0x13, 0x61, 0xeb, 0xb1, 0x6d, 0x12, - 0xcc, 0x20, 0xf4, 0x9e, 0x47, 0xff, 0xfa, 0x77, 0x38, 0x76, 0xcf, 0xa3, 0x60, 0x17, 0xf3, 0x72, - 0xf5, 0x8b, 0x03, 0xc1, 0x64, 0xb3, 0x4b, 0xfd, 0x0d, 0x18, 0xd1, 0x1d, 0xa2, 0x79, 0xa4, 0xb6, - 0xb0, 0xd7, 0x4d, 0xe7, 0x98, 0x1c, 0x50, 0xf2, 0x5b, 0xe0, 0xb0, 0x31, 0x3d, 0x72, 0x65, 0x03, - 0xcf, 0x42, 0x28, 0x9d, 0x64, 0x1a, 0x77, 0x7e, 0x3b, 0x0c, 0xd8, 0x77, 0xac, 0xc0, 0x39, 0xa4, - 0x23, 0x61, 0x36, 0x94, 0x5b, 0xb4, 0x36, 0xe6, 0x8d, 0xe4, 0xb4, 0x0d, 0xfd, 0x1d, 0xd2, 0x36, - 0x98, 0x30, 0xd4, 0x60, 0xcb, 0xd0, 0x53, 0x62, 0xdc, 0xc8, 0x82, 0x86, 0x4b, 0xc4, 0x7f, 0xbb, - 0xd8, 0x27, 0x41, 0x45, 0x27, 0xcb, 0xd7, 0xe8, 0xc8, 0xa2, 0x53, 0xa0, 0xe6, 0xc1, 0x21, 0x1c, - 0xed, 0x45, 0xf3, 0x81, 0x0c, 0xe5, 0xd7, 0x93, 0x8a, 0xee, 0x49, 0x29, 0x40, 0xf8, 0xd4, 0x67, - 0xe5, 0x04, 0x41, 0x5f, 0x50, 0xe0, 0x5c, 0x2d, 0x3d, 0xe1, 0x9a, 0x78, 0x2a, 0xcf, 0xe5, 0x98, - 0x9d, 0x91, 0xc3, 0x6d, 0xa1, 0x28, 0x26, 0x2c, 0x2b, 0xc9, 0x1b, 0xce, 0xea, 0x8c, 0xfa, 0xfd, - 0xfd, 0xc1, 0xd7, 0x24, 0x34, 0x1d, 0xe9, 0x7a, 0x28, 0x25, 0x8f, 0x1e, 0x0a, 0xbd, 0xd3, 0x4f, - 0xff, 0xc5, 0xb7, 0xeb, 0xa5, 0x78, 0xfa, 0xaf, 0x31, 0x41, 0x3a, 0x92, 0xf2, 0xab, 0x05, 0xa7, - 0x5d, 0x4f, 0x33, 0x49, 0xd5, 0x10, 0x8f, 0x83, 0xae, 0xa7, 0x35, 0x9a, 0x39, 0xf2, 0x6f, 0xf1, - 0x40, 0x0d, 0x49, 0x54, 0x38, 0x0d, 0x3f, 0xfa, 0x5e, 0x16, 0x7c, 0x4f, 0x33, 0xd9, 0xe3, 0x29, - 0xcf, 0x41, 0x1a, 0x12, 0x3f, 0xbc, 0xb9, 0xbb, 0x08, 0xad, 0x97, 0x8e, 0x0f, 0x67, 0x52, 0x42, - 0xaf, 0xc0, 0x19, 0x7a, 0x00, 0xce, 0xeb, 0x9e, 0xb1, 0x63, 0x78, 0x7b, 0x61, 0x17, 0x0e, 0x9f, - 0x74, 0x8b, 0xdd, 0xb6, 0x97, 0xd3, 0x90, 0xe1, 0x74, 0x1a, 0xea, 0xdf, 0x2a, 0x80, 0x92, 0x7b, - 0x1d, 0x99, 0x30, 0x5c, 0xf3, 0x23, 0x27, 0x28, 0x47, 0x92, 0x55, 0x27, 0x38, 0x42, 0x82, 0x80, - 0x0b, 0x01, 0x05, 0x64, 0xc3, 0xc8, 0x9d, 0x6d, 0xc3, 0x23, 0xa6, 0xe1, 0x7a, 0x47, 0x94, 0xc4, - 0x27, 0x88, 0x09, 0xff, 0xac, 0x8f, 0x18, 0x87, 0x34, 0xd4, 0x4f, 0xf5, 0xc3, 0x70, 0x90, 0x4c, - 0xf3, 0x60, 0x33, 0xeb, 0x16, 0x20, 0x5d, 0x0a, 0x08, 0xd8, 0x8b, 0xce, 0x94, 0xc9, 0x40, 0xa5, - 0x04, 0x32, 0x9c, 0x42, 0x00, 0xbd, 0x02, 0xd3, 0x86, 0xb5, 0xe5, 0x68, 0x41, 0xb8, 0xc3, 0x92, - 0xaf, 0x28, 0xcb, 0x41, 0x98, 0xdd, 0xa2, 0x2b, 0x29, 0xe8, 0x70, 0x2a, 0x11, 0x44, 0x60, 0x88, - 0xe7, 0x33, 0xf6, 0x5f, 0x5d, 0x72, 0xe9, 0xea, 0xb9, 0xac, 0x19, 0xb2, 0x77, 0x5f, 0xf6, 0xf4, - 0x71, 0xf3, 0xe0, 0xb4, 0xfc, 0x7f, 0xff, 0x41, 0x4a, 0xec, 0xfb, 0x52, 0x7e, 0x7a, 0xe1, 0xdb, - 0x16, 0x0f, 0x4e, 0x1b, 0x2d, 0xc4, 0x71, 0x82, 0xea, 0xef, 0x29, 0x30, 0xc0, 0x63, 0x80, 0x1d, - 0xbf, 0xa8, 0xf9, 0x81, 0x88, 0xa8, 0xf9, 0x78, 0x9e, 0x41, 0xb2, 0xae, 0x66, 0x66, 0xe4, 0xff, - 0x5d, 0x05, 0x46, 0x58, 0x8d, 0x13, 0x90, 0xfd, 0x5e, 0x8c, 0xca, 0x7e, 0x8f, 0xe6, 0x1e, 0x4d, - 0x86, 0xe4, 0xf7, 0x7b, 0x7d, 0x62, 0x2c, 0x4c, 0xb4, 0xaa, 0xc0, 0x69, 0xe1, 0x18, 0xbb, 0x6c, - 0x6c, 0x11, 0xba, 0xc5, 0xcb, 0xda, 0x9e, 0x2b, 0x5c, 0x35, 0x78, 0xd0, 0x99, 0x24, 0x18, 0xa7, - 0xb5, 0x41, 0xbf, 0xa6, 0x50, 0x21, 0xc6, 0x73, 0x0c, 0xbd, 0xa7, 0xc7, 0xe0, 0xa0, 0x6f, 0x73, - 0x2b, 0x1c, 0x19, 0xbf, 0x9b, 0x6e, 0x84, 0xd2, 0x0c, 0x2b, 0xbd, 0xd7, 0x2e, 0x16, 0x53, 0x74, - 0xc6, 0x61, 0x56, 0x6a, 0xd7, 0xfb, 0xd8, 0x5f, 0x76, 0xac, 0xc2, 0x6e, 0x63, 0x7e, 0x8f, 0xd1, - 0x0d, 0x18, 0x70, 0x75, 0xbb, 0xe9, 0xbb, 0x56, 0xdf, 0x97, 0xa6, 0x1b, 0x8c, 0xab, 0x05, 0x83, - 0x09, 0xae, 0xd2, 0x96, 0x98, 0x23, 0x98, 0x7d, 0x09, 0xc6, 0xe4, 0x9e, 0xa7, 0xdc, 0x7d, 0xcb, - 0x51, 0x8b, 0xb9, 0x43, 0x1a, 0x87, 0xc9, 0x77, 0xe5, 0x3f, 0xee, 0x83, 0x41, 0x4c, 0xea, 0x22, - 0x65, 0xda, 0x01, 0x77, 0x4e, 0xc3, 0x4f, 0xff, 0x5b, 0xc8, 0xef, 0x87, 0x27, 0x27, 0xc5, 0x79, - 0xde, 0xb6, 0xa4, 0x39, 0x90, 0x33, 0x00, 0x23, 0x2b, 0x48, 0x24, 0xd5, 0x97, 0xdf, 0xa0, 0x96, - 0x0f, 0xac, 0x9b, 0xd4, 0x51, 0xe8, 0x87, 0x15, 0x40, 0x9a, 0xae, 0x13, 0xd7, 0xc5, 0xc4, 0xa5, - 0x73, 0xcf, 0x85, 0x55, 0xce, 0x65, 0xf3, 0x45, 0xc5, 0x8e, 0x63, 0x0b, 0xc5, 0xb6, 0x04, 0xc8, - 0xc5, 0x29, 0xc4, 0x7b, 0x49, 0x67, 0xf5, 0x07, 0x0a, 0x8c, 0x45, 0xb2, 0x85, 0x35, 0x42, 0x5d, - 0x7a, 0x7e, 0x93, 0x1d, 0xdf, 0xfb, 0xeb, 0x42, 0x87, 0x4a, 0x5c, 0x3f, 0x7f, 0x2b, 0x48, 0x77, - 0x71, 0x34, 0x89, 0xc5, 0xd4, 0x4f, 0x2b, 0x70, 0xd6, 0x1f, 0x50, 0x34, 0xae, 0x39, 0xba, 0x0a, - 0xc3, 0x5a, 0xd3, 0x60, 0x8a, 0x5e, 0x59, 0x55, 0x3e, 0xbf, 0x56, 0x61, 0x65, 0x38, 0x80, 0x46, - 0x72, 0x2c, 0x17, 0x0e, 0xcc, 0xb1, 0xfc, 0x56, 0x29, 0x6b, 0xf4, 0x40, 0x28, 0xbb, 0x04, 0x84, - 0xb9, 0x31, 0xa7, 0xfa, 0x6e, 0x18, 0xa9, 0x56, 0x6f, 0xf0, 0x25, 0x3d, 0xc4, 0x8b, 0x8f, 0xfa, - 0xc9, 0x3e, 0x18, 0x17, 0x09, 0x1a, 0x0c, 0xa6, 0x02, 0x3a, 0x81, 0x73, 0x6e, 0x1d, 0x46, 0xdc, - 0xe0, 0x0d, 0xa3, 0x90, 0xcd, 0xa7, 0x82, 0x67, 0x89, 0x78, 0x96, 0xbd, 0xf0, 0xf9, 0x22, 0x44, - 0x84, 0x6e, 0xc2, 0xe0, 0xcb, 0x94, 0xe7, 0xfa, 0xdf, 0x6a, 0x57, 0xac, 0x2f, 0xf8, 0x10, 0x19, - 0xbb, 0x76, 0xb1, 0x40, 0x81, 0x5c, 0xe6, 0x9e, 0xc8, 0x84, 0xc0, 0x5e, 0x42, 0x75, 0x46, 0x66, - 0x36, 0xc8, 0x19, 0x3f, 0x26, 0xbc, 0x1c, 0xd9, 0x2f, 0x1c, 0x10, 0x62, 0x29, 0x42, 0x23, 0x2d, - 0xde, 0x20, 0x29, 0x42, 0x23, 0x7d, 0xce, 0x38, 0xae, 0x1f, 0x85, 0x33, 0xa9, 0x93, 0x71, 0xb0, - 0x88, 0xad, 0xfe, 0xf3, 0x02, 0xf4, 0x57, 0x09, 0xa9, 0x9d, 0xc0, 0xce, 0x7c, 0x31, 0x22, 0x81, - 0x7d, 0x7b, 0xee, 0x24, 0xa5, 0x59, 0x9a, 0xbe, 0xad, 0x98, 0xa6, 0xef, 0x89, 0xdc, 0x14, 0x3a, - 0xab, 0xf9, 0xfe, 0xba, 0x00, 0x43, 0xb4, 0x5a, 0x79, 0xb5, 0x8a, 0x1a, 0xd2, 0x56, 0x2e, 0xe4, - 0x17, 0x9f, 0x05, 0xba, 0x83, 0x36, 0x31, 0xfd, 0x72, 0x0c, 0x91, 0xf8, 0x45, 0x0c, 0xb2, 0x72, - 0x04, 0xe4, 0x44, 0xec, 0x09, 0x46, 0xd4, 0xcf, 0x2b, 0x83, 0x03, 0x42, 0xe8, 0x8e, 0x74, 0x27, - 0xed, 0xcf, 0x6f, 0xbb, 0x9b, 0x4e, 0xb4, 0xc3, 0xf5, 0x54, 0xfd, 0xa4, 0x02, 0xa7, 0x62, 0xad, - 0xba, 0xb8, 0x34, 0x1e, 0x0b, 0x03, 0x54, 0xbf, 0xae, 0xd0, 0x6f, 0x2c, 0x65, 0x04, 0x5d, 0xf4, - 0x28, 0x4c, 0x00, 0x56, 0xe8, 0x98, 0x00, 0xec, 0x22, 0xf4, 0x53, 0xc9, 0x48, 0xb6, 0xd9, 0xa1, - 0x02, 0x13, 0x66, 0xa5, 0x48, 0x87, 0x09, 0x5d, 0x4a, 0xe1, 0x4e, 0xb6, 0x04, 0xef, 0xec, 0x8a, - 0x15, 0x07, 0xcf, 0x8d, 0xa5, 0x08, 0x0a, 0x1c, 0x43, 0xa9, 0xfe, 0x8e, 0x02, 0xc3, 0x74, 0x98, - 0x27, 0xc0, 0x1c, 0xbf, 0x33, 0xca, 0x1c, 0xdf, 0x93, 0x77, 0x4f, 0x65, 0xf0, 0xc4, 0xbf, 0x29, - 0x00, 0xcb, 0x60, 0x2c, 0xec, 0x8b, 0x24, 0xcb, 0x21, 0x25, 0xc3, 0xe6, 0xe9, 0x8a, 0x30, 0x3c, - 0x8a, 0x29, 0xa5, 0x25, 0xe3, 0xa3, 0xb7, 0x47, 0x6c, 0x8b, 0x22, 0xc2, 0x45, 0x8a, 0x7d, 0xd1, - 0x5d, 0x18, 0x77, 0xb7, 0x6d, 0xdb, 0x2b, 0x87, 0x1f, 0x4f, 0xee, 0x07, 0x08, 0xe6, 0x2c, 0xee, - 0x0f, 0x85, 0x3f, 0xe5, 0x56, 0x65, 0xdc, 0x38, 0x4a, 0x0a, 0xcd, 0x01, 0x6c, 0x9a, 0xb6, 0x7e, - 0x9b, 0x9b, 0x36, 0x71, 0xe7, 0x60, 0x66, 0x3c, 0xb1, 0x10, 0x94, 0x62, 0xa9, 0x46, 0x4f, 0x56, - 0x5c, 0x5f, 0x57, 0xf8, 0x4c, 0x1f, 0xe2, 0x1b, 0x3d, 0x41, 0x43, 0xb8, 0xb7, 0xc1, 0xa0, 0xc3, - 0xee, 0x08, 0x62, 0xc5, 0x82, 0x8f, 0x8f, 0xdf, 0x1c, 0xb0, 0x80, 0xd2, 0xed, 0xc1, 0x2f, 0x3e, - 0xfd, 0xe1, 0x83, 0x83, 0x7c, 0x5d, 0x51, 0x7f, 0x45, 0x0c, 0x33, 0x48, 0x82, 0xdd, 0x84, 0x71, - 0x76, 0xb3, 0x88, 0x65, 0xdf, 0x7e, 0x67, 0x97, 0xdf, 0x88, 0xdc, 0x34, 0x34, 0xcb, 0x8d, 0x14, - 0xe3, 0x28, 0x01, 0xf4, 0x08, 0x8c, 0xfb, 0xa3, 0xe3, 0xd6, 0xb1, 0x85, 0xd0, 0x43, 0x73, 0x4d, - 0x06, 0xe0, 0x68, 0x3d, 0xf5, 0x33, 0x05, 0xb8, 0xc4, 0xfb, 0xce, 0x34, 0x2f, 0x65, 0xd2, 0x24, - 0x56, 0x8d, 0x58, 0xfa, 0x1e, 0x93, 0xb3, 0x6b, 0x76, 0x1d, 0xbd, 0x02, 0x83, 0x77, 0x08, 0xa9, - 0x05, 0x4f, 0x18, 0xcf, 0xe6, 0xcf, 0x21, 0x9e, 0x41, 0xe2, 0x59, 0x86, 0x9e, 0xbf, 0x06, 0xf3, - 0xff, 0xb1, 0x20, 0x49, 0x89, 0x37, 0x1d, 0x7b, 0x33, 0x10, 0x07, 0x8f, 0x9e, 0xf8, 0x1a, 0x43, - 0xcf, 0x89, 0xf3, 0xff, 0xb1, 0x20, 0xa9, 0xae, 0xc1, 0x7d, 0x5d, 0x34, 0x3d, 0x8c, 0xd8, 0x7f, - 0x10, 0x46, 0x3e, 0xfa, 0xc3, 0x60, 0xfc, 0x53, 0x05, 0xde, 0x22, 0xa1, 0x5c, 0xdc, 0xa5, 0x37, - 0x91, 0x92, 0xd6, 0xd4, 0x74, 0x7a, 0xd7, 0x67, 0x31, 0xea, 0x0e, 0x95, 0xb5, 0xf7, 0x93, 0x0a, - 0x0c, 0x71, 0x8b, 0x3c, 0x9f, 0xfd, 0xbe, 0xd8, 0xe3, 0x94, 0x67, 0x76, 0xc9, 0xcf, 0x66, 0xe6, - 0x8f, 0x8d, 0xff, 0x76, 0xb1, 0x4f, 0x5f, 0xfd, 0x57, 0x03, 0xf0, 0xad, 0xdd, 0x23, 0x42, 0x5f, - 0x57, 0xe4, 0x6c, 0xe3, 0x5c, 0x47, 0xde, 0x38, 0xde, 0xce, 0x07, 0xda, 0x20, 0xa1, 0x60, 0x78, - 0x36, 0x91, 0x90, 0xfc, 0x88, 0x14, 0x4d, 0xe1, 0xc0, 0xd0, 0x3f, 0x51, 0x60, 0x8c, 0x1e, 0x4b, - 0x01, 0x73, 0xe1, 0xcb, 0xd4, 0x3c, 0xe6, 0x91, 0xae, 0x4a, 0x24, 0x63, 0x71, 0xad, 0x64, 0x10, - 0x8e, 0xf4, 0x0d, 0x6d, 0x44, 0x9f, 0xff, 0xf8, 0x15, 0xf1, 0x72, 0x9a, 0x5c, 0x72, 0x98, 0x74, - 0xff, 0xb3, 0x26, 0x4c, 0x44, 0x67, 0xfe, 0x38, 0xd5, 0x64, 0xb3, 0x4f, 0xc2, 0x54, 0x62, 0xf4, - 0x87, 0x52, 0xc8, 0x7c, 0x79, 0x10, 0x8a, 0xd2, 0x54, 0xa7, 0x85, 0xa7, 0x41, 0x9f, 0x53, 0x60, - 0x54, 0xb3, 0x2c, 0x61, 0xd8, 0xe4, 0xef, 0xdf, 0x5a, 0x8f, 0xab, 0x9a, 0x46, 0x6a, 0x6e, 0x3e, - 0x24, 0x13, 0xb3, 0xdc, 0x91, 0x20, 0x58, 0xee, 0x4d, 0x07, 0xeb, 0xdc, 0xc2, 0x89, 0x59, 0xe7, - 0xa2, 0x0f, 0xf9, 0x07, 0x71, 0x5f, 0xfe, 0x40, 0x15, 0x07, 0xcc, 0x0d, 0x3b, 0xd7, 0x33, 0xb4, - 0x92, 0x3f, 0xa8, 0xb0, 0x43, 0x36, 0x8c, 0x22, 0x24, 0xce, 0xa4, 0x5c, 0x46, 0x96, 0x07, 0x86, - 0x28, 0x0a, 0xce, 0xee, 0xb0, 0x08, 0x47, 0xc9, 0xa3, 0x1f, 0x53, 0x60, 0xec, 0xae, 0x6d, 0x69, - 0xa6, 0xb0, 0xb3, 0x16, 0xcf, 0x34, 0x1f, 0x38, 0x9e, 0x79, 0x09, 0xc8, 0x70, 0xdb, 0x4a, 0xb9, - 0x04, 0x47, 0xba, 0xc1, 0x32, 0xe6, 0x33, 0xdf, 0xea, 0xc1, 0x50, 0x9e, 0x96, 0xfd, 0xaa, 0x67, - 0x9f, 0x80, 0xc9, 0xf8, 0x1e, 0x3c, 0xd4, 0xf7, 0xb4, 0x0e, 0x0f, 0x1c, 0xa2, 0xbf, 0x5d, 0x1e, - 0x7d, 0xea, 0x6f, 0xf6, 0x47, 0x8e, 0xd2, 0xcc, 0xed, 0xd1, 0x85, 0xae, 0xfc, 0xb5, 0xd8, 0xc7, - 0xcc, 0x59, 0xb4, 0x71, 0x5c, 0x1b, 0xf6, 0x68, 0xbf, 0xe8, 0xbe, 0x93, 0xfb, 0xa2, 0x5f, 0x6f, - 0x9f, 0x54, 0xcf, 0x3b, 0x73, 0x81, 0xeb, 0x02, 0xc4, 0x82, 0x85, 0xc9, 0x9a, 0x58, 0xa0, 0x4e, - 0xc3, 0x35, 0xfc, 0x70, 0xd3, 0x92, 0x48, 0xf7, 0x0c, 0x2f, 0xc6, 0x3e, 0x5c, 0x5d, 0x8e, 0x1c, - 0x16, 0xeb, 0x76, 0xd3, 0x36, 0xed, 0xfa, 0xde, 0xfc, 0x1d, 0xcd, 0x21, 0xd8, 0x6e, 0x79, 0x02, - 0x5b, 0xb7, 0x02, 0xe2, 0x47, 0x07, 0xe0, 0x8a, 0x84, 0x2e, 0x35, 0x2a, 0xe7, 0x61, 0x7c, 0x15, - 0x5e, 0x8b, 0x07, 0xcf, 0x2c, 0xe4, 0x37, 0x32, 0x3d, 0xa8, 0x5f, 0xf9, 0x82, 0x69, 0xa2, 0xb6, - 0x02, 0xd0, 0xd0, 0x76, 0x45, 0x24, 0x37, 0x71, 0x5a, 0xd4, 0x8e, 0xa5, 0x83, 0x2b, 0x01, 0x19, - 0xde, 0xbd, 0xe7, 0x7c, 0x3d, 0x67, 0x08, 0x38, 0x22, 0x09, 0x50, 0x1a, 0x51, 0xcf, 0xd1, 0x42, - 0x67, 0x1b, 0x70, 0x2a, 0xd6, 0xf3, 0x63, 0x7d, 0x67, 0xfc, 0xf2, 0x90, 0x7f, 0x41, 0x16, 0x21, - 0xe6, 0x7e, 0x59, 0x81, 0xf3, 0x24, 0x4b, 0xe0, 0x14, 0xb7, 0xe5, 0xe7, 0x8e, 0x4b, 0xa0, 0x15, - 0x09, 0xb7, 0xb2, 0xc0, 0x38, 0xbb, 0x67, 0x68, 0x0f, 0xc0, 0x0d, 0xbe, 0xe9, 0x5e, 0xe2, 0x56, - 0xa4, 0x32, 0x09, 0xae, 0x87, 0x09, 0x7f, 0x63, 0x89, 0x18, 0xfa, 0x69, 0x05, 0xa6, 0xcd, 0x14, - 0x0e, 0x27, 0x38, 0x66, 0xf5, 0x18, 0xce, 0x16, 0x6e, 0xa1, 0x92, 0x06, 0xc1, 0xa9, 0x5d, 0x41, - 0x3f, 0x9b, 0x19, 0x62, 0x98, 0x4b, 0x26, 0xeb, 0xc7, 0xf1, 0x0d, 0xe6, 0x88, 0x36, 0xfc, 0x19, - 0x05, 0x50, 0x2d, 0x71, 0xf9, 0x16, 0xc6, 0x89, 0x4f, 0x1f, 0xb9, 0x8a, 0x81, 0x9b, 0x18, 0x25, - 0xcb, 0x71, 0x4a, 0x27, 0xd8, 0x3a, 0x7b, 0x29, 0x3c, 0x5f, 0xb8, 0x00, 0xf7, 0xba, 0xce, 0x69, - 0xc7, 0x09, 0x5f, 0xe7, 0x34, 0x08, 0x4e, 0xed, 0x8a, 0xfa, 0xdb, 0x23, 0x5c, 0x17, 0xcc, 0x6c, - 0x40, 0x5e, 0x84, 0xc1, 0x4d, 0x4d, 0xbf, 0x2d, 0xde, 0x5f, 0x73, 0x5a, 0x25, 0x2d, 0x30, 0x0c, - 0x5c, 0x0b, 0xc3, 0xff, 0xc7, 0x02, 0x2b, 0x7a, 0x1e, 0xfa, 0x6a, 0x96, 0xef, 0x61, 0xff, 0xde, - 0x1e, 0xde, 0x17, 0xc2, 0x38, 0x25, 0xe5, 0xd5, 0x2a, 0xa6, 0x48, 0x91, 0x05, 0xc3, 0x96, 0x50, - 0x9d, 0x0a, 0xed, 0xd6, 0x53, 0x79, 0x09, 0x04, 0x2a, 0xd8, 0x40, 0xf1, 0xeb, 0x97, 0xe0, 0x80, - 0x06, 0xa5, 0x17, 0x7b, 0xdf, 0xcc, 0x4d, 0x2f, 0x78, 0x11, 0x92, 0x03, 0xb8, 0xc6, 0x5f, 0x85, - 0x08, 0x0c, 0x7a, 0x9a, 0x61, 0x79, 0xbe, 0xb7, 0xfc, 0xe3, 0x79, 0xa9, 0xad, 0x53, 0x2c, 0xa1, - 0x86, 0x94, 0xfd, 0x74, 0xb1, 0x40, 0x8e, 0x36, 0x61, 0x90, 0x7b, 0xcc, 0x8b, 0x4f, 0x28, 0xf7, - 0xfb, 0x1a, 0x77, 0xc2, 0xe7, 0xdb, 0x80, 0xff, 0x8f, 0x05, 0x66, 0xf4, 0x12, 0x0c, 0xbb, 0xbe, - 0x39, 0xda, 0x70, 0x6f, 0x53, 0x17, 0xd8, 0xa2, 0x09, 0xff, 0x5f, 0x61, 0x84, 0x16, 0xe0, 0x47, - 0x9b, 0x30, 0x64, 0x88, 0x2b, 0xd5, 0x48, 0xfe, 0x6d, 0xe7, 0x5f, 0x97, 0xd8, 0x6d, 0xc3, 0xbf, - 0x29, 0xf9, 0x88, 0xb3, 0x6c, 0x4e, 0xe0, 0x1b, 0x68, 0x73, 0x82, 0x5e, 0x06, 0x20, 0xbe, 0x0a, - 0xdd, 0x9d, 0x19, 0xcd, 0xbf, 0x65, 0x24, 0x45, 0xbc, 0x2f, 0x10, 0x05, 0x45, 0x2e, 0x96, 0x88, - 0xa0, 0x57, 0x64, 0x9d, 0xdd, 0x58, 0x6f, 0x41, 0x40, 0x92, 0x61, 0x6d, 0xc2, 0xa7, 0xbb, 0x40, - 0x5d, 0x24, 0xa9, 0xd2, 0xd4, 0x2f, 0x03, 0x40, 0xf8, 0xac, 0x8b, 0xb6, 0x60, 0xd8, 0xa7, 0xd2, - 0x4b, 0x18, 0x9e, 0xeb, 0x02, 0xcc, 0xb7, 0x97, 0xff, 0x0b, 0x07, 0xb8, 0x51, 0x29, 0x2d, 0x1c, - 0x54, 0x98, 0x21, 0xb7, 0xbb, 0x50, 0x50, 0x2f, 0x03, 0xe8, 0x61, 0x90, 0xd7, 0xbe, 0xfc, 0x6b, - 0x15, 0x04, 0x80, 0x0d, 0xd7, 0x4a, 0x8a, 0x11, 0x2b, 0x11, 0xc9, 0xb0, 0x4a, 0xef, 0xcf, 0x65, - 0x95, 0xfe, 0x38, 0x9c, 0x12, 0x56, 0x80, 0x15, 0xf6, 0xca, 0xe8, 0xed, 0x09, 0xc7, 0x4d, 0x66, - 0x1f, 0x5a, 0x8a, 0x82, 0x70, 0xbc, 0x2e, 0xfa, 0x97, 0x0a, 0x0c, 0xeb, 0x42, 0x40, 0x13, 0xbc, - 0x6d, 0xb9, 0xb7, 0x47, 0xfd, 0x39, 0x5f, 0xde, 0xe3, 0x82, 0xfc, 0x33, 0x3e, 0x57, 0xf5, 0x8b, - 0x8f, 0x48, 0x8c, 0x0f, 0x7a, 0x8d, 0x7e, 0x5f, 0x81, 0x51, 0xcd, 0x34, 0x6d, 0x5d, 0xf3, 0x58, - 0x78, 0x40, 0xee, 0x51, 0x7a, 0xab, 0xc7, 0x51, 0xcc, 0x87, 0x18, 0xf9, 0x40, 0xde, 0x1f, 0x68, - 0x02, 0x42, 0xc8, 0x11, 0x8d, 0x45, 0xee, 0x3e, 0xfa, 0xff, 0x14, 0x78, 0x0b, 0x77, 0xe3, 0x2d, - 0x51, 0x99, 0x6b, 0xcb, 0xd0, 0x35, 0x8f, 0xf0, 0x98, 0xa5, 0xbe, 0x0b, 0x19, 0xb7, 0xa1, 0x1f, - 0x3e, 0xb4, 0x0d, 0xfd, 0xd5, 0xfd, 0x76, 0xf1, 0x2d, 0xa5, 0x2e, 0x70, 0xe3, 0xae, 0x7a, 0x80, - 0xee, 0xc2, 0xb8, 0x29, 0xc7, 0x1f, 0x17, 0x4c, 0x3e, 0xd7, 0xf3, 0x6b, 0x24, 0x90, 0x39, 0x57, - 0x30, 0x44, 0x8a, 0x70, 0x94, 0xd4, 0xec, 0x6d, 0x18, 0x8f, 0x6c, 0xb4, 0x63, 0x55, 0x5c, 0x5b, - 0x30, 0x19, 0xdf, 0x0f, 0xc7, 0x7a, 0xcf, 0xbb, 0x09, 0x23, 0x81, 0xb0, 0x80, 0x2e, 0x49, 0x84, - 0x42, 0xd1, 0xeb, 0x26, 0xd9, 0xe3, 0x54, 0x8b, 0x11, 0x1d, 0x0a, 0x57, 0x12, 0xb2, 0xe8, 0x8d, - 0x02, 0xa1, 0xfa, 0x87, 0xe2, 0x55, 0x75, 0x9d, 0x34, 0x9a, 0xa6, 0xe6, 0x91, 0x37, 0xbe, 0x1d, - 0x92, 0xfa, 0x1f, 0x14, 0x7e, 0xde, 0x70, 0xd1, 0x06, 0x69, 0x30, 0xda, 0xe0, 0xc9, 0xef, 0x58, - 0xac, 0x47, 0x25, 0x7f, 0x94, 0xc9, 0x95, 0x10, 0x0d, 0x96, 0x71, 0xa2, 0x3b, 0x30, 0xe2, 0x0b, - 0x83, 0xbe, 0xa6, 0x66, 0xa9, 0x37, 0xe1, 0x2c, 0x90, 0x3b, 0x83, 0xa3, 0xd5, 0x2f, 0x71, 0x71, - 0x48, 0x4b, 0xd5, 0x00, 0x25, 0xdb, 0xa0, 0xfb, 0x43, 0x7f, 0x36, 0x25, 0x9a, 0xae, 0x26, 0xe1, - 0xd3, 0xe6, 0x2b, 0x59, 0x0b, 0x59, 0x4a, 0x56, 0xf5, 0xb7, 0x0a, 0x30, 0x2d, 0xae, 0x9e, 0xf3, - 0xba, 0x6e, 0xb7, 0x2c, 0x2f, 0x74, 0x33, 0xe6, 0xbe, 0xfb, 0x82, 0x08, 0x13, 0x27, 0xb9, 0x63, - 0x3f, 0x16, 0x10, 0x74, 0x8b, 0x6b, 0x3f, 0xad, 0xda, 0xba, 0x7d, 0x9b, 0x58, 0x21, 0x97, 0x90, - 0x83, 0x64, 0x2c, 0xa6, 0x55, 0xc0, 0xe9, 0xed, 0xd0, 0x0e, 0xa0, 0x86, 0xb6, 0x1b, 0xc7, 0x96, - 0x2f, 0x09, 0x1a, 0xbb, 0x2f, 0xae, 0x24, 0xb0, 0xe1, 0x14, 0x0a, 0xf4, 0x20, 0xa5, 0x92, 0x5c, - 0xd3, 0x23, 0x35, 0x3e, 0x44, 0xdf, 0xa8, 0x83, 0x1d, 0xa4, 0xf3, 0x51, 0x10, 0x8e, 0xd7, 0x55, - 0x3f, 0x3e, 0x08, 0xe7, 0xa3, 0x93, 0x48, 0xbf, 0x50, 0xdf, 0xbd, 0xfe, 0x49, 0xdf, 0x77, 0x8c, - 0x4f, 0xe4, 0xfd, 0x71, 0xdf, 0xb1, 0x19, 0xd9, 0x68, 0x48, 0x34, 0x8a, 0xf8, 0x91, 0x7d, 0x03, - 0x7c, 0xe5, 0x33, 0x9c, 0xa3, 0xfb, 0x8e, 0x35, 0x26, 0xc0, 0xab, 0x0a, 0xcc, 0x46, 0x8b, 0x97, - 0x0c, 0xcb, 0x70, 0xb7, 0x45, 0xde, 0x93, 0xc3, 0xbb, 0xae, 0xb1, 0xcc, 0xc9, 0xcb, 0x99, 0x18, - 0x71, 0x07, 0x6a, 0xe8, 0x07, 0x14, 0xb8, 0x10, 0x9b, 0x97, 0x48, 0x16, 0x96, 0xc3, 0x7b, 0xb1, - 0xb1, 0xe0, 0x2e, 0xcb, 0xd9, 0x28, 0x71, 0x27, 0x7a, 0x9d, 0x22, 0x7e, 0x0e, 0xbe, 0x5e, 0x22, - 0x7e, 0xfe, 0x52, 0x01, 0x06, 0x98, 0xdd, 0xd4, 0x1b, 0xc3, 0xe1, 0x88, 0x75, 0x35, 0xd3, 0xde, - 0xb5, 0x1e, 0xb3, 0x77, 0x7d, 0x32, 0x3f, 0x89, 0xce, 0x06, 0xaf, 0xef, 0x87, 0xb3, 0xac, 0xda, - 0x7c, 0x8d, 0x29, 0xda, 0x5c, 0x52, 0x9b, 0xaf, 0xd5, 0xd8, 0xf5, 0xf6, 0xe0, 0x37, 0xb2, 0x4b, - 0xd0, 0xd7, 0x72, 0xcc, 0x78, 0x08, 0xd9, 0x0d, 0xbc, 0x8c, 0x69, 0xb9, 0xfa, 0xaa, 0x02, 0x93, - 0x0c, 0xb7, 0xc4, 0x62, 0xd0, 0x0e, 0x0c, 0x3b, 0x82, 0xcd, 0x88, 0xb5, 0x59, 0xce, 0x3d, 0xb4, - 0x14, 0xd6, 0xc5, 0x6f, 0x6c, 0xfe, 0x2f, 0x1c, 0xd0, 0x52, 0xff, 0x6c, 0x00, 0x66, 0xb2, 0x1a, - 0xa1, 0x1f, 0x51, 0xe0, 0xac, 0x1e, 0x4a, 0x9c, 0xf3, 0x2d, 0x6f, 0xdb, 0x76, 0xfc, 0x8c, 0x0a, - 0xb9, 0xd5, 0x21, 0xa5, 0xf9, 0xa0, 0x57, 0x2c, 0x8d, 0x48, 0x29, 0x95, 0x02, 0xce, 0xa0, 0x8c, - 0x5e, 0x01, 0x70, 0xdd, 0xed, 0x9b, 0x64, 0xaf, 0xa9, 0x19, 0xbe, 0xe5, 0xd6, 0xcd, 0xfc, 0xdb, - 0xa0, 0x7a, 0x43, 0xa0, 0x0a, 0x3a, 0xc5, 0x75, 0xd5, 0x61, 0xb9, 0x44, 0x0e, 0x7d, 0x4c, 0x81, - 0x71, 0x5b, 0x8e, 0xd7, 0xd2, 0x8b, 0x31, 0x7f, 0x6a, 0xe0, 0x17, 0x2e, 0x69, 0x47, 0x41, 0x51, - 0x92, 0x74, 0x59, 0xa6, 0xdc, 0xf8, 0xc9, 0x26, 0x78, 0xdf, 0x4a, 0x3e, 0x19, 0x28, 0xe3, 0x98, - 0xe4, 0xb7, 0xf6, 0x24, 0x38, 0x49, 0x9e, 0x75, 0x8a, 0x78, 0x7a, 0x6d, 0xd1, 0xd2, 0x9d, 0x3d, - 0x16, 0x21, 0x80, 0x76, 0x6a, 0x30, 0x7f, 0xa7, 0x16, 0xd7, 0x4b, 0xe5, 0x08, 0xb2, 0x68, 0xa7, - 0x92, 0xe0, 0x24, 0x79, 0xf5, 0xa3, 0x05, 0x38, 0xc7, 0x96, 0xf9, 0x66, 0x6b, 0x93, 0x70, 0xe3, - 0xab, 0x7f, 0x74, 0x01, 0x76, 0x7e, 0x57, 0x81, 0x11, 0x36, 0x07, 0x6f, 0x10, 0x1f, 0x4d, 0xd6, - 0xd7, 0x0c, 0x03, 0xe7, 0xdf, 0x51, 0x60, 0x2a, 0x91, 0x43, 0xaa, 0x2b, 0x0f, 0xbf, 0x13, 0xb3, - 0xbd, 0x7d, 0x6b, 0x98, 0xb7, 0xb2, 0x2f, 0x0c, 0x6c, 0x11, 0xcf, 0x59, 0xa9, 0x3e, 0x0b, 0xe3, - 0x11, 0xfb, 0x66, 0x29, 0x04, 0x64, 0x5a, 0xf0, 0x4a, 0x39, 0xc2, 0x63, 0xa1, 0x53, 0x6c, 0xca, - 0x70, 0xcb, 0x27, 0x39, 0xdb, 0x3f, 0x9e, 0x2d, 0x8f, 0xc4, 0x96, 0xf7, 0x9f, 0x71, 0x58, 0x60, - 0x49, 0xb7, 0x97, 0x67, 0x1c, 0x16, 0xa1, 0x52, 0x04, 0x82, 0xe5, 0xff, 0x63, 0x81, 0x15, 0x3d, - 0x15, 0x0d, 0xf3, 0xba, 0x1a, 0xde, 0xed, 0xa6, 0xe3, 0xc1, 0x59, 0xd9, 0x96, 0x4c, 0xd4, 0x46, - 0x98, 0x3f, 0x04, 0xf1, 0xb3, 0x2c, 0x57, 0x76, 0x9b, 0xf2, 0x6a, 0x95, 0xc7, 0xff, 0x0b, 0x1e, - 0x80, 0xa2, 0x1a, 0xef, 0xfe, 0x93, 0xd0, 0x78, 0x3b, 0x30, 0xba, 0x6d, 0x6c, 0x12, 0xc7, 0xe2, - 0x62, 0xcc, 0x40, 0x7e, 0x09, 0xed, 0x46, 0x88, 0x86, 0xab, 0x01, 0xa4, 0x02, 0x2c, 0x13, 0x41, - 0x4e, 0x24, 0x58, 0xf5, 0x60, 0x7e, 0xa9, 0x24, 0x54, 0x4d, 0x87, 0xe3, 0xcc, 0x08, 0x54, 0x6d, - 0x01, 0x58, 0x41, 0x04, 0xd7, 0x5e, 0x1e, 0x86, 0xc2, 0x38, 0xb0, 0x5c, 0xe8, 0x08, 0x7f, 0x63, - 0x89, 0x02, 0x9d, 0xd7, 0x46, 0x98, 0xd0, 0x40, 0xa8, 0x19, 0x9f, 0xec, 0x31, 0xa9, 0x84, 0x50, - 0xaf, 0x84, 0x05, 0x58, 0x26, 0x42, 0xc7, 0xd8, 0x08, 0xc2, 0xf8, 0x0b, 0x35, 0x62, 0xae, 0x31, - 0x86, 0xc9, 0x00, 0xf8, 0x18, 0xc3, 0xdf, 0x58, 0xa2, 0x80, 0x5e, 0x92, 0xde, 0x0f, 0x21, 0xbf, - 0x92, 0xaa, 0xab, 0xb7, 0xc3, 0x77, 0x85, 0xba, 0x9a, 0x51, 0xf6, 0x9d, 0x5e, 0x90, 0xf4, 0x34, - 0x2c, 0xbd, 0x01, 0xe5, 0x1d, 0x09, 0xbd, 0x4d, 0xe8, 0x55, 0x31, 0xd6, 0xd1, 0xab, 0xa2, 0x44, - 0xa5, 0x33, 0xc9, 0x33, 0x91, 0x31, 0x84, 0xf1, 0xf0, 0x11, 0xa4, 0x1a, 0x07, 0xe2, 0x64, 0x7d, - 0xce, 0xf0, 0x49, 0x8d, 0xb5, 0x9d, 0x90, 0x19, 0x3e, 0x2f, 0xc3, 0x01, 0x14, 0xed, 0xc0, 0x98, - 0x2b, 0xb9, 0x68, 0xcc, 0x9c, 0xea, 0xf5, 0x09, 0x51, 0xb8, 0x67, 0x30, 0x5b, 0x48, 0xb9, 0x04, - 0x47, 0xe8, 0x44, 0xdf, 0xb7, 0x26, 0x4f, 0xf6, 0x7d, 0x0b, 0xb5, 0xa2, 0xd6, 0xd7, 0x53, 0x47, - 0x12, 0xc7, 0xe5, 0x40, 0xeb, 0x6c, 0xba, 0xb4, 0x64, 0xb7, 0x69, 0xbb, 0x2d, 0x87, 0x30, 0xb3, - 0x4f, 0xb6, 0x3c, 0x28, 0x5c, 0xda, 0xc5, 0x38, 0x10, 0x27, 0xeb, 0xa3, 0xef, 0x53, 0x60, 0xd2, - 0xdd, 0x73, 0x3d, 0xd2, 0xa0, 0xc7, 0x96, 0x6d, 0x11, 0xcb, 0x73, 0x67, 0x4e, 0xe7, 0x8f, 0x0b, - 0x5e, 0x8d, 0xe1, 0xe2, 0xc7, 0x4e, 0xbc, 0x14, 0x27, 0x68, 0xd2, 0x9d, 0x23, 0x47, 0x82, 0x99, - 0x99, 0xce, 0xbf, 0x73, 0xe4, 0x28, 0x33, 0x7c, 0xe7, 0xc8, 0x25, 0x38, 0x42, 0x07, 0x3d, 0x02, - 0xe3, 0xae, 0x9f, 0xc3, 0x9c, 0xcd, 0xe0, 0x99, 0x30, 0x58, 0x67, 0x55, 0x06, 0xe0, 0x68, 0x3d, - 0xf4, 0x61, 0x18, 0x93, 0xcf, 0xce, 0x99, 0xb3, 0x47, 0x1d, 0xb6, 0x9e, 0xf7, 0x5c, 0x06, 0x45, - 0x08, 0x22, 0x0c, 0x67, 0x25, 0xe7, 0x41, 0xf9, 0xfb, 0x3e, 0xc7, 0x86, 0xc0, 0xef, 0xb3, 0xa9, - 0x35, 0x70, 0x46, 0x4b, 0xf4, 0x13, 0xe9, 0xcf, 0xe5, 0x33, 0x6c, 0x4b, 0xaf, 0x1d, 0xc9, 0x73, - 0xf9, 0xb3, 0x86, 0xb7, 0x7d, 0xab, 0xc9, 0x23, 0x8b, 0x1d, 0xf2, 0xe5, 0x5c, 0xfd, 0x63, 0x05, - 0x20, 0x50, 0x98, 0x9c, 0xc4, 0x53, 0x45, 0x2d, 0xa2, 0x43, 0x5a, 0xe8, 0x49, 0xc1, 0x93, 0x99, - 0x95, 0x44, 0xfd, 0x8a, 0x02, 0x13, 0x61, 0xb5, 0x13, 0xb8, 0x1a, 0xe9, 0xd1, 0xab, 0xd1, 0x13, - 0xbd, 0x8d, 0x2b, 0xe3, 0x7e, 0xf4, 0x3f, 0x0b, 0xf2, 0xa8, 0x98, 0xf4, 0xbb, 0x13, 0x79, 0xfa, - 0xa7, 0xa4, 0x6f, 0xf4, 0xf2, 0xf4, 0x2f, 0x47, 0xdf, 0x08, 0xc7, 0x9b, 0x62, 0x0a, 0xf0, 0xdd, - 0x11, 0xf9, 0xb3, 0x87, 0xb8, 0x37, 0x81, 0xb0, 0xe9, 0x93, 0xe6, 0x13, 0x70, 0x90, 0x30, 0xfa, - 0xb2, 0x7c, 0x3c, 0xf5, 0x90, 0x49, 0x24, 0x32, 0xe0, 0xce, 0x46, 0x17, 0xbf, 0x34, 0x05, 0xa3, - 0x92, 0x6e, 0x31, 0x66, 0xc8, 0xa0, 0x9c, 0x84, 0x21, 0x83, 0x07, 0xa3, 0x7a, 0x90, 0x4a, 0xd2, - 0x9f, 0xf6, 0x1e, 0x69, 0x06, 0xc7, 0x62, 0x98, 0xa4, 0xd2, 0xc5, 0x32, 0x19, 0x2a, 0xbc, 0x05, - 0x7b, 0xac, 0xef, 0x08, 0xcc, 0x4b, 0x3a, 0xed, 0xab, 0x87, 0x01, 0x7c, 0xf9, 0x9f, 0xd4, 0x44, - 0x84, 0xf6, 0xc0, 0x74, 0xba, 0xe2, 0xde, 0x08, 0x60, 0x58, 0xaa, 0x97, 0x7c, 0x18, 0x1f, 0x38, - 0xb1, 0x87, 0x71, 0xba, 0x0d, 0x4c, 0x3f, 0xe9, 0x7e, 0x4f, 0xe6, 0x6a, 0x41, 0xea, 0xfe, 0x70, - 0x1b, 0x04, 0x45, 0x2e, 0x96, 0x88, 0x64, 0xd8, 0xb3, 0x0c, 0xe5, 0xb2, 0x67, 0x69, 0xc1, 0x69, - 0x87, 0x78, 0xce, 0x5e, 0x69, 0x4f, 0x67, 0xa9, 0x4d, 0x1c, 0x8f, 0xdd, 0xe0, 0x87, 0xf3, 0x05, - 0x4c, 0xc4, 0x49, 0x54, 0x38, 0x0d, 0x7f, 0x44, 0x00, 0x1e, 0xe9, 0x28, 0x00, 0xbf, 0x0b, 0x46, - 0x3d, 0xa2, 0x6f, 0x5b, 0x86, 0xae, 0x99, 0x95, 0xb2, 0x88, 0xdf, 0x1d, 0xca, 0x72, 0x21, 0x08, - 0xcb, 0xf5, 0xd0, 0x02, 0xf4, 0xb5, 0x8c, 0x9a, 0xb8, 0x01, 0x7c, 0x5b, 0xa0, 0xa5, 0xaf, 0x94, - 0xef, 0xb5, 0x8b, 0x6f, 0x0e, 0x0d, 0x44, 0x82, 0x51, 0x5d, 0x6b, 0xde, 0xae, 0x5f, 0xf3, 0xf6, - 0x9a, 0xc4, 0x9d, 0xdb, 0xa8, 0x94, 0x31, 0x6d, 0x9c, 0x66, 0xeb, 0x33, 0x76, 0x08, 0x5b, 0x9f, - 0xcf, 0x28, 0x70, 0x5a, 0x8b, 0x3f, 0x30, 0x10, 0x77, 0x66, 0x3c, 0x3f, 0xb7, 0x4c, 0x7f, 0xb4, - 0x58, 0xb8, 0x20, 0xc6, 0x77, 0x7a, 0x3e, 0x49, 0x0e, 0xa7, 0xf5, 0x01, 0x39, 0x80, 0x1a, 0x46, - 0x3d, 0xc8, 0x7f, 0x2f, 0x56, 0x7d, 0x22, 0x9f, 0xde, 0x66, 0x25, 0x81, 0x09, 0xa7, 0x60, 0x47, - 0x77, 0x60, 0x54, 0x12, 0x92, 0xc4, 0x4d, 0xa6, 0x7c, 0x14, 0xef, 0x20, 0xfc, 0xb6, 0x2b, 0xbf, - 0x71, 0xc8, 0x94, 0x82, 0x47, 0x4e, 0x49, 0xcd, 0x20, 0x1e, 0xfa, 0xd8, 0xa8, 0x27, 0xf3, 0x3f, - 0x72, 0xa6, 0x63, 0xc4, 0x1d, 0xa8, 0xb1, 0x30, 0x85, 0x14, 0x2c, 0xdd, 0xcd, 0x67, 0xa6, 0xf2, - 0xc7, 0x59, 0x59, 0x8e, 0xa2, 0xe2, 0x5b, 0x33, 0x56, 0x88, 0xe3, 0x04, 0xd1, 0x12, 0x20, 0xc2, - 0x55, 0xe9, 0xe1, 0xe5, 0xcc, 0x9d, 0x41, 0xec, 0xfd, 0x9d, 0x2d, 0xe9, 0x62, 0x02, 0x8a, 0x53, - 0x5a, 0x20, 0x2f, 0xa2, 0x2b, 0xe9, 0xe1, 0x96, 0x13, 0xcf, 0x99, 0xd3, 0x51, 0x63, 0xf2, 0xbd, - 0x4a, 0x22, 0x89, 0x32, 0xbf, 0xdc, 0xdc, 0xe8, 0x3d, 0x89, 0xb2, 0x20, 0xdf, 0x4d, 0x2a, 0xe5, - 0xcf, 0x2b, 0x70, 0xae, 0x91, 0x9e, 0x2c, 0x92, 0xdd, 0x79, 0x72, 0x3e, 0x5c, 0x65, 0xe4, 0x9f, - 0x64, 0x6a, 0x8c, 0xac, 0xe4, 0x94, 0x38, 0xab, 0x23, 0xea, 0x1f, 0x29, 0x42, 0x15, 0x7d, 0x82, - 0xe6, 0x48, 0xc7, 0xfd, 0x4e, 0xac, 0x3e, 0x0b, 0x33, 0x55, 0x3f, 0xc8, 0x68, 0x2d, 0x96, 0x4b, - 0xe0, 0xbd, 0x30, 0xce, 0x9f, 0x82, 0x56, 0xb4, 0xe6, 0x6a, 0xf8, 0x6e, 0x10, 0x84, 0xa3, 0x28, - 0xc9, 0x40, 0x1c, 0xad, 0xab, 0x7e, 0x4d, 0x81, 0x73, 0x51, 0xcc, 0xb6, 0x63, 0xdc, 0xed, 0x1d, - 0x31, 0xfa, 0x84, 0x02, 0xa3, 0xb7, 0x83, 0x57, 0x2a, 0x5f, 0x74, 0xcb, 0xe5, 0xb2, 0xe1, 0xf7, - 0x8a, 0x38, 0xd2, 0xb3, 0x57, 0x32, 0x41, 0x65, 0x08, 0x74, 0xb1, 0x4c, 0x5a, 0xfd, 0x2f, 0x0a, - 0x24, 0xd4, 0x07, 0x68, 0x13, 0x86, 0x28, 0x91, 0xf2, 0x6a, 0x55, 0xec, 0x89, 0xf7, 0xe6, 0x93, - 0x2a, 0x19, 0x0a, 0xfe, 0x28, 0x22, 0x7e, 0x60, 0x1f, 0x31, 0xda, 0xe1, 0xfe, 0xff, 0x7e, 0xa2, - 0x21, 0xb1, 0x3d, 0x72, 0x89, 0xed, 0x72, 0xc2, 0x22, 0x7e, 0xad, 0x97, 0x4b, 0x70, 0x84, 0x8e, - 0xba, 0x0c, 0x10, 0xaa, 0x7c, 0x7a, 0x36, 0xef, 0xfb, 0xb7, 0xa7, 0xe1, 0x4c, 0xcf, 0xce, 0x88, - 0x1f, 0x53, 0xe0, 0x2c, 0xd9, 0x31, 0x74, 0x6f, 0x7e, 0xcb, 0x23, 0xce, 0xad, 0x5b, 0x2b, 0xeb, - 0xdb, 0x0e, 0x71, 0xb7, 0x6d, 0xb3, 0xd6, 0x8d, 0x31, 0x63, 0x8a, 0xe5, 0x15, 0x53, 0x4d, 0x2c, - 0xa6, 0x62, 0xc4, 0x19, 0x94, 0x98, 0xba, 0x6b, 0x87, 0x2b, 0x02, 0x30, 0xbd, 0x73, 0xb5, 0x1c, - 0xd7, 0x13, 0x71, 0x03, 0xb9, 0xba, 0x2b, 0x0e, 0xc4, 0xc9, 0xfa, 0x71, 0x24, 0x2c, 0x8b, 0x1e, - 0x93, 0xdb, 0x95, 0x24, 0x12, 0x06, 0xc4, 0xc9, 0xfa, 0x32, 0x12, 0xbe, 0x52, 0xf4, 0x50, 0x1c, - 0x48, 0x22, 0x09, 0x80, 0x38, 0x59, 0x1f, 0xd5, 0xe0, 0xa2, 0x43, 0x74, 0xbb, 0xd1, 0x20, 0x56, - 0x8d, 0x4d, 0xca, 0x8a, 0xe6, 0xd4, 0x0d, 0x6b, 0xc9, 0xd1, 0x58, 0x45, 0xf6, 0x7a, 0xa0, 0xb0, - 0x84, 0xb7, 0x17, 0x71, 0x87, 0x7a, 0xb8, 0x23, 0x16, 0xd4, 0x80, 0x53, 0x2d, 0xc6, 0xff, 0x1d, - 0x16, 0x57, 0x6c, 0x47, 0x33, 0xc5, 0x13, 0xc1, 0x61, 0x57, 0x8c, 0x1d, 0xd4, 0x1b, 0x51, 0x54, - 0x38, 0x8e, 0x1b, 0xed, 0x51, 0xf1, 0x5c, 0x74, 0x47, 0x22, 0x39, 0x9c, 0x8b, 0xa4, 0x10, 0xd1, - 0x13, 0xe8, 0x70, 0x1a, 0x0d, 0x54, 0x81, 0xd3, 0x9e, 0xe6, 0xd4, 0x89, 0x57, 0x5a, 0xdb, 0x58, - 0x23, 0x8e, 0x4e, 0x79, 0xac, 0xc9, 0xa5, 0x75, 0x85, 0xa3, 0x5a, 0x4f, 0x82, 0x71, 0x5a, 0x1b, - 0xf4, 0x61, 0x78, 0x6b, 0x74, 0x52, 0x97, 0xed, 0x3b, 0xc4, 0x59, 0xb0, 0x5b, 0x56, 0x2d, 0x8a, - 0x1c, 0x18, 0xf2, 0xfb, 0xf7, 0xdb, 0xc5, 0xb7, 0xe2, 0x6e, 0x1a, 0xe0, 0xee, 0xf0, 0x26, 0x3b, - 0xb0, 0xd1, 0x6c, 0xa6, 0x76, 0x60, 0x34, 0xab, 0x03, 0x19, 0x0d, 0x70, 0x77, 0x78, 0x11, 0x86, - 0xb3, 0x7c, 0x62, 0x78, 0x7a, 0x66, 0x89, 0xe2, 0x18, 0xa3, 0xc8, 0xbe, 0xdf, 0xf5, 0xd4, 0x1a, - 0x38, 0xa3, 0x25, 0x3d, 0x53, 0xae, 0x66, 0x0d, 0x3f, 0x41, 0x66, 0x9c, 0x91, 0x79, 0xfb, 0x7e, - 0xbb, 0x78, 0x15, 0x77, 0xd9, 0x06, 0x77, 0x8d, 0x3d, 0xa5, 0x2b, 0xe1, 0x44, 0x24, 0xba, 0x32, - 0x91, 0xd5, 0x95, 0xec, 0x36, 0xb8, 0x6b, 0xec, 0xe8, 0xfb, 0x15, 0x38, 0xaf, 0x37, 0x5b, 0x37, - 0x0c, 0xd7, 0xb3, 0xeb, 0x8e, 0xd6, 0x28, 0x13, 0x5d, 0xdb, 0xbb, 0xa1, 0x99, 0x5b, 0xcb, 0xc6, - 0x16, 0x11, 0x97, 0x8e, 0xc3, 0x7e, 0x38, 0xcc, 0xf1, 0xb6, 0xb4, 0xb6, 0x91, 0x8e, 0x14, 0x67, - 0xd3, 0x43, 0x3f, 0xaa, 0xc0, 0xc5, 0x06, 0xeb, 0x62, 0x46, 0x87, 0x26, 0x73, 0x75, 0x88, 0x71, - 0xb1, 0x95, 0x0e, 0x78, 0x71, 0x47, 0xaa, 0x6c, 0x92, 0x78, 0x85, 0xf9, 0x7a, 0xdd, 0x21, 0x75, - 0x86, 0x35, 0xe0, 0x2e, 0x53, 0xf9, 0x27, 0x69, 0x25, 0x0b, 0x29, 0xce, 0xa6, 0x87, 0x5e, 0x82, - 0xcb, 0x99, 0xc0, 0x92, 0xdd, 0xb2, 0x3c, 0xf6, 0x08, 0xd3, 0xb7, 0xa0, 0xee, 0xb7, 0x8b, 0x97, - 0x57, 0x3a, 0xd6, 0xc4, 0x07, 0x60, 0x62, 0xb1, 0x47, 0x22, 0x61, 0x00, 0x4e, 0x33, 0x49, 0xec, - 0x85, 0x5c, 0xe9, 0x78, 0x8f, 0xd0, 0xf7, 0xff, 0x2b, 0x51, 0xdf, 0xff, 0x69, 0xd6, 0xab, 0xf7, - 0x1f, 0x5d, 0xaf, 0xbe, 0xe9, 0xf0, 0xdf, 0x8d, 0x23, 0xc8, 0x67, 0x14, 0x10, 0xfe, 0x9b, 0xe8, - 0x62, 0xc4, 0xec, 0x68, 0x38, 0x66, 0x72, 0xe4, 0x67, 0xc5, 0x2d, 0xa4, 0x66, 0xc5, 0x7d, 0x9b, - 0x14, 0x84, 0x79, 0x24, 0xbc, 0x2c, 0x71, 0xcc, 0x61, 0x14, 0x66, 0xf4, 0x00, 0x8c, 0x04, 0x97, - 0x6a, 0xa1, 0xec, 0x64, 0x19, 0x69, 0xc2, 0xdb, 0x77, 0x08, 0x57, 0xff, 0x40, 0x01, 0x08, 0x93, - 0x3d, 0xa3, 0xfb, 0xfc, 0x60, 0x35, 0xbc, 0x83, 0xc1, 0x8b, 0x81, 0x1c, 0xb0, 0xe6, 0x60, 0x67, - 0x04, 0xa4, 0xc2, 0x60, 0x8b, 0xa5, 0xb7, 0x14, 0x0e, 0x04, 0xcc, 0x04, 0x66, 0x83, 0x95, 0x60, - 0x01, 0x41, 0x1b, 0x30, 0xd4, 0x30, 0x2c, 0xe6, 0xeb, 0xd1, 0x9f, 0xcb, 0xd7, 0x83, 0xdd, 0x07, - 0x56, 0x38, 0x0a, 0xec, 0xe3, 0x52, 0x7f, 0x59, 0x81, 0x53, 0xd1, 0xa8, 0xd8, 0x2c, 0xe4, 0x8d, - 0x08, 0x96, 0x2a, 0x82, 0xf1, 0xb3, 0xa6, 0x22, 0x08, 0x24, 0xf6, 0x61, 0xd1, 0xd7, 0xe9, 0x1e, - 0x5e, 0x1f, 0xd2, 0x83, 0x73, 0x1f, 0xf0, 0x10, 0xf0, 0xe3, 0xa7, 0x61, 0x90, 0x5f, 0xb4, 0xa9, - 0x1c, 0x9f, 0x12, 0xfc, 0xea, 0x66, 0xfe, 0x7c, 0x13, 0x79, 0x22, 0xe2, 0xc8, 0x59, 0x37, 0x0b, - 0x1d, 0xb3, 0x6e, 0x62, 0xe8, 0xd3, 0x1d, 0xa3, 0x17, 0x4b, 0xa4, 0x12, 0xae, 0x70, 0x4b, 0xa4, - 0x12, 0xae, 0x60, 0x8a, 0x0c, 0x79, 0x11, 0x13, 0x9d, 0xfe, 0xfc, 0x2a, 0x20, 0x3e, 0x01, 0x92, - 0xa1, 0xce, 0x44, 0x47, 0x23, 0x1d, 0x3f, 0xd2, 0xfe, 0x40, 0x7e, 0xe7, 0x20, 0x31, 0xe5, 0xdd, - 0x44, 0xda, 0xf7, 0x3f, 0xa4, 0xc1, 0xcc, 0x0f, 0x69, 0x0b, 0x86, 0xc4, 0xa7, 0x20, 0x2e, 0x04, - 0xef, 0xed, 0x21, 0xb7, 0xbc, 0x94, 0xc5, 0x8a, 0x17, 0x60, 0x1f, 0x39, 0xbd, 0x65, 0x36, 0xb4, - 0x5d, 0xa3, 0xd1, 0x6a, 0xb0, 0x5b, 0xc0, 0x80, 0x5c, 0x95, 0x15, 0x63, 0x1f, 0xce, 0xaa, 0x72, - 0x9f, 0x2a, 0x26, 0xb5, 0xcb, 0x55, 0x79, 0x31, 0xf6, 0xe1, 0xe8, 0x79, 0x18, 0x6e, 0x68, 0xbb, - 0xd5, 0x96, 0x53, 0x27, 0xc2, 0x40, 0x27, 0x5b, 0x29, 0xd4, 0xf2, 0x0c, 0x73, 0xce, 0xb0, 0x3c, - 0xd7, 0x73, 0xe6, 0x2a, 0x96, 0x77, 0xcb, 0xa9, 0x7a, 0xcc, 0x00, 0x88, 0xed, 0xba, 0x15, 0x81, - 0x05, 0x07, 0xf8, 0x90, 0x09, 0x13, 0x0d, 0x6d, 0x77, 0xc3, 0xd2, 0x78, 0x12, 0x05, 0x21, 0x65, - 0xe7, 0xa1, 0xc0, 0xb4, 0x72, 0x2b, 0x11, 0x5c, 0x38, 0x86, 0x3b, 0xc5, 0x18, 0x74, 0xec, 0xb8, - 0x8c, 0x41, 0xe7, 0x83, 0x28, 0x05, 0x5c, 0xa5, 0x7f, 0x3e, 0x35, 0x3e, 0x60, 0xc7, 0x08, 0x04, - 0x2f, 0x06, 0x11, 0x08, 0x26, 0xf2, 0x5b, 0x2f, 0x76, 0x88, 0x3e, 0xd0, 0x82, 0xd1, 0x9a, 0xe6, - 0x69, 0xbc, 0xd4, 0x9d, 0x39, 0x95, 0xff, 0x75, 0xba, 0x1c, 0xa0, 0x09, 0x59, 0x52, 0x58, 0xe6, - 0x62, 0x99, 0x0e, 0xba, 0x05, 0x67, 0xe8, 0xc7, 0x6a, 0x12, 0x2f, 0xac, 0xc2, 0x74, 0x66, 0x93, - 0xec, 0xfb, 0x61, 0x5e, 0x6a, 0x37, 0xd3, 0x2a, 0xe0, 0xf4, 0x76, 0x61, 0x2c, 0xdb, 0xa9, 0xf4, - 0x58, 0xb6, 0xe8, 0x53, 0x69, 0x66, 0x37, 0x88, 0xcd, 0xe9, 0xfb, 0xf2, 0xf3, 0x86, 0xdc, 0xc6, - 0x37, 0xff, 0x42, 0x81, 0x19, 0xb1, 0xcb, 0x84, 0xa9, 0x8c, 0x49, 0x9c, 0x15, 0xcd, 0xd2, 0xea, - 0xc4, 0x11, 0x7a, 0xf2, 0xf5, 0x1e, 0xf8, 0x43, 0x02, 0x67, 0x10, 0x1a, 0xe2, 0x2d, 0xfb, 0xed, - 0xe2, 0x95, 0x83, 0x6a, 0xe1, 0xcc, 0xbe, 0x21, 0x07, 0x86, 0xdc, 0x3d, 0x57, 0xf7, 0x4c, 0x57, - 0xc8, 0xa0, 0xd7, 0x7b, 0xe0, 0xac, 0x55, 0x8e, 0x89, 0xb3, 0xd6, 0x30, 0x77, 0x22, 0x2f, 0xc5, - 0x3e, 0x21, 0xf4, 0xc3, 0x0a, 0x4c, 0x89, 0xc7, 0x33, 0x29, 0xf4, 0xce, 0x99, 0xfc, 0x7e, 0x32, - 0xa5, 0x38, 0x32, 0xdf, 0x3c, 0x86, 0x69, 0x93, 0x12, 0x50, 0x9c, 0xa4, 0x4e, 0x0f, 0xd5, 0xa6, - 0x63, 0xd8, 0x8e, 0xe1, 0xed, 0x31, 0x43, 0xa4, 0x01, 0x3f, 0x82, 0x3d, 0x2f, 0xc3, 0x01, 0x14, - 0x55, 0x61, 0x82, 0x6b, 0x6d, 0xaa, 0x9e, 0xa3, 0x79, 0xa4, 0xbe, 0x27, 0xac, 0x85, 0x1e, 0x60, - 0x39, 0x86, 0x23, 0x90, 0x7b, 0xed, 0xe2, 0x19, 0xb1, 0x36, 0x51, 0x00, 0x8e, 0xa1, 0x40, 0x1f, - 0x8c, 0x19, 0x6f, 0xcd, 0xe4, 0xcf, 0x3f, 0xc8, 0xd7, 0xe2, 0x30, 0x26, 0x5c, 0xbd, 0x46, 0x93, - 0xeb, 0x21, 0x07, 0xcc, 0xec, 0x63, 0x30, 0x26, 0xef, 0x9a, 0x43, 0x05, 0xb1, 0xf3, 0x00, 0x25, - 0x07, 0x7b, 0xdc, 0x71, 0x7e, 0xd4, 0x9f, 0x51, 0x60, 0x32, 0x2e, 0xbb, 0xa0, 0x6d, 0x18, 0x12, - 0x8c, 0x4c, 0x50, 0x9d, 0xcf, 0x6b, 0xb5, 0x6c, 0x12, 0xe1, 0x1e, 0xcc, 0x45, 0x61, 0x51, 0x84, - 0x7d, 0xf4, 0xb2, 0x47, 0x42, 0xa1, 0x83, 0x47, 0xc2, 0xe3, 0x70, 0x36, 0x9d, 0xa5, 0xd1, 0x8b, - 0x84, 0x46, 0xef, 0x57, 0x42, 0x69, 0x1d, 0x5c, 0x24, 0xd8, 0xa5, 0x0b, 0x73, 0x98, 0xfa, 0x21, - 0x88, 0xe7, 0x3e, 0x43, 0x2f, 0xc1, 0x88, 0xeb, 0x6e, 0x73, 0x3b, 0x33, 0x31, 0xc8, 0x7c, 0x4f, - 0x3d, 0x7e, 0x1e, 0x1a, 0x7e, 0xf7, 0x09, 0x7e, 0xe2, 0x10, 0xfd, 0xc2, 0x73, 0x5f, 0xfa, 0xda, - 0xe5, 0x37, 0xfd, 0xe1, 0xd7, 0x2e, 0xbf, 0xe9, 0xab, 0x5f, 0xbb, 0xfc, 0xa6, 0x8f, 0xec, 0x5f, - 0x56, 0xbe, 0xb4, 0x7f, 0x59, 0xf9, 0xc3, 0xfd, 0xcb, 0xca, 0x57, 0xf7, 0x2f, 0x2b, 0x7f, 0xb5, - 0x7f, 0x59, 0xf9, 0xa1, 0x7f, 0x77, 0xf9, 0x4d, 0xcf, 0x3f, 0x14, 0x52, 0xbf, 0xe6, 0x13, 0x0d, - 0xff, 0x69, 0xde, 0xae, 0xd3, 0x4b, 0xab, 0xeb, 0xdf, 0x5a, 0x19, 0xf5, 0xff, 0x1d, 0x00, 0x00, - 0xff, 0xff, 0x89, 0x6e, 0x7a, 0xd1, 0x2e, 0x27, 0x01, 0x00, + // 14824 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x6d, 0x70, 0x24, 0xc9, + 0x55, 0xa0, 0xab, 0xf5, 0xfd, 0xf4, 0x31, 0x52, 0x8e, 0x66, 0x46, 0xa3, 0xf9, 0xe8, 0x71, 0xad, + 0x6d, 0x66, 0x59, 0x5b, 0xc3, 0xae, 0xd7, 0x5e, 0xef, 0x9a, 0xfd, 0x90, 0xba, 0xa5, 0x99, 0xf6, + 0x48, 0x1a, 0x6d, 0xb6, 0xb4, 0xbb, 0xde, 0x85, 0x35, 0xa5, 0xea, 0x54, 0xab, 0x76, 0xaa, 0xab, + 0x7a, 0xab, 0xaa, 0x35, 0xd2, 0xac, 0x8d, 0x3f, 0xc0, 0x9c, 0x6d, 0x0c, 0xc7, 0xb7, 0xcf, 0x36, + 0x04, 0x8b, 0x09, 0xcc, 0xdd, 0x71, 0x67, 0x2e, 0x20, 0xb8, 0x08, 0x20, 0x20, 0x0e, 0x22, 0x38, + 0x0c, 0x81, 0x09, 0x02, 0xc3, 0x9d, 0x39, 0xa0, 0xc1, 0x3a, 0x1f, 0x5c, 0xc4, 0x7d, 0xc6, 0x11, + 0x71, 0x44, 0xcc, 0x11, 0xdc, 0x45, 0x7e, 0x54, 0x55, 0xd6, 0x57, 0xab, 0x55, 0x2d, 0xc9, 0xbb, + 0x87, 0x7f, 0x49, 0x9d, 0x2f, 0xf3, 0xbd, 0xfc, 0xaa, 0x97, 0xef, 0xbd, 0x7c, 0xf9, 0x1e, 0x2c, + 0xd4, 0x0d, 0x6f, 0xbb, 0xb5, 0x39, 0xa7, 0xdb, 0x8d, 0x6b, 0x75, 0xcd, 0xa9, 0x11, 0x8b, 0x38, + 0xe1, 0x3f, 0xcd, 0xdb, 0xf5, 0x6b, 0x5a, 0xd3, 0x70, 0xaf, 0xe9, 0xb6, 0x43, 0xae, 0xed, 0x3c, + 0xb8, 0x49, 0x3c, 0xed, 0xc1, 0x6b, 0x75, 0x0a, 0xd3, 0x3c, 0x52, 0x9b, 0x6b, 0x3a, 0xb6, 0x67, + 0xa3, 0x87, 0x42, 0x1c, 0x73, 0x7e, 0xd3, 0xf0, 0x9f, 0xe6, 0xed, 0xfa, 0x1c, 0xc5, 0x31, 0x47, + 0x71, 0xcc, 0x09, 0x1c, 0xb3, 0x6f, 0x93, 0xe9, 0xda, 0x75, 0xfb, 0x1a, 0x43, 0xb5, 0xd9, 0xda, + 0x62, 0xbf, 0xd8, 0x0f, 0xf6, 0x1f, 0x27, 0x31, 0x7b, 0xff, 0xed, 0x77, 0xb9, 0x73, 0x86, 0x4d, + 0x3b, 0x73, 0x4d, 0x6b, 0x79, 0xb6, 0xab, 0x6b, 0xa6, 0x61, 0xd5, 0xaf, 0xed, 0x24, 0x7a, 0x33, + 0xab, 0x4a, 0x55, 0x45, 0xb7, 0x3b, 0xd6, 0x71, 0x36, 0x35, 0x3d, 0xad, 0xce, 0x8d, 0xb0, 0x0e, + 0xd9, 0xf5, 0x88, 0xe5, 0x1a, 0xb6, 0xe5, 0xbe, 0x8d, 0x8e, 0x84, 0x38, 0x3b, 0xf2, 0xdc, 0x44, + 0x2a, 0xa4, 0x61, 0x7a, 0x38, 0xc4, 0xd4, 0xd0, 0xf4, 0x6d, 0xc3, 0x22, 0xce, 0x9e, 0xdf, 0xfc, + 0x9a, 0x43, 0x5c, 0xbb, 0xe5, 0xe8, 0xe4, 0x50, 0xad, 0xdc, 0x6b, 0x0d, 0xe2, 0x69, 0x69, 0xb4, + 0xae, 0x65, 0xb5, 0x72, 0x5a, 0x96, 0x67, 0x34, 0x92, 0x64, 0xde, 0x79, 0x50, 0x03, 0x57, 0xdf, + 0x26, 0x0d, 0x2d, 0xd1, 0xee, 0xed, 0x59, 0xed, 0x5a, 0x9e, 0x61, 0x5e, 0x33, 0x2c, 0xcf, 0xf5, + 0x9c, 0x78, 0x23, 0xf5, 0x13, 0x0a, 0x4c, 0xce, 0xaf, 0x55, 0xaa, 0x6c, 0x06, 0x97, 0xed, 0x7a, + 0xdd, 0xb0, 0xea, 0xe8, 0x01, 0x18, 0xd9, 0x21, 0xce, 0xa6, 0xed, 0x1a, 0xde, 0xde, 0x8c, 0x72, + 0x45, 0xb9, 0x3a, 0xb0, 0x30, 0xbe, 0xdf, 0x2e, 0x8e, 0x3c, 0xe3, 0x17, 0xe2, 0x10, 0x8e, 0x2a, + 0x70, 0x7a, 0xdb, 0xf3, 0x9a, 0xf3, 0xba, 0x4e, 0x5c, 0x37, 0xa8, 0x31, 0x53, 0x60, 0xcd, 0xce, + 0xed, 0xb7, 0x8b, 0xa7, 0x6f, 0xac, 0xaf, 0xaf, 0xc5, 0xc0, 0x38, 0xad, 0x8d, 0xfa, 0x0b, 0x0a, + 0x4c, 0x05, 0x9d, 0xc1, 0xe4, 0xe5, 0x16, 0x71, 0x3d, 0x17, 0x61, 0x38, 0xdb, 0xd0, 0x76, 0x57, + 0x6d, 0x6b, 0xa5, 0xe5, 0x69, 0x9e, 0x61, 0xd5, 0x2b, 0xd6, 0x96, 0x69, 0xd4, 0xb7, 0x3d, 0xd1, + 0xb5, 0xd9, 0xfd, 0x76, 0xf1, 0xec, 0x4a, 0x6a, 0x0d, 0x9c, 0xd1, 0x92, 0x76, 0xba, 0xa1, 0xed, + 0x26, 0x10, 0x4a, 0x9d, 0x5e, 0x49, 0x82, 0x71, 0x5a, 0x1b, 0xf5, 0x1d, 0x30, 0xc5, 0xc7, 0x81, + 0x89, 0xeb, 0x39, 0x86, 0xee, 0x19, 0xb6, 0x85, 0xae, 0x40, 0xbf, 0xa5, 0x35, 0x08, 0xeb, 0xe1, + 0xc8, 0xc2, 0xd8, 0x17, 0xdb, 0xc5, 0x37, 0xec, 0xb7, 0x8b, 0xfd, 0xab, 0x5a, 0x83, 0x60, 0x06, + 0x51, 0xff, 0x77, 0x01, 0x2e, 0x26, 0xda, 0x3d, 0x6b, 0x78, 0xdb, 0xb7, 0x9a, 0xf4, 0x3f, 0x17, + 0x7d, 0xbf, 0x02, 0x53, 0x5a, 0xbc, 0x02, 0x43, 0x38, 0xfa, 0xd0, 0xe2, 0xdc, 0xe1, 0x3f, 0xf0, + 0xb9, 0x04, 0xb5, 0x85, 0xf3, 0xa2, 0x5f, 0xc9, 0x01, 0xe0, 0x24, 0x69, 0xf4, 0x31, 0x05, 0x86, + 0x6c, 0xde, 0xb9, 0x99, 0xc2, 0x95, 0xbe, 0xab, 0xa3, 0x0f, 0x7d, 0xfb, 0x91, 0x74, 0x43, 0x1a, + 0xf4, 0x9c, 0xf8, 0xbb, 0x68, 0x79, 0xce, 0xde, 0xc2, 0x29, 0xd1, 0xbd, 0x21, 0x51, 0x8a, 0x7d, + 0xf2, 0xb3, 0x8f, 0xc1, 0x98, 0x5c, 0x13, 0x4d, 0x42, 0xdf, 0x6d, 0xc2, 0xb7, 0xea, 0x08, 0xa6, + 0xff, 0xa2, 0x69, 0x18, 0xd8, 0xd1, 0xcc, 0x16, 0x61, 0x4b, 0x3a, 0x82, 0xf9, 0x8f, 0xc7, 0x0a, + 0xef, 0x52, 0xd4, 0x87, 0x60, 0x60, 0xbe, 0x56, 0xb3, 0x2d, 0x74, 0x3f, 0x0c, 0x11, 0x4b, 0xdb, + 0x34, 0x49, 0x8d, 0x35, 0x1c, 0x0e, 0xe9, 0x2d, 0xf2, 0x62, 0xec, 0xc3, 0xd5, 0x1f, 0x2d, 0xc0, + 0x20, 0x6b, 0xe4, 0xa2, 0x1f, 0x52, 0xe0, 0xf4, 0xed, 0xd6, 0x26, 0x71, 0x2c, 0xe2, 0x11, 0xb7, + 0xac, 0xb9, 0xdb, 0x9b, 0xb6, 0xe6, 0xd4, 0xc4, 0xc2, 0x5c, 0xcf, 0x33, 0x23, 0x37, 0x93, 0xe8, + 0xf8, 0x1e, 0x4c, 0x01, 0xe0, 0x34, 0xe2, 0x68, 0x07, 0xc6, 0xac, 0xba, 0x61, 0xed, 0x56, 0xac, + 0xba, 0x43, 0x5c, 0x97, 0x0d, 0x7a, 0xf4, 0xa1, 0xa7, 0xf2, 0x74, 0x66, 0x55, 0xc2, 0xb3, 0x30, + 0xb9, 0xdf, 0x2e, 0x8e, 0xc9, 0x25, 0x38, 0x42, 0x47, 0xfd, 0x7b, 0x05, 0x4e, 0xcd, 0xd7, 0x1a, + 0x86, 0x4b, 0x39, 0xed, 0x9a, 0xd9, 0xaa, 0x1b, 0x5d, 0x6c, 0x7d, 0xf4, 0x34, 0x0c, 0xea, 0xb6, + 0xb5, 0x65, 0xd4, 0x45, 0x3f, 0xdf, 0x36, 0xc7, 0x39, 0xd7, 0x9c, 0xcc, 0xb9, 0x58, 0xf7, 0x04, + 0xc7, 0x9b, 0xc3, 0xda, 0x9d, 0x45, 0x9f, 0xa1, 0x2f, 0xc0, 0x7e, 0xbb, 0x38, 0x58, 0x62, 0x08, + 0xb0, 0x40, 0x84, 0xae, 0xc2, 0x70, 0xcd, 0x70, 0xf9, 0x62, 0xf6, 0xb1, 0xc5, 0x1c, 0xdb, 0x6f, + 0x17, 0x87, 0xcb, 0xa2, 0x0c, 0x07, 0x50, 0xb4, 0x0c, 0xd3, 0x74, 0x06, 0x79, 0xbb, 0x2a, 0xd1, + 0x1d, 0xe2, 0xd1, 0xae, 0xcd, 0xf4, 0xb3, 0xee, 0xce, 0xec, 0xb7, 0x8b, 0xd3, 0x37, 0x53, 0xe0, + 0x38, 0xb5, 0x95, 0xba, 0x04, 0xc3, 0xf3, 0x26, 0x71, 0x28, 0x43, 0x40, 0x8f, 0xc1, 0x04, 0x69, + 0x68, 0x86, 0x89, 0x89, 0x4e, 0x8c, 0x1d, 0xe2, 0xb8, 0x33, 0xca, 0x95, 0xbe, 0xab, 0x23, 0x0b, + 0x68, 0xbf, 0x5d, 0x9c, 0x58, 0x8c, 0x40, 0x70, 0xac, 0xa6, 0xfa, 0x61, 0x05, 0x46, 0xe7, 0x5b, + 0x35, 0xc3, 0xe3, 0xe3, 0x42, 0x0e, 0x8c, 0x6a, 0xf4, 0xe7, 0x9a, 0x6d, 0x1a, 0xfa, 0x9e, 0xd8, + 0x5c, 0x4f, 0xe6, 0xfa, 0xdc, 0x42, 0x34, 0x0b, 0xa7, 0xf6, 0xdb, 0xc5, 0x51, 0xa9, 0x00, 0xcb, + 0x44, 0xd4, 0x6d, 0x90, 0x61, 0xe8, 0xbd, 0x30, 0xc6, 0x87, 0xbb, 0xa2, 0x35, 0x31, 0xd9, 0x12, + 0x7d, 0xb8, 0x4f, 0x5a, 0x2b, 0x9f, 0xd0, 0xdc, 0xad, 0xcd, 0x97, 0x88, 0xee, 0x61, 0xb2, 0x45, + 0x1c, 0x62, 0xe9, 0x84, 0x6f, 0x9b, 0x92, 0xd4, 0x18, 0x47, 0x50, 0xa9, 0x3f, 0xac, 0xc0, 0xa5, + 0xf9, 0x96, 0xb7, 0x6d, 0x3b, 0xc6, 0x5d, 0xe2, 0x84, 0xd3, 0x1d, 0x60, 0x40, 0x4f, 0xc0, 0x84, + 0x16, 0x54, 0x58, 0x0d, 0xb7, 0xd3, 0x59, 0xb1, 0x9d, 0x26, 0xe6, 0x23, 0x50, 0x1c, 0xab, 0x8d, + 0x1e, 0x02, 0x70, 0xc3, 0xb5, 0x65, 0x3c, 0x60, 0x01, 0x89, 0xb6, 0x20, 0xad, 0xaa, 0x54, 0x4b, + 0xfd, 0x0b, 0x7a, 0x14, 0xee, 0x68, 0x86, 0xa9, 0x6d, 0x1a, 0xa6, 0xe1, 0xed, 0x3d, 0x6f, 0x5b, + 0xa4, 0x8b, 0xdd, 0xbc, 0x01, 0xe7, 0x5a, 0x96, 0xc6, 0xdb, 0x99, 0x64, 0x85, 0xef, 0xdf, 0xf5, + 0xbd, 0x26, 0xe1, 0x5c, 0x72, 0x64, 0xe1, 0xc2, 0x7e, 0xbb, 0x78, 0x6e, 0x23, 0xbd, 0x0a, 0xce, + 0x6a, 0x4b, 0x4f, 0x3d, 0x09, 0xf4, 0x8c, 0x6d, 0xb6, 0x1a, 0x02, 0x6b, 0x1f, 0xc3, 0xca, 0x4e, + 0xbd, 0x8d, 0xd4, 0x1a, 0x38, 0xa3, 0xa5, 0xfa, 0xf9, 0x02, 0x0c, 0x2e, 0x68, 0xfa, 0xed, 0x56, + 0x13, 0xbd, 0x15, 0x86, 0x9b, 0x8e, 0xbd, 0x63, 0xd4, 0x88, 0x23, 0xc6, 0x36, 0x29, 0xc6, 0x36, + 0xbc, 0x26, 0xca, 0x71, 0x50, 0x03, 0x19, 0x30, 0xe1, 0xff, 0x5f, 0xea, 0xe1, 0xcb, 0x65, 0x5f, + 0xc2, 0x5a, 0x04, 0x11, 0x8e, 0x21, 0x46, 0x2a, 0x0c, 0x3a, 0xa4, 0x4e, 0x8f, 0xba, 0x3e, 0xd6, + 0x2d, 0xf6, 0xb5, 0x63, 0x56, 0x82, 0x05, 0x04, 0xbd, 0x0f, 0x26, 0x74, 0x87, 0xd4, 0x88, 0xe5, + 0x19, 0x9a, 0xe9, 0xd2, 0xcd, 0x39, 0xd0, 0xfd, 0xe6, 0x64, 0x9d, 0x28, 0x45, 0x9a, 0xe3, 0x18, + 0x3a, 0xf5, 0x8b, 0x05, 0x18, 0xe3, 0x13, 0xb5, 0xd0, 0xd2, 0x6f, 0x13, 0x0f, 0x7d, 0x07, 0x0c, + 0x53, 0xf9, 0xae, 0xa6, 0x79, 0x9a, 0xf8, 0x10, 0xbe, 0x25, 0x73, 0xe8, 0xec, 0x1b, 0xa4, 0xb5, + 0x43, 0xea, 0x2b, 0xc4, 0xd3, 0xc2, 0xfd, 0x17, 0x96, 0xe1, 0x00, 0x2b, 0xda, 0x82, 0x7e, 0xb7, + 0x49, 0x74, 0x31, 0xb1, 0xe5, 0x3c, 0x9f, 0xba, 0xdc, 0xe3, 0x6a, 0x93, 0xe8, 0xe1, 0x76, 0xa5, + 0xbf, 0x30, 0xc3, 0x8f, 0x2c, 0x18, 0x74, 0x3d, 0xcd, 0x6b, 0xb9, 0x6c, 0x7e, 0x47, 0x1f, 0x5a, + 0xea, 0x99, 0x12, 0xc3, 0xb6, 0x30, 0x21, 0x68, 0x0d, 0xf2, 0xdf, 0x58, 0x50, 0x51, 0xff, 0xbd, + 0x02, 0x93, 0x72, 0xf5, 0x65, 0xc3, 0xf5, 0xd0, 0xb7, 0x25, 0xa6, 0x73, 0xae, 0xbb, 0xe9, 0xa4, + 0xad, 0xd9, 0x64, 0x06, 0xbb, 0xd5, 0x2f, 0x91, 0xa6, 0x92, 0xc0, 0x80, 0xe1, 0x91, 0x86, 0x2f, + 0xa5, 0x3c, 0xd5, 0xeb, 0x08, 0x17, 0xc6, 0x05, 0xb1, 0x81, 0x0a, 0x45, 0x8b, 0x39, 0x76, 0xf5, + 0x3b, 0x60, 0x5a, 0xae, 0xe5, 0xef, 0x6b, 0xca, 0x32, 0xbc, 0xbd, 0x66, 0x82, 0x65, 0xd0, 0x4f, + 0x10, 0x33, 0x08, 0x7a, 0x4b, 0xb0, 0xc7, 0x39, 0x67, 0x0a, 0xe6, 0x2e, 0xba, 0xcf, 0xd5, 0x0f, + 0xf5, 0x45, 0xe7, 0x8e, 0x2e, 0x23, 0xda, 0x89, 0x7d, 0xb9, 0xa3, 0x0f, 0xdd, 0xe8, 0x75, 0x80, + 0x7e, 0xd7, 0x5f, 0x2b, 0x3c, 0xe0, 0x2a, 0x0c, 0xbb, 0x84, 0xd4, 0xa4, 0x73, 0x99, 0x9d, 0xe6, + 0x55, 0x51, 0x86, 0x03, 0xe8, 0xf1, 0x73, 0x82, 0x57, 0xfb, 0x01, 0x25, 0x77, 0xbb, 0x3c, 0x19, + 0xbc, 0x44, 0x2c, 0x45, 0x2f, 0x93, 0x21, 0x3e, 0x9c, 0x18, 0x62, 0x74, 0x17, 0xc6, 0x4d, 0xcd, + 0xf5, 0x6e, 0x35, 0xa9, 0xde, 0xe6, 0xef, 0x99, 0xd1, 0x87, 0xe6, 0xf3, 0x2c, 0xfa, 0xb2, 0x8c, + 0x68, 0x61, 0x6a, 0xbf, 0x5d, 0x1c, 0x8f, 0x14, 0xe1, 0x28, 0x29, 0xf4, 0x12, 0x8c, 0xd0, 0x82, + 0x45, 0xc7, 0xb1, 0x1d, 0xc1, 0x2f, 0x1e, 0xcf, 0x4b, 0x97, 0x21, 0xe1, 0x7a, 0x64, 0xf0, 0x13, + 0x87, 0xe8, 0xd1, 0x7b, 0x00, 0xd9, 0x9b, 0x4c, 0x93, 0xaf, 0x5d, 0xe7, 0x4a, 0x2a, 0x1d, 0x2c, + 0x5d, 0xfe, 0xbe, 0x85, 0x59, 0xb1, 0x2f, 0xd1, 0xad, 0x44, 0x0d, 0x9c, 0xd2, 0x0a, 0xdd, 0x06, + 0x14, 0x28, 0xba, 0xfc, 0xb4, 0x3f, 0x60, 0x6b, 0x04, 0x95, 0xc4, 0xd6, 0x38, 0x4b, 0x89, 0x5d, + 0x4f, 0xa0, 0xc0, 0x29, 0x68, 0xd5, 0xdf, 0x2a, 0xc0, 0x28, 0xdf, 0x22, 0x5c, 0x19, 0x39, 0xfe, + 0xb3, 0x82, 0x44, 0xce, 0x8a, 0x52, 0xfe, 0xcf, 0x9f, 0x75, 0x38, 0xf3, 0xa8, 0x68, 0xc4, 0x8e, + 0x8a, 0xc5, 0x5e, 0x09, 0x75, 0x3e, 0x29, 0xfe, 0x58, 0x81, 0x53, 0x52, 0xed, 0x13, 0x38, 0x28, + 0x6a, 0xd1, 0x83, 0xe2, 0xc9, 0x1e, 0xc7, 0x97, 0x71, 0x4e, 0xd8, 0x91, 0x61, 0x31, 0x1e, 0xfe, + 0x10, 0xc0, 0x26, 0x63, 0x27, 0x92, 0x68, 0x1b, 0x2c, 0xf9, 0x42, 0x00, 0xc1, 0x52, 0xad, 0x08, + 0x53, 0x2c, 0x74, 0x62, 0x8a, 0xea, 0x7f, 0xea, 0x83, 0xa9, 0xc4, 0xb4, 0x27, 0xf9, 0x88, 0xf2, + 0x75, 0xe2, 0x23, 0x85, 0xaf, 0x07, 0x1f, 0xe9, 0xcb, 0xc5, 0x47, 0xba, 0x3f, 0x88, 0x1c, 0x40, + 0x0d, 0xa3, 0xce, 0x9b, 0x55, 0x3d, 0xcd, 0xf1, 0xd6, 0x8d, 0x06, 0x11, 0x1c, 0xe7, 0x9b, 0xbb, + 0xdb, 0xb2, 0xb4, 0x05, 0x67, 0x3c, 0x2b, 0x09, 0x4c, 0x38, 0x05, 0xbb, 0xfa, 0x5d, 0x05, 0x18, + 0x5a, 0xd0, 0x5c, 0xd6, 0xd3, 0x0f, 0xc0, 0x98, 0x40, 0x5d, 0x69, 0x68, 0x75, 0xd2, 0x8b, 0x39, + 0x42, 0xa0, 0x5c, 0x91, 0xd0, 0x71, 0x8d, 0x4e, 0x2e, 0xc1, 0x11, 0x72, 0x68, 0x0f, 0x46, 0x1b, + 0xa1, 0xf6, 0x22, 0x96, 0x78, 0xa9, 0x77, 0xea, 0x14, 0x1b, 0x57, 0x5b, 0xa5, 0x02, 0x2c, 0xd3, + 0x52, 0x5f, 0x84, 0xd3, 0x29, 0x3d, 0xee, 0x42, 0x71, 0x7b, 0x33, 0x0c, 0x51, 0xdd, 0x3b, 0x14, + 0xc3, 0x46, 0xf7, 0xdb, 0xc5, 0xa1, 0x67, 0x78, 0x11, 0xf6, 0x61, 0xea, 0x3b, 0xa9, 0x00, 0x10, + 0xef, 0x53, 0x37, 0x06, 0xbe, 0x01, 0x80, 0xd2, 0x3c, 0xb6, 0x3d, 0xbe, 0x95, 0x9e, 0x84, 0x81, + 0xe6, 0xb6, 0xe6, 0xfa, 0x2d, 0xee, 0xf7, 0x59, 0xc5, 0x1a, 0x2d, 0xbc, 0xd7, 0x2e, 0xce, 0xc8, + 0x82, 0x88, 0x68, 0xc4, 0x60, 0x98, 0xb7, 0xa3, 0x3b, 0x8c, 0x6e, 0xf2, 0x92, 0xdd, 0x68, 0x9a, + 0x84, 0x42, 0xd9, 0x0e, 0x2b, 0xe4, 0xdb, 0x61, 0xcb, 0x09, 0x4c, 0x38, 0x05, 0xbb, 0x4f, 0xb3, + 0x62, 0x19, 0x9e, 0xa1, 0x05, 0x34, 0xfb, 0xf2, 0xd3, 0x8c, 0x62, 0xc2, 0x29, 0xd8, 0xd1, 0x27, + 0x14, 0x98, 0x8d, 0x16, 0x2f, 0x19, 0x96, 0xe1, 0x6e, 0x93, 0x1a, 0x23, 0xde, 0x7f, 0x68, 0xe2, + 0x97, 0xf7, 0xdb, 0xc5, 0xd9, 0xe5, 0x4c, 0x8c, 0xb8, 0x03, 0x35, 0xf4, 0x7d, 0x0a, 0x5c, 0x88, + 0xcd, 0x8b, 0x63, 0xd4, 0xeb, 0xc4, 0x11, 0xbd, 0x39, 0xfc, 0x07, 0x5e, 0xdc, 0x6f, 0x17, 0x2f, + 0x2c, 0x67, 0xa3, 0xc4, 0x9d, 0xe8, 0xa1, 0x9f, 0x52, 0xe0, 0x6c, 0x93, 0x58, 0x35, 0xc3, 0xaa, + 0x3f, 0x6b, 0x3b, 0xb7, 0x89, 0xe3, 0x62, 0xdb, 0x34, 0xed, 0x96, 0xe7, 0xce, 0x0c, 0xb2, 0x33, + 0xac, 0x92, 0xe7, 0x9b, 0x5b, 0x4b, 0xc3, 0xb8, 0x70, 0x59, 0x6c, 0xd1, 0xb3, 0xa9, 0x60, 0x17, + 0x67, 0x74, 0x44, 0xfd, 0x4d, 0x05, 0xfa, 0x4a, 0xb8, 0x82, 0x1e, 0x88, 0x7c, 0x22, 0xe7, 0xe4, + 0x4f, 0xe4, 0x5e, 0xbb, 0x38, 0x54, 0xc2, 0x15, 0xe9, 0x63, 0xfc, 0x3e, 0x05, 0xa6, 0x74, 0xdb, + 0xf2, 0x34, 0x3a, 0x77, 0x98, 0xcb, 0xca, 0xfe, 0xb9, 0x9c, 0x4b, 0x19, 0x2e, 0xc5, 0x90, 0x85, + 0xc6, 0xee, 0x38, 0xc4, 0xc5, 0x49, 0xca, 0xea, 0xe7, 0x14, 0x98, 0x2e, 0x69, 0x4d, 0x61, 0x0a, + 0x2a, 0x93, 0x2d, 0x83, 0x6e, 0x90, 0x6e, 0x2c, 0xfb, 0x68, 0x1b, 0x06, 0x99, 0xb5, 0xd9, 0xed, + 0x45, 0x97, 0x0f, 0x69, 0x3f, 0xc3, 0x70, 0x71, 0x3b, 0x08, 0xff, 0x1f, 0x0b, 0xfc, 0xea, 0xe3, + 0x30, 0x19, 0xaf, 0x87, 0x8a, 0xbe, 0x4c, 0xc3, 0x8d, 0x8f, 0x23, 0x71, 0x71, 0xe4, 0xb1, 0xe1, + 0x7f, 0xf2, 0x6a, 0xf1, 0x0d, 0x1f, 0xfa, 0xb3, 0x2b, 0x6f, 0x50, 0xbf, 0xa2, 0xc0, 0x58, 0xc9, + 0xb4, 0x5b, 0xb5, 0x35, 0xc7, 0xde, 0x32, 0x4c, 0xf2, 0xfa, 0xb0, 0x72, 0xc8, 0x3d, 0xce, 0x12, + 0x5d, 0x99, 0xd5, 0x41, 0xae, 0xf8, 0x3a, 0xb1, 0x3a, 0xc8, 0x5d, 0xce, 0x90, 0x26, 0x5f, 0x80, + 0x33, 0x72, 0xad, 0xd0, 0x64, 0x7a, 0x05, 0xfa, 0x6f, 0x1b, 0x56, 0x2d, 0xbe, 0x31, 0x6f, 0x1a, + 0x56, 0x0d, 0x33, 0x48, 0xb0, 0x75, 0x0b, 0x99, 0x67, 0xd6, 0xdf, 0x8c, 0x44, 0xa7, 0x8d, 0x09, + 0xab, 0x57, 0x61, 0x58, 0xd7, 0x16, 0x5a, 0x56, 0xcd, 0x0c, 0x76, 0x3d, 0x9d, 0x82, 0xd2, 0x3c, + 0x2f, 0xc3, 0x01, 0x14, 0xdd, 0x05, 0x08, 0x6f, 0x27, 0x7a, 0x11, 0x02, 0xc2, 0x8b, 0x8f, 0x2a, + 0xf1, 0x3c, 0xc3, 0xaa, 0xbb, 0xe1, 0xbe, 0x0a, 0x61, 0x58, 0xa2, 0x86, 0x3e, 0x00, 0xe3, 0xb2, + 0x44, 0xc2, 0xcd, 0xa4, 0x39, 0x97, 0x21, 0x22, 0xfa, 0x9c, 0x11, 0x84, 0xc7, 0xe5, 0x52, 0x17, + 0x47, 0xa9, 0xa1, 0xbd, 0x40, 0xfe, 0xe2, 0x46, 0xda, 0xfe, 0xfc, 0x1a, 0x85, 0x2c, 0xfa, 0x4c, + 0x0b, 0xe2, 0x63, 0x11, 0xa3, 0x71, 0x84, 0x54, 0x8a, 0x61, 0x66, 0xe0, 0xb8, 0x0c, 0x33, 0x04, + 0x86, 0xb8, 0x69, 0xca, 0x3f, 0x6e, 0x1e, 0xcb, 0x33, 0x40, 0x6e, 0xe5, 0x0a, 0xaf, 0xdb, 0xf8, + 0x6f, 0x17, 0xfb, 0xb8, 0xd1, 0x0e, 0x8c, 0x51, 0xc1, 0xba, 0x4a, 0x4c, 0xa2, 0x7b, 0xb6, 0x33, + 0x33, 0x94, 0xff, 0x3a, 0xab, 0x2a, 0xe1, 0xe1, 0x52, 0xac, 0x5c, 0x82, 0x23, 0x74, 0x02, 0xcb, + 0xdd, 0x70, 0xa6, 0xe5, 0xae, 0x05, 0xa3, 0x3b, 0x92, 0x29, 0x7e, 0x84, 0x4d, 0xc2, 0x13, 0x79, + 0x3a, 0x16, 0xda, 0xe5, 0x17, 0x4e, 0x0b, 0x42, 0xa3, 0xb2, 0x0d, 0x5f, 0xa6, 0x83, 0x36, 0x61, + 0x68, 0x93, 0xcb, 0xa0, 0x33, 0xc0, 0xe6, 0xe2, 0xdd, 0x3d, 0x88, 0xd6, 0x5c, 0xce, 0x15, 0x3f, + 0xb0, 0x8f, 0x18, 0xbd, 0x08, 0x83, 0xa6, 0xd1, 0x30, 0x3c, 0x77, 0x66, 0x94, 0x91, 0xc8, 0xb5, + 0xb4, 0xcb, 0x0c, 0x03, 0x3f, 0xac, 0xf8, 0xff, 0x58, 0x60, 0x45, 0x9f, 0x52, 0xe0, 0xb4, 0xd8, + 0x86, 0xc1, 0xa1, 0x65, 0x10, 0x77, 0x66, 0x8c, 0xcd, 0xe1, 0x8d, 0xde, 0x0e, 0xc9, 0xf0, 0x80, + 0x5e, 0xb8, 0x20, 0x66, 0xf3, 0xf4, 0x4a, 0x92, 0x18, 0x4e, 0xeb, 0x81, 0xfa, 0x89, 0x29, 0x98, + 0x2a, 0x99, 0x2d, 0xd7, 0x23, 0xce, 0xbc, 0xf0, 0x64, 0x21, 0x0e, 0xfa, 0x88, 0x02, 0x67, 0xd9, + 0xbf, 0x65, 0xfb, 0x8e, 0x55, 0x26, 0xa6, 0xb6, 0x37, 0xbf, 0x45, 0x6b, 0xd4, 0x6a, 0x87, 0x3b, + 0x3c, 0xca, 0x2d, 0xa1, 0x26, 0xb3, 0x1b, 0x9b, 0x6a, 0x2a, 0x46, 0x9c, 0x41, 0x09, 0x7d, 0xaf, + 0x02, 0xe7, 0x53, 0x40, 0x65, 0x62, 0x12, 0xcf, 0x17, 0xfe, 0x0f, 0xdb, 0x8f, 0x4b, 0xfb, 0xed, + 0xe2, 0xf9, 0x6a, 0x16, 0x52, 0x9c, 0x4d, 0x0f, 0x7d, 0xbf, 0x02, 0xb3, 0x29, 0xd0, 0x25, 0xcd, + 0x30, 0x5b, 0x8e, 0xaf, 0x17, 0x1c, 0xb6, 0x3b, 0x4c, 0x3c, 0xaf, 0x66, 0x62, 0xc5, 0x1d, 0x28, + 0xa2, 0x0f, 0xc2, 0x99, 0x00, 0xba, 0x61, 0x59, 0x84, 0xd4, 0x22, 0x5a, 0xc2, 0x61, 0xbb, 0x72, + 0x7e, 0xbf, 0x5d, 0x3c, 0x53, 0x4d, 0x43, 0x88, 0xd3, 0xe9, 0xa0, 0x3a, 0x5c, 0x0a, 0x01, 0x9e, + 0x61, 0x1a, 0x77, 0xb9, 0x22, 0xb3, 0xed, 0x10, 0x77, 0xdb, 0x36, 0x6b, 0x8c, 0x15, 0x2b, 0x0b, + 0x6f, 0xdc, 0x6f, 0x17, 0x2f, 0x55, 0x3b, 0x55, 0xc4, 0x9d, 0xf1, 0xa0, 0x1a, 0x8c, 0xb9, 0xba, + 0x66, 0x55, 0x2c, 0x8f, 0x38, 0x3b, 0x9a, 0x39, 0x33, 0x98, 0x6b, 0x80, 0x9c, 0x01, 0x4a, 0x78, + 0x70, 0x04, 0x2b, 0x7a, 0x17, 0x0c, 0x93, 0xdd, 0xa6, 0x66, 0xd5, 0x08, 0x67, 0xba, 0x23, 0x0b, + 0x17, 0xe9, 0x51, 0xbf, 0x28, 0xca, 0xee, 0xb5, 0x8b, 0x63, 0xfe, 0xff, 0x2b, 0x76, 0x8d, 0xe0, + 0xa0, 0x36, 0x7a, 0x3f, 0x4c, 0x33, 0x57, 0x9b, 0x1a, 0x61, 0x47, 0x88, 0xeb, 0xeb, 0x8a, 0xc3, + 0xb9, 0xfa, 0xc9, 0xae, 0xe1, 0x57, 0x52, 0xf0, 0xe1, 0x54, 0x2a, 0x74, 0x19, 0x1a, 0xda, 0xee, + 0x75, 0x47, 0xd3, 0xc9, 0x56, 0xcb, 0x5c, 0x27, 0x4e, 0xc3, 0xb0, 0xb8, 0xb1, 0x84, 0xe8, 0xb6, + 0x55, 0xa3, 0x8c, 0x5a, 0xb9, 0x3a, 0xc0, 0x97, 0x61, 0xa5, 0x53, 0x45, 0xdc, 0x19, 0x0f, 0x7a, + 0x18, 0xc6, 0x8c, 0xba, 0x65, 0x3b, 0x64, 0x5d, 0x33, 0x2c, 0xcf, 0x9d, 0x01, 0x26, 0x64, 0xb3, + 0x69, 0xad, 0x48, 0xe5, 0x38, 0x52, 0x0b, 0xed, 0x00, 0xb2, 0xc8, 0x9d, 0x35, 0xbb, 0xc6, 0xb6, + 0xc0, 0x46, 0x93, 0x6d, 0x64, 0xc1, 0x66, 0x0f, 0x3b, 0x35, 0x4c, 0x95, 0x5e, 0x4d, 0x60, 0xc3, + 0x29, 0x14, 0xd0, 0x12, 0xa0, 0x86, 0xb6, 0xbb, 0xd8, 0x68, 0x7a, 0x7b, 0x0b, 0x2d, 0xf3, 0xb6, + 0xe0, 0x1a, 0x63, 0x6c, 0x2e, 0xb8, 0xa1, 0x29, 0x01, 0xc5, 0x29, 0x2d, 0x90, 0x06, 0x17, 0xf8, + 0x78, 0xca, 0x1a, 0x69, 0xd8, 0x96, 0x4b, 0x3c, 0x57, 0xda, 0xa4, 0x33, 0xe3, 0xcc, 0xe1, 0x82, + 0x29, 0xb6, 0x95, 0xec, 0x6a, 0xb8, 0x13, 0x8e, 0xa8, 0xcb, 0xd9, 0xc4, 0x01, 0x2e, 0x67, 0x8f, + 0xc0, 0xb8, 0xeb, 0x69, 0x8e, 0xd7, 0x6a, 0x8a, 0x65, 0x38, 0xc5, 0x96, 0x81, 0xd9, 0x21, 0xab, + 0x32, 0x00, 0x47, 0xeb, 0xd1, 0xe5, 0xe3, 0xc6, 0x66, 0xd1, 0x6e, 0x32, 0x5c, 0xbe, 0xaa, 0x54, + 0x8e, 0x23, 0xb5, 0xd0, 0x06, 0x9c, 0x6b, 0x68, 0xbb, 0xc1, 0xe7, 0xbb, 0xa6, 0x39, 0x9a, 0x69, + 0x12, 0xd3, 0x70, 0x1b, 0x33, 0x53, 0xac, 0xa7, 0xec, 0x86, 0x7f, 0x25, 0xbd, 0x0a, 0xce, 0x6a, + 0x2b, 0x7c, 0xd0, 0xca, 0x8e, 0x66, 0x44, 0x50, 0xa2, 0x88, 0x0f, 0x5a, 0x1c, 0x8c, 0xd3, 0xda, + 0xa0, 0x9f, 0x50, 0xa0, 0xc8, 0xce, 0x3f, 0xcd, 0xa4, 0x1f, 0xc7, 0x75, 0xc7, 0x6e, 0x35, 0x17, + 0x34, 0xfd, 0xb6, 0xbd, 0xb5, 0xe5, 0x6f, 0x9c, 0x99, 0xd3, 0xb9, 0xb6, 0xdb, 0x7d, 0xfb, 0xed, + 0x62, 0xb1, 0xd2, 0x19, 0x35, 0x3e, 0x88, 0x36, 0xfa, 0x41, 0x05, 0x2e, 0x88, 0x0f, 0x37, 0xb5, + 0x6f, 0xd3, 0xb9, 0xfa, 0xc6, 0x76, 0xdc, 0x4a, 0x36, 0x5a, 0xdc, 0x89, 0x26, 0xfa, 0x11, 0x05, + 0x2e, 0x5a, 0x31, 0x20, 0x26, 0x2e, 0x61, 0xb6, 0x55, 0xbb, 0xe5, 0xcd, 0x9c, 0xc9, 0xd5, 0xa9, + 0x2b, 0xfb, 0xed, 0xe2, 0xc5, 0xd5, 0x0e, 0x78, 0x71, 0x47, 0xaa, 0xea, 0xff, 0xea, 0x87, 0x99, + 0x84, 0x30, 0xe2, 0xfb, 0x04, 0x1e, 0x78, 0xdc, 0x28, 0x47, 0x74, 0xdc, 0x34, 0xe1, 0x4a, 0x50, + 0xe1, 0x7a, 0xb3, 0x95, 0x4a, 0xab, 0xc0, 0x68, 0xbd, 0x69, 0xbf, 0x5d, 0xbc, 0x52, 0x3d, 0xa0, + 0x2e, 0x3e, 0x10, 0x5b, 0xf6, 0x51, 0xde, 0x77, 0x42, 0x47, 0xf9, 0xfb, 0x61, 0x5a, 0x02, 0x38, + 0x44, 0xab, 0xed, 0xf5, 0x20, 0x4a, 0xb0, 0x13, 0xac, 0x9a, 0x82, 0x0f, 0xa7, 0x52, 0xc9, 0x3c, + 0x3f, 0x07, 0x4e, 0xe2, 0xfc, 0x54, 0xdb, 0x7d, 0x30, 0x52, 0xb2, 0xad, 0x1a, 0x37, 0x71, 0x3d, + 0x18, 0x71, 0x60, 0xb8, 0x24, 0xab, 0x41, 0xf7, 0xda, 0xc5, 0xf1, 0xa0, 0xa2, 0xa4, 0x17, 0x3d, + 0x1a, 0x5c, 0x15, 0x72, 0xe3, 0xc2, 0x1b, 0xa3, 0x77, 0x7c, 0xf7, 0xda, 0xc5, 0x53, 0x41, 0xb3, + 0xe8, 0xb5, 0x1f, 0x3d, 0x1c, 0x4d, 0xcd, 0xf5, 0xd6, 0x1d, 0xcd, 0x72, 0x8d, 0x1e, 0x6c, 0xcc, + 0xc1, 0xdd, 0xce, 0x72, 0x02, 0x1b, 0x4e, 0xa1, 0x80, 0x5e, 0x82, 0x09, 0x5a, 0xba, 0xd1, 0xac, + 0x69, 0x1e, 0xc9, 0x69, 0x5a, 0x0e, 0xdc, 0xd1, 0x96, 0x23, 0x98, 0x70, 0x0c, 0x33, 0x77, 0xf8, + 0xd0, 0x5c, 0xdb, 0x62, 0xeb, 0x19, 0x71, 0xf8, 0xa0, 0xa5, 0x58, 0x40, 0xd1, 0xfd, 0x30, 0xd4, + 0x20, 0xae, 0xab, 0xd5, 0x09, 0x13, 0xf0, 0x46, 0x42, 0x1d, 0x79, 0x85, 0x17, 0x63, 0x1f, 0x8e, + 0xde, 0x0a, 0x03, 0xba, 0x5d, 0x23, 0xee, 0xcc, 0x10, 0x3b, 0xc3, 0xe8, 0x71, 0x3e, 0x50, 0xa2, + 0x05, 0xf7, 0xda, 0xc5, 0x11, 0x76, 0x13, 0x46, 0x7f, 0x61, 0x5e, 0x49, 0xfd, 0x49, 0x05, 0x26, + 0xe3, 0x76, 0xcf, 0x2e, 0x1c, 0x55, 0x4e, 0xce, 0xe7, 0x43, 0xfd, 0x94, 0x02, 0x63, 0xb4, 0x87, + 0x8e, 0x6d, 0xae, 0x99, 0x9a, 0x45, 0xd0, 0xf7, 0x28, 0x30, 0xb9, 0x6d, 0xd4, 0xb7, 0x65, 0x97, + 0x3c, 0xa1, 0x79, 0xe5, 0xb2, 0x1b, 0xde, 0x88, 0xe1, 0x5a, 0x98, 0xde, 0x6f, 0x17, 0x27, 0xe3, + 0xa5, 0x38, 0x41, 0x53, 0xfd, 0xcb, 0x02, 0x9c, 0x93, 0x7b, 0x36, 0x1f, 0xbe, 0x76, 0x40, 0x7f, + 0xac, 0x00, 0x34, 0x0c, 0x6b, 0xde, 0x34, 0xed, 0x3b, 0xcc, 0x8f, 0x98, 0xea, 0xb2, 0x2f, 0xe4, + 0xb5, 0x57, 0xa7, 0x50, 0x98, 0x5b, 0x09, 0xb0, 0xf3, 0x3b, 0xe6, 0xe7, 0x7c, 0x3b, 0x58, 0x08, + 0xb8, 0xd7, 0x2e, 0x16, 0x93, 0x4f, 0x2c, 0xe6, 0xb0, 0x78, 0xc7, 0xb0, 0x6c, 0xb8, 0xde, 0x47, + 0xfe, 0xa2, 0x63, 0x15, 0x7e, 0xd9, 0x1c, 0x0e, 0x64, 0xb6, 0x01, 0xa7, 0x62, 0x84, 0x53, 0x7c, + 0xac, 0xcb, 0xb2, 0x8f, 0xf5, 0x01, 0x4c, 0x6a, 0xce, 0x7f, 0x55, 0x31, 0xf7, 0x74, 0x4b, 0xb3, + 0x3c, 0x3a, 0xd3, 0x92, 0x4f, 0xf6, 0x9f, 0x17, 0x60, 0x5a, 0x4c, 0x80, 0x49, 0xb5, 0xcd, 0xa6, + 0x69, 0xef, 0x35, 0x88, 0x75, 0x12, 0x7e, 0x77, 0xfe, 0x47, 0x50, 0xc8, 0xfc, 0x08, 0x1a, 0x89, + 0x8f, 0xa0, 0x2f, 0xcf, 0x47, 0x10, 0xf0, 0x8a, 0x03, 0x6c, 0x6c, 0x18, 0xce, 0x1a, 0x16, 0xed, + 0xe8, 0x75, 0xb6, 0x61, 0x42, 0xef, 0x58, 0xc6, 0x9f, 0x86, 0xb9, 0x19, 0xa1, 0x92, 0x5a, 0x03, + 0x67, 0xb4, 0x54, 0xff, 0x5a, 0x81, 0x99, 0xb4, 0xf9, 0x3d, 0x01, 0xb3, 0x78, 0x23, 0x6a, 0x16, + 0xbf, 0xd1, 0xc3, 0xb7, 0x11, 0xe9, 0x7a, 0x86, 0x79, 0xfc, 0xaf, 0x0a, 0x70, 0x36, 0xac, 0x5e, + 0xb1, 0x5c, 0x4f, 0x33, 0x4d, 0x2e, 0xf0, 0x1d, 0xff, 0x5e, 0x6a, 0x46, 0x6e, 0x37, 0x56, 0x7b, + 0x1b, 0xaa, 0xdc, 0xf7, 0x4c, 0x17, 0x9d, 0xdd, 0x98, 0x8b, 0xce, 0xda, 0x11, 0xd2, 0xec, 0xec, + 0xad, 0xf3, 0x5f, 0x14, 0x98, 0x4d, 0x6f, 0x78, 0x02, 0x9b, 0xca, 0x8e, 0x6e, 0xaa, 0xf7, 0x1c, + 0xdd, 0xa8, 0x33, 0xb6, 0xd5, 0x2f, 0x14, 0xb2, 0x46, 0xcb, 0xae, 0x48, 0xb6, 0xe0, 0x94, 0x43, + 0xea, 0x86, 0xeb, 0x09, 0xc5, 0xe3, 0x70, 0xee, 0xf2, 0xfe, 0xd5, 0xe8, 0x29, 0x1c, 0xc5, 0x81, + 0xe3, 0x48, 0xd1, 0x2a, 0x0c, 0xb9, 0x84, 0xd4, 0x28, 0xfe, 0x42, 0xf7, 0xf8, 0x03, 0x21, 0xa2, + 0xca, 0xdb, 0x62, 0x1f, 0x09, 0xfa, 0x36, 0x18, 0xaf, 0x05, 0x5f, 0x14, 0xc5, 0xda, 0xd7, 0x3d, + 0x56, 0xa6, 0x6d, 0x97, 0xe5, 0xd6, 0x38, 0x8a, 0x4c, 0xfd, 0x3b, 0x05, 0x2e, 0x76, 0xda, 0x5b, + 0xe8, 0x65, 0x00, 0xdd, 0x97, 0x0a, 0x5d, 0x71, 0x78, 0x3e, 0x9e, 0x73, 0x2d, 0x39, 0x96, 0xf0, + 0x03, 0x0d, 0x8a, 0x5c, 0x2c, 0x11, 0x49, 0x71, 0xdc, 0x2c, 0x1c, 0x93, 0xe3, 0xa6, 0xfa, 0x5f, + 0x15, 0x99, 0x15, 0xc9, 0x6b, 0xfb, 0x7a, 0x63, 0x45, 0x72, 0xdf, 0x33, 0xaf, 0x5c, 0xbf, 0x5c, + 0x80, 0x2b, 0xe9, 0x4d, 0xa4, 0xf3, 0xfc, 0x29, 0x18, 0x6c, 0xf2, 0x27, 0x2d, 0xdc, 0xbb, 0xff, + 0x2a, 0xe5, 0x2c, 0xfc, 0xc1, 0xc9, 0xbd, 0x76, 0x71, 0x36, 0x8d, 0xd1, 0x8b, 0xa7, 0x2a, 0xa2, + 0x1d, 0x32, 0x62, 0x77, 0x43, 0x5c, 0x68, 0x7f, 0x7b, 0x97, 0xcc, 0x45, 0xdb, 0x24, 0x66, 0xd7, + 0xd7, 0x41, 0x1f, 0x56, 0x60, 0x22, 0xb2, 0xa3, 0xdd, 0x99, 0x01, 0xb6, 0x47, 0x73, 0xf9, 0xcc, + 0x45, 0x3e, 0x95, 0x50, 0x1a, 0x88, 0x14, 0xbb, 0x38, 0x46, 0x30, 0xc6, 0x66, 0xe5, 0x59, 0x7d, + 0xdd, 0xb1, 0x59, 0xb9, 0xf3, 0x19, 0x6c, 0xf6, 0xc7, 0x0b, 0x59, 0xa3, 0x65, 0x6c, 0xf6, 0x0e, + 0x8c, 0xf8, 0x62, 0xa4, 0xcf, 0x2e, 0x96, 0x7a, 0xed, 0x13, 0x47, 0xb7, 0x30, 0x25, 0xfa, 0x33, + 0xe2, 0x97, 0xb8, 0x38, 0xa4, 0x85, 0xbe, 0x5b, 0x01, 0x08, 0x17, 0x46, 0x7c, 0x54, 0xeb, 0x47, + 0x37, 0x1d, 0x92, 0x58, 0x33, 0x41, 0x3f, 0x69, 0x69, 0x53, 0x48, 0x74, 0xd5, 0x7f, 0xd7, 0x0f, + 0x28, 0xd9, 0xf7, 0xee, 0x6e, 0xfe, 0x0f, 0x10, 0x72, 0x9b, 0x30, 0xe9, 0x50, 0x69, 0x51, 0x37, + 0x4c, 0xe2, 0x1b, 0xc0, 0xf2, 0x59, 0x3e, 0x98, 0x5a, 0x85, 0x63, 0xb8, 0x70, 0x02, 0x3b, 0x7a, + 0x33, 0x0c, 0x35, 0x1d, 0xa3, 0xa1, 0x39, 0x7b, 0x4c, 0x29, 0x1e, 0xe6, 0xd7, 0x92, 0x6b, 0xbc, + 0x08, 0xfb, 0x30, 0xf4, 0x7e, 0x18, 0x31, 0x8d, 0x2d, 0xa2, 0xef, 0xe9, 0x26, 0x11, 0xb7, 0x1e, + 0xb7, 0x8e, 0x66, 0xcd, 0x97, 0x7d, 0xb4, 0xc2, 0x99, 0xd4, 0xff, 0x89, 0x43, 0x82, 0xa8, 0x02, + 0xa7, 0xef, 0x30, 0xf7, 0x26, 0x93, 0xb8, 0x6e, 0xb5, 0xd5, 0x6c, 0xda, 0x8e, 0x47, 0x6a, 0xec, + 0x6e, 0x64, 0x98, 0xdb, 0x68, 0x9f, 0x4d, 0x82, 0x71, 0x5a, 0x1b, 0xf4, 0x38, 0x80, 0xd6, 0xf2, + 0x6c, 0xfe, 0xb6, 0x74, 0x66, 0x98, 0x69, 0xed, 0x97, 0xe8, 0x62, 0xcf, 0x07, 0xa5, 0xf7, 0xda, + 0xc5, 0x51, 0x61, 0xfb, 0x63, 0x4b, 0x23, 0x35, 0x40, 0x2f, 0xc0, 0xb4, 0xce, 0x41, 0x25, 0xbb, + 0xd1, 0xd4, 0x3c, 0x43, 0x68, 0xc4, 0x23, 0x0c, 0xd1, 0x37, 0xed, 0xb7, 0x8b, 0xd3, 0xa5, 0x14, + 0x78, 0x1c, 0x65, 0x2a, 0x12, 0xf5, 0x13, 0x05, 0xb8, 0xd0, 0x61, 0x82, 0x10, 0xa6, 0x1f, 0x9e, + 0x58, 0x3f, 0xb1, 0xcd, 0x1e, 0xe6, 0x1f, 0x8b, 0x28, 0xbc, 0xd7, 0x2e, 0xde, 0xd7, 0x01, 0x41, + 0x95, 0xee, 0x73, 0x52, 0xdf, 0xc3, 0x21, 0x1a, 0x54, 0x81, 0xc1, 0x5a, 0x78, 0x8d, 0x39, 0xb2, + 0xf0, 0x20, 0x3d, 0x0a, 0xf8, 0x85, 0x43, 0xb7, 0xd8, 0x04, 0x02, 0xb4, 0x0c, 0x43, 0xdc, 0x3d, + 0x96, 0x88, 0x63, 0xe5, 0x21, 0x66, 0x32, 0xe1, 0x45, 0xdd, 0x22, 0xf3, 0x51, 0xa8, 0x7f, 0xab, + 0xc0, 0x50, 0xc9, 0x76, 0x48, 0x79, 0xb5, 0x8a, 0xf6, 0x60, 0x54, 0x0a, 0x6e, 0x20, 0x58, 0x6c, + 0x4e, 0x9e, 0xc3, 0x30, 0x4a, 0xaa, 0xbd, 0xff, 0x1c, 0x33, 0x28, 0xc0, 0x32, 0x2d, 0xf4, 0x32, + 0x9d, 0xf3, 0x3b, 0x8e, 0xe1, 0x51, 0xc2, 0xbd, 0xf8, 0x4b, 0x71, 0xc2, 0xd8, 0xc7, 0xc5, 0x77, + 0x7b, 0xf0, 0x13, 0x87, 0x54, 0xd4, 0x35, 0xca, 0x5e, 0xe2, 0xdd, 0x44, 0x8f, 0x41, 0x7f, 0xc3, + 0xae, 0xf9, 0xeb, 0xfe, 0x16, 0x9f, 0x79, 0xac, 0xd8, 0x35, 0x3a, 0xb7, 0x67, 0x93, 0x2d, 0xd8, + 0xd5, 0x20, 0x6b, 0xa3, 0xae, 0xc2, 0x64, 0x9c, 0x3e, 0x7a, 0x0c, 0x26, 0x74, 0xbb, 0xd1, 0xb0, + 0xad, 0x6a, 0x6b, 0x6b, 0xcb, 0xd8, 0x25, 0x91, 0x77, 0xb2, 0xa5, 0x08, 0x04, 0xc7, 0x6a, 0xaa, + 0x9f, 0x55, 0xa0, 0x8f, 0xae, 0x8b, 0x0a, 0x83, 0x35, 0xbb, 0xa1, 0x19, 0x96, 0xe8, 0x15, 0x73, + 0x38, 0x28, 0xb3, 0x12, 0x2c, 0x20, 0xa8, 0x09, 0x23, 0xbe, 0x44, 0xd6, 0x93, 0x87, 0x7f, 0x79, + 0xb5, 0x1a, 0x3c, 0x90, 0x0a, 0x8e, 0x09, 0xbf, 0xc4, 0xc5, 0x21, 0x11, 0x55, 0x83, 0xa9, 0xf2, + 0x6a, 0xb5, 0x62, 0xe9, 0x66, 0xab, 0x46, 0x16, 0x77, 0xd9, 0x1f, 0xca, 0xe7, 0x0c, 0x5e, 0x22, + 0xc6, 0xc9, 0xf8, 0x9c, 0xa8, 0x84, 0x7d, 0x18, 0xad, 0x46, 0x78, 0x0b, 0xf1, 0x6c, 0x94, 0x55, + 0x13, 0x48, 0xb0, 0x0f, 0x53, 0xbf, 0x52, 0x80, 0x51, 0xa9, 0x43, 0xc8, 0x84, 0x21, 0x3e, 0x5c, + 0xb7, 0x97, 0xd0, 0x00, 0x89, 0x5e, 0x73, 0xea, 0x7c, 0x42, 0x5d, 0xec, 0x93, 0x90, 0x79, 0x76, + 0xa1, 0x03, 0xcf, 0x9e, 0x8b, 0xbc, 0xbe, 0xe5, 0x9f, 0xe4, 0x44, 0xf6, 0xcb, 0x5b, 0x74, 0x51, + 0x1c, 0x4f, 0xdc, 0xc5, 0x7e, 0x38, 0x76, 0x34, 0x6d, 0xc1, 0xc0, 0x5d, 0xdb, 0x22, 0xae, 0xb0, + 0x85, 0x1f, 0xd1, 0x00, 0x99, 0x63, 0xe4, 0xf3, 0x14, 0x2f, 0xe6, 0xe8, 0xd5, 0x9f, 0x52, 0x00, + 0xca, 0x9a, 0xa7, 0x71, 0x2f, 0x9c, 0x2e, 0x1c, 0x3d, 0x2f, 0x46, 0x4e, 0xd5, 0xe1, 0xc4, 0x23, + 0xbf, 0x7e, 0xd7, 0xb8, 0xeb, 0x0f, 0x3f, 0x90, 0xd6, 0x39, 0xf6, 0xaa, 0x71, 0x97, 0x60, 0x06, + 0x47, 0x0f, 0xc0, 0x08, 0xb1, 0x74, 0x67, 0xaf, 0x49, 0x0f, 0x16, 0x6e, 0xe2, 0x61, 0x5f, 0xe8, + 0xa2, 0x5f, 0x88, 0x43, 0xb8, 0xfa, 0x20, 0x44, 0x55, 0xae, 0x2e, 0xfc, 0xd0, 0xff, 0x5e, 0x81, + 0x73, 0xe5, 0x96, 0x66, 0xce, 0x37, 0xe9, 0x46, 0xd5, 0xcc, 0x25, 0x9b, 0xbb, 0x73, 0x50, 0x3d, + 0xe4, 0xad, 0x30, 0xec, 0x0b, 0x39, 0xf1, 0x57, 0xc0, 0x3e, 0xa3, 0xc4, 0x41, 0x0d, 0xa4, 0xc1, + 0xb0, 0xeb, 0x8b, 0xdd, 0x85, 0x1e, 0xc4, 0x6e, 0x9f, 0x44, 0x20, 0x76, 0x07, 0x68, 0xb9, 0xf1, + 0x8b, 0x2d, 0x50, 0x95, 0x38, 0x3b, 0x86, 0x4e, 0xe6, 0x75, 0xdd, 0x6e, 0x59, 0x9e, 0x2b, 0x5e, + 0xf5, 0x0b, 0xe3, 0x57, 0x5a, 0x0d, 0x9c, 0xd1, 0x52, 0xfd, 0xa2, 0x02, 0xfd, 0x8b, 0xeb, 0xa5, + 0x32, 0xfa, 0x36, 0xe8, 0x0f, 0x58, 0x46, 0x4e, 0xaf, 0x2d, 0x8a, 0x87, 0xdb, 0xe9, 0xf8, 0x7a, + 0xaf, 0x50, 0x86, 0xc3, 0xb0, 0xa2, 0x4d, 0x18, 0x24, 0x3b, 0x84, 0x76, 0xb5, 0x70, 0x24, 0xf8, + 0x19, 0x4b, 0x5b, 0x64, 0x18, 0xb1, 0xc0, 0xac, 0x7e, 0x52, 0x01, 0x08, 0xab, 0xa0, 0xef, 0x4c, + 0x3b, 0x9d, 0x6e, 0x1e, 0xa1, 0xf5, 0xb9, 0xf3, 0x11, 0xa5, 0xfe, 0xfa, 0x00, 0x9c, 0xa7, 0xdd, + 0x11, 0x5b, 0xd5, 0xb0, 0xad, 0x9b, 0x64, 0xef, 0x1b, 0x2f, 0x1e, 0xbe, 0xf1, 0xe2, 0xe1, 0x08, + 0x5f, 0x3c, 0xbc, 0x00, 0xe7, 0xe9, 0x96, 0x13, 0x60, 0xc2, 0xdc, 0xbf, 0xd6, 0x1c, 0xd2, 0xd4, + 0x1c, 0x52, 0x63, 0xfa, 0xc0, 0x30, 0x77, 0x68, 0x9b, 0xcf, 0xaa, 0x84, 0xb3, 0xdb, 0xab, 0x4f, + 0xc2, 0x64, 0xb8, 0x77, 0xe7, 0x3d, 0x4c, 0x5c, 0x8f, 0xf2, 0xe3, 0xa8, 0x92, 0x39, 0xe2, 0x4b, + 0x4c, 0x49, 0xc5, 0x30, 0x8a, 0x40, 0x7c, 0x95, 0x87, 0x42, 0x70, 0x4f, 0x81, 0xc9, 0xc5, 0xdd, + 0xa6, 0xe1, 0xb0, 0x58, 0x0d, 0xfc, 0xc9, 0x11, 0xba, 0x3f, 0x7c, 0x99, 0xa4, 0x44, 0xaf, 0x01, + 0xe3, 0xaf, 0x93, 0xd0, 0x16, 0x4c, 0x10, 0xd6, 0x9c, 0xa9, 0x91, 0x9a, 0x97, 0xe7, 0xfb, 0xe0, + 0x01, 0x4a, 0x22, 0x58, 0x70, 0x0c, 0x2b, 0xaa, 0xc2, 0x84, 0x6e, 0x6a, 0xae, 0x6b, 0x6c, 0x19, + 0x7a, 0xf8, 0xa2, 0x6e, 0x64, 0xe1, 0x01, 0x26, 0xb4, 0x45, 0x20, 0xf7, 0xda, 0xc5, 0x33, 0xa2, + 0x9f, 0x51, 0x00, 0x8e, 0xa1, 0x50, 0x3f, 0x5d, 0x80, 0xf1, 0xc5, 0xdd, 0xa6, 0xed, 0xb6, 0x1c, + 0xc2, 0xaa, 0x9e, 0x80, 0x61, 0xec, 0x7e, 0x18, 0xda, 0xd6, 0xac, 0x9a, 0x49, 0x1c, 0x71, 0x6e, + 0x07, 0x73, 0x7b, 0x83, 0x17, 0x63, 0x1f, 0x8e, 0x5e, 0x01, 0x70, 0xf5, 0x6d, 0x52, 0x6b, 0x31, + 0xee, 0xda, 0x97, 0x9f, 0xbb, 0x46, 0xc6, 0x58, 0x0d, 0x50, 0x0a, 0x99, 0x28, 0xf8, 0x8d, 0x25, + 0x72, 0xea, 0x9f, 0x28, 0x30, 0x15, 0x69, 0x77, 0x02, 0xf6, 0x9e, 0xad, 0xa8, 0xbd, 0x67, 0xbe, + 0xe7, 0xb1, 0x66, 0x98, 0x79, 0x3e, 0x56, 0x80, 0x73, 0x19, 0x73, 0x92, 0xf0, 0xfd, 0x56, 0x4e, + 0xc8, 0xf7, 0xbb, 0x05, 0xa3, 0x9e, 0x6d, 0x8a, 0x87, 0x9f, 0xfe, 0x0c, 0xe4, 0x3a, 0xc3, 0xd7, + 0x03, 0x34, 0xa1, 0x67, 0x77, 0x58, 0xe6, 0x62, 0x99, 0x8e, 0xfa, 0x9b, 0x0a, 0x8c, 0x04, 0x66, + 0xe5, 0xd7, 0xd4, 0x8d, 0x7c, 0xf7, 0x31, 0x95, 0xd4, 0xdf, 0x2b, 0xc0, 0xd9, 0x00, 0xb7, 0xcf, + 0xe6, 0xaa, 0x1e, 0xe5, 0x1b, 0x07, 0xdb, 0xa6, 0x2e, 0x46, 0x5e, 0xa5, 0x0c, 0x27, 0x1f, 0x69, + 0x36, 0x5b, 0x4e, 0xd3, 0x76, 0x7d, 0x41, 0x9a, 0x6b, 0x1c, 0xbc, 0x08, 0xfb, 0x30, 0xb4, 0x0a, + 0x03, 0x2e, 0xa5, 0x27, 0x0e, 0xcb, 0x43, 0xce, 0x06, 0xd3, 0x05, 0x58, 0x7f, 0x31, 0x47, 0x83, + 0x5e, 0x91, 0x79, 0xf8, 0x40, 0x7e, 0xeb, 0x27, 0x1d, 0x49, 0x2d, 0x10, 0xa5, 0xc3, 0xbb, 0x93, + 0x4e, 0xd6, 0x46, 0x75, 0x19, 0x26, 0x85, 0x83, 0x33, 0xdf, 0x36, 0x96, 0x4e, 0xd0, 0xbb, 0x22, + 0x3b, 0xe3, 0x4d, 0x31, 0x9f, 0x9c, 0xe9, 0x78, 0xfd, 0x70, 0xc7, 0xa8, 0x2e, 0x0c, 0x5f, 0x17, + 0x9d, 0x44, 0xb3, 0x50, 0x30, 0xfc, 0xb5, 0x00, 0x81, 0xa3, 0x50, 0x29, 0xe3, 0x82, 0xd1, 0xc5, + 0xeb, 0x20, 0xf9, 0x58, 0xea, 0xeb, 0x7c, 0x2c, 0xa9, 0x5f, 0x2b, 0xc0, 0xb4, 0x4f, 0xd5, 0x1f, + 0x63, 0x59, 0x5c, 0xb7, 0x1f, 0xa0, 0x55, 0x1d, 0x6c, 0xab, 0xbc, 0x05, 0xfd, 0x8c, 0x01, 0xe6, + 0xba, 0x86, 0x0f, 0x10, 0xd2, 0xee, 0x60, 0x86, 0x08, 0xbd, 0x1f, 0x06, 0x4d, 0xaa, 0xa2, 0xf8, + 0xcf, 0x76, 0x72, 0x59, 0x76, 0xd3, 0x86, 0xcb, 0x35, 0x1f, 0x11, 0xce, 0x2e, 0xb8, 0x49, 0xe5, + 0x85, 0x58, 0xd0, 0x9c, 0x7d, 0x14, 0x46, 0xa5, 0x6a, 0x87, 0x8a, 0x65, 0xf7, 0xd9, 0x02, 0xcc, + 0xdc, 0x20, 0x66, 0x23, 0xd5, 0x77, 0xa2, 0x08, 0x03, 0xfa, 0xb6, 0xe6, 0xf0, 0x30, 0x89, 0x63, + 0x7c, 0x93, 0x97, 0x68, 0x01, 0xe6, 0xe5, 0x54, 0x63, 0x89, 0x3c, 0x54, 0x7c, 0x42, 0x9a, 0xc9, + 0x30, 0x7e, 0xe6, 0xfb, 0x82, 0x00, 0x9b, 0xe1, 0xc0, 0x23, 0x15, 0xe8, 0xf1, 0xf2, 0x9e, 0xea, + 0xad, 0xd5, 0xb4, 0x27, 0x8a, 0xe8, 0x2e, 0x8c, 0xdb, 0xba, 0x81, 0x49, 0xd3, 0x76, 0x0d, 0xcf, + 0x76, 0xf6, 0xc4, 0xa2, 0xe5, 0x3a, 0x5a, 0x6e, 0x95, 0x2a, 0x21, 0x22, 0x7e, 0xff, 0x18, 0x29, + 0xc2, 0x51, 0x52, 0xea, 0x17, 0x14, 0x18, 0xbd, 0x61, 0x6c, 0x12, 0x87, 0xfb, 0x70, 0x33, 0x13, + 0x4b, 0x24, 0xe0, 0xdf, 0x68, 0x5a, 0xb0, 0x3f, 0xb4, 0x0b, 0x23, 0xe2, 0x1c, 0x0e, 0x5e, 0xa0, + 0x5e, 0xcf, 0xe7, 0x70, 0x14, 0x90, 0x16, 0xe7, 0x9b, 0xf4, 0xe1, 0xfb, 0x25, 0x2e, 0x0e, 0x89, + 0xa9, 0xaf, 0xc0, 0xe9, 0x94, 0x46, 0x74, 0x21, 0x99, 0x1b, 0xb3, 0xf8, 0x68, 0x7c, 0x6e, 0x45, + 0x17, 0x92, 0x95, 0xa3, 0xf3, 0xd0, 0x47, 0xac, 0x9a, 0xf8, 0x62, 0x86, 0xf6, 0xdb, 0xc5, 0xbe, + 0x45, 0xab, 0x86, 0x69, 0x19, 0x65, 0xe2, 0xa6, 0x1d, 0x91, 0xd8, 0x18, 0x13, 0x5f, 0x16, 0x65, + 0x38, 0x80, 0x32, 0x17, 0xb1, 0xb8, 0x37, 0x14, 0x55, 0x4d, 0x26, 0xb7, 0x62, 0xbc, 0xa5, 0x17, + 0x27, 0xac, 0x38, 0x9f, 0x5a, 0x98, 0x11, 0x13, 0x92, 0xe0, 0x78, 0x38, 0x41, 0x57, 0xfd, 0x95, + 0x7e, 0xb8, 0x74, 0xc3, 0x76, 0x8c, 0xbb, 0xb6, 0xe5, 0x69, 0xe6, 0x9a, 0x5d, 0x0b, 0x1d, 0x64, + 0xc5, 0x91, 0xf5, 0x51, 0x05, 0xce, 0xe9, 0xcd, 0x96, 0xf0, 0x57, 0x16, 0x3e, 0xa6, 0x6b, 0xc4, + 0x31, 0xec, 0xbc, 0x8f, 0x76, 0x98, 0x6b, 0x77, 0x69, 0x6d, 0x23, 0x0d, 0x25, 0xce, 0xa2, 0xc5, + 0xde, 0x0e, 0xd5, 0xec, 0x3b, 0x16, 0xeb, 0x5c, 0xd5, 0x63, 0xb3, 0x79, 0x37, 0x5c, 0x84, 0x9c, + 0x6f, 0x87, 0xca, 0xa9, 0x18, 0x71, 0x06, 0x25, 0xf4, 0x41, 0x38, 0x23, 0xfc, 0xb2, 0x31, 0xd1, + 0x6a, 0x86, 0x45, 0x5c, 0x97, 0x3f, 0x3c, 0xe8, 0xe1, 0x71, 0x4c, 0x25, 0x0d, 0x21, 0x4e, 0xa7, + 0x83, 0x5e, 0x04, 0x70, 0xf7, 0x2c, 0x5d, 0xcc, 0x7f, 0x3e, 0x4f, 0x56, 0x2e, 0x22, 0x07, 0x58, + 0xb0, 0x84, 0x91, 0x2a, 0x5a, 0x5e, 0xb0, 0x29, 0x07, 0x99, 0x37, 0x32, 0x53, 0xb4, 0xc2, 0x3d, + 0x14, 0xc2, 0xd5, 0x4f, 0x2a, 0x30, 0x51, 0xb1, 0xd6, 0x4c, 0x4d, 0x27, 0xdc, 0x31, 0xd3, 0x45, + 0xd7, 0x60, 0xc4, 0x0d, 0xae, 0x74, 0x38, 0x47, 0x08, 0xbf, 0xcf, 0xe0, 0x32, 0x27, 0xac, 0xc3, + 0x3c, 0xf6, 0x0d, 0x4b, 0x1c, 0x76, 0x4b, 0xb6, 0xc3, 0x11, 0x89, 0xef, 0x8e, 0x7b, 0xec, 0x27, + 0xc1, 0x38, 0xad, 0x8d, 0xfa, 0xf3, 0x0a, 0x4c, 0x47, 0xbb, 0x23, 0x7c, 0x22, 0x7e, 0x4c, 0x81, + 0xe9, 0xc8, 0xc3, 0x7a, 0x01, 0xee, 0x25, 0xd6, 0xd7, 0x5a, 0x0a, 0x3e, 0xee, 0x23, 0x9c, 0x06, + 0xc1, 0xa9, 0xf4, 0xd5, 0x7f, 0xa1, 0xc0, 0x90, 0x88, 0xfb, 0x89, 0xde, 0x12, 0x33, 0xbf, 0x07, + 0x47, 0x5b, 0xcc, 0x04, 0xbf, 0xc7, 0x1c, 0x3c, 0xc4, 0xd1, 0x24, 0x4e, 0x99, 0x5c, 0xf6, 0x5b, + 0x41, 0x38, 0x3c, 0xe7, 0x22, 0x8e, 0x1e, 0xfe, 0xdd, 0x8e, 0x44, 0x4c, 0x7d, 0x55, 0x81, 0xa9, + 0x44, 0xab, 0x2e, 0xc4, 0xd1, 0x13, 0x74, 0x79, 0xfd, 0x72, 0x3f, 0xdd, 0x92, 0x1e, 0xe5, 0xf7, + 0x26, 0xb7, 0x8c, 0x9f, 0x80, 0xfe, 0xfb, 0x00, 0x8c, 0x18, 0x8d, 0x46, 0xcb, 0x63, 0xb7, 0x90, + 0x03, 0xa1, 0xb9, 0xb9, 0xe2, 0x17, 0xe2, 0x10, 0x8e, 0x2c, 0x21, 0x69, 0xf1, 0x53, 0x70, 0x39, + 0xdf, 0xca, 0xc9, 0x03, 0x9c, 0xa3, 0x52, 0x11, 0x17, 0x87, 0xd2, 0x04, 0xb1, 0xef, 0x51, 0x00, + 0x5c, 0xcf, 0x31, 0xac, 0x3a, 0x2d, 0x14, 0xd2, 0x18, 0x3e, 0x02, 0xb2, 0xd5, 0x00, 0x29, 0x27, + 0x1e, 0xc6, 0x02, 0x0d, 0x00, 0x58, 0xa2, 0x8c, 0xe6, 0x85, 0x10, 0xca, 0x8f, 0xcc, 0xb7, 0xc5, + 0xc4, 0xed, 0x4b, 0x29, 0xae, 0xb4, 0x9c, 0x50, 0x28, 0xa5, 0xce, 0x3e, 0x02, 0x23, 0x01, 0xbd, + 0x83, 0x84, 0xba, 0x31, 0x49, 0xa8, 0x9b, 0x7d, 0x1c, 0x4e, 0xc5, 0xba, 0x7b, 0x28, 0x99, 0xf0, + 0x4f, 0x15, 0x40, 0xd1, 0xd1, 0x9f, 0x80, 0xe5, 0xa0, 0x1e, 0xb5, 0x1c, 0x2c, 0xf4, 0xbe, 0x64, + 0x19, 0xa6, 0x83, 0x9f, 0x45, 0xc0, 0xc2, 0x22, 0x07, 0x61, 0xc2, 0xc5, 0xc9, 0x4f, 0x05, 0x95, + 0x30, 0x1a, 0x80, 0xf8, 0x72, 0x7b, 0x10, 0x54, 0x6e, 0xc6, 0x70, 0x85, 0x82, 0x4a, 0x1c, 0x82, + 0x13, 0x74, 0xd1, 0xc7, 0x15, 0x98, 0xd4, 0xa2, 0x61, 0x91, 0xfd, 0x99, 0xc9, 0x15, 0xac, 0x2d, + 0x16, 0x62, 0x39, 0xec, 0x4b, 0x0c, 0xe0, 0xe2, 0x04, 0x59, 0xf4, 0x30, 0x8c, 0x69, 0x4d, 0x63, + 0xbe, 0x55, 0x33, 0xa8, 0xe6, 0xe9, 0x47, 0x8f, 0x65, 0xd6, 0x90, 0xf9, 0xb5, 0x4a, 0x50, 0x8e, + 0x23, 0xb5, 0x82, 0xf8, 0xc3, 0xa5, 0xd0, 0xf3, 0xb8, 0x97, 0xf8, 0xc3, 0x62, 0x0e, 0xc3, 0xf8, + 0xc3, 0x62, 0xea, 0x64, 0x22, 0xc8, 0x02, 0xb0, 0x8d, 0x9a, 0x2e, 0x48, 0x0e, 0xe6, 0xbf, 0x44, + 0xb9, 0x55, 0x29, 0x97, 0x04, 0x45, 0x26, 0x3e, 0x84, 0xbf, 0xb1, 0x44, 0x01, 0x7d, 0x4a, 0x81, + 0x71, 0xc1, 0xbb, 0x05, 0xcd, 0x21, 0xb6, 0x44, 0xcf, 0xe7, 0xdd, 0x2f, 0xb1, 0x3d, 0x39, 0x87, + 0x65, 0xe4, 0x9c, 0xef, 0x04, 0xc1, 0x24, 0x22, 0x30, 0x1c, 0xed, 0x07, 0x93, 0x01, 0xdc, 0xc8, + 0x2d, 0x96, 0xe8, 0xe0, 0x70, 0x7e, 0x19, 0xa0, 0x9a, 0x82, 0x4f, 0xbc, 0x52, 0x4a, 0x81, 0xe0, + 0x54, 0xfa, 0x54, 0xae, 0x3d, 0x75, 0x47, 0xf3, 0xf4, 0xed, 0x92, 0xa6, 0x6f, 0xb3, 0x4b, 0x4c, + 0xfe, 0xb4, 0x36, 0xe7, 0xbe, 0x7e, 0x36, 0x8a, 0x6a, 0xe1, 0xf4, 0x7e, 0xbb, 0x78, 0x2a, 0x56, + 0x88, 0xe3, 0x04, 0x91, 0x0d, 0xc3, 0x8e, 0xc8, 0x0d, 0x20, 0xa2, 0x21, 0xe4, 0x0b, 0x87, 0x1f, + 0x4f, 0x34, 0xc0, 0x35, 0x23, 0xff, 0x17, 0x0e, 0x88, 0xa0, 0x3a, 0x5c, 0xe2, 0xba, 0xe1, 0xbc, + 0x65, 0x5b, 0x7b, 0x0d, 0xbb, 0xe5, 0xce, 0xb7, 0xbc, 0x6d, 0x62, 0x79, 0xbe, 0x29, 0x7c, 0x94, + 0x1d, 0xa3, 0xec, 0xd5, 0xdd, 0x62, 0xa7, 0x8a, 0xb8, 0x33, 0x1e, 0xf4, 0x1c, 0x0c, 0xb3, 0x8b, + 0xbe, 0xf5, 0xf5, 0x65, 0xf6, 0x4a, 0xf7, 0xf0, 0xe2, 0x32, 0x1b, 0xc2, 0xa2, 0xc0, 0x81, 0x03, + 0x6c, 0xe8, 0x36, 0x0c, 0x99, 0x3c, 0xb9, 0x03, 0x7b, 0xad, 0x9b, 0x93, 0x29, 0xc6, 0x13, 0x45, + 0x70, 0x05, 0x5a, 0xfc, 0xc0, 0x3e, 0x05, 0xd4, 0x84, 0x2b, 0x35, 0xb2, 0xa5, 0xb5, 0x4c, 0x6f, + 0xd5, 0xf6, 0x30, 0x7b, 0xe2, 0x16, 0x58, 0x3c, 0xfd, 0x07, 0xd9, 0x13, 0x2c, 0x1e, 0x1f, 0x7b, + 0x3c, 0x58, 0x3e, 0xa0, 0x2e, 0x3e, 0x10, 0x1b, 0xda, 0x83, 0xfb, 0x44, 0x1d, 0xf6, 0xa6, 0x4e, + 0xdf, 0xa6, 0xb3, 0x9c, 0x24, 0x7a, 0x8a, 0x11, 0xfd, 0xa6, 0xfd, 0x76, 0xf1, 0xbe, 0xf2, 0xc1, + 0xd5, 0x71, 0x37, 0x38, 0xd9, 0x33, 0x25, 0x12, 0xbb, 0x02, 0x9a, 0x99, 0xcc, 0x3f, 0xc7, 0xf1, + 0xeb, 0x24, 0xee, 0x4f, 0x17, 0x2f, 0xc5, 0x09, 0x9a, 0xe8, 0xf3, 0x0a, 0xcc, 0xb8, 0x9e, 0xd3, + 0xd2, 0xbd, 0x96, 0x43, 0x6a, 0xb1, 0x1d, 0x3a, 0xc5, 0x3a, 0x94, 0x4b, 0x80, 0xab, 0x66, 0xe0, + 0x64, 0xa1, 0x01, 0x66, 0xb2, 0xa0, 0x38, 0xb3, 0x2f, 0xe8, 0x73, 0x0a, 0x9c, 0x8b, 0x02, 0xa9, + 0x4e, 0xcf, 0xfb, 0x89, 0xf2, 0x5f, 0xb2, 0x54, 0xd3, 0x51, 0x72, 0x0d, 0x3e, 0x03, 0x88, 0xb3, + 0x3a, 0x12, 0xbf, 0x5a, 0x3f, 0x7d, 0xc2, 0x57, 0xeb, 0xb3, 0x4f, 0x01, 0x4a, 0x1e, 0x1f, 0x07, + 0xc9, 0x81, 0xc3, 0xb2, 0x1c, 0xf8, 0x99, 0x01, 0xb8, 0x40, 0x4f, 0xa5, 0x50, 0xfb, 0x59, 0xd1, + 0x2c, 0xad, 0xfe, 0xda, 0x94, 0x98, 0xbe, 0xa0, 0xc0, 0xb9, 0xed, 0x74, 0xd3, 0x8e, 0xd0, 0xbf, + 0x9e, 0xce, 0x65, 0x82, 0xeb, 0x64, 0x2d, 0xe2, 0x0c, 0xbb, 0x63, 0x15, 0x9c, 0xd5, 0x29, 0xf4, + 0x14, 0x4c, 0x5a, 0x76, 0x8d, 0x94, 0x2a, 0x65, 0xbc, 0xa2, 0xb9, 0xb7, 0xab, 0xbe, 0xa7, 0xcf, + 0x00, 0xff, 0x5e, 0x57, 0x63, 0x30, 0x9c, 0xa8, 0x8d, 0x76, 0x00, 0x35, 0xed, 0xda, 0xe2, 0x0e, + 0x4f, 0x82, 0xd2, 0x9b, 0xcf, 0x2d, 0xf3, 0x36, 0x58, 0x4b, 0x60, 0xc3, 0x29, 0x14, 0x98, 0x6d, + 0x8a, 0x76, 0x66, 0xc5, 0xb6, 0x0c, 0xcf, 0x76, 0x58, 0xb0, 0x8b, 0x9e, 0x4c, 0x34, 0xcc, 0x36, + 0xb5, 0x9a, 0x8a, 0x11, 0x67, 0x50, 0x52, 0xff, 0xa7, 0x02, 0xa7, 0xe8, 0xb6, 0x58, 0x73, 0xec, + 0xdd, 0xbd, 0xd7, 0xe2, 0x86, 0xbc, 0x5f, 0x38, 0x3d, 0x72, 0xdb, 0xce, 0x19, 0xc9, 0xe1, 0x71, + 0x84, 0xf5, 0x39, 0xf4, 0x71, 0x94, 0xcd, 0xca, 0x7d, 0xd9, 0x66, 0x65, 0xf5, 0x53, 0x05, 0xae, + 0xb9, 0xf8, 0x66, 0xdd, 0xd7, 0xe4, 0x77, 0xf8, 0x08, 0x8c, 0xd3, 0xb2, 0x15, 0x6d, 0x77, 0xad, + 0xfc, 0x8c, 0x6d, 0xfa, 0xcf, 0xb9, 0x99, 0xad, 0xfd, 0xa6, 0x0c, 0xc0, 0xd1, 0x7a, 0xe8, 0x31, + 0x18, 0x6a, 0xf2, 0x98, 0x71, 0x42, 0x67, 0xbe, 0xc2, 0x3d, 0x03, 0x59, 0xd1, 0xbd, 0x76, 0x71, + 0x2a, 0xbc, 0xe2, 0xf5, 0x23, 0xd7, 0xf9, 0x0d, 0xd4, 0x5f, 0x39, 0x0b, 0x0c, 0xb9, 0x49, 0xbc, + 0xd7, 0xe2, 0x9c, 0x3c, 0x08, 0xa3, 0x7a, 0xb3, 0x55, 0x5a, 0xaa, 0x3e, 0xdd, 0xb2, 0x99, 0x2d, + 0x84, 0xe5, 0x0a, 0xa2, 0xdc, 0xbb, 0xb4, 0xb6, 0xe1, 0x17, 0x63, 0xb9, 0x0e, 0xe5, 0x0e, 0x7a, + 0xb3, 0x25, 0xf8, 0xed, 0x9a, 0xfc, 0xe0, 0x85, 0x71, 0x87, 0xd2, 0xda, 0x46, 0x04, 0x86, 0x13, + 0xb5, 0xd1, 0x07, 0x61, 0x8c, 0x88, 0x0f, 0xf7, 0x86, 0xe6, 0xd4, 0x04, 0x5f, 0xa8, 0xe4, 0x1d, + 0x7c, 0x30, 0xb5, 0x3e, 0x37, 0xe0, 0x1a, 0xe0, 0xa2, 0x44, 0x02, 0x47, 0x08, 0xa2, 0x17, 0xe0, + 0xbc, 0xff, 0x9b, 0xae, 0xb2, 0x5d, 0x8b, 0x33, 0x8a, 0x01, 0xee, 0x77, 0xb3, 0x98, 0x55, 0x09, + 0x67, 0xb7, 0x47, 0x3f, 0xa7, 0xc0, 0xd9, 0x00, 0x6a, 0x58, 0x46, 0xa3, 0xd5, 0xc0, 0x44, 0x37, + 0x35, 0xa3, 0x21, 0xf4, 0xbe, 0x67, 0x8f, 0x6c, 0xa0, 0x51, 0xf4, 0x9c, 0x59, 0xa5, 0xc3, 0x70, + 0x46, 0x97, 0xd0, 0xab, 0x0a, 0x5c, 0xf1, 0x41, 0x6b, 0x0e, 0x71, 0xdd, 0x96, 0x43, 0xc2, 0x60, + 0x02, 0x62, 0x4a, 0x86, 0x72, 0xf1, 0x4e, 0x26, 0x00, 0x2f, 0x1e, 0x80, 0x1b, 0x1f, 0x48, 0x5d, + 0xde, 0x2e, 0x55, 0x7b, 0xcb, 0x13, 0x8a, 0xe2, 0x71, 0x6d, 0x17, 0x4a, 0x02, 0x47, 0x08, 0xa2, + 0x9f, 0x57, 0xe0, 0x9c, 0x5c, 0x20, 0xef, 0x16, 0xae, 0x21, 0x3e, 0x77, 0x64, 0x9d, 0x89, 0xe1, + 0xe7, 0x12, 0x5e, 0x06, 0x10, 0x67, 0xf5, 0x8a, 0xb2, 0xed, 0x06, 0xdb, 0x98, 0x5c, 0x8b, 0x1c, + 0xe0, 0x6c, 0x9b, 0xef, 0x55, 0x17, 0xfb, 0x30, 0xf4, 0x30, 0x8c, 0x35, 0xed, 0xda, 0x9a, 0x51, + 0x73, 0x59, 0x3c, 0x3b, 0xa6, 0xeb, 0xf5, 0xf1, 0xe9, 0x58, 0xb3, 0x6b, 0x6b, 0x95, 0x32, 0x2f, + 0xc7, 0x91, 0x5a, 0x68, 0x0e, 0x60, 0x4b, 0x33, 0xcc, 0xea, 0x1d, 0xad, 0x79, 0xcb, 0x0f, 0x90, + 0xc4, 0x6c, 0x11, 0x4b, 0x41, 0x29, 0x96, 0x6a, 0xd0, 0xf5, 0xa3, 0x7c, 0x07, 0x13, 0x1e, 0x82, + 0x9c, 0xa9, 0x47, 0x47, 0xb1, 0x7e, 0x3e, 0x42, 0xde, 0xe1, 0x9b, 0x12, 0x09, 0x1c, 0x21, 0x88, + 0x3e, 0xaa, 0xc0, 0x84, 0xbb, 0xe7, 0x7a, 0xa4, 0x11, 0xf4, 0xe1, 0xd4, 0x51, 0xf7, 0x81, 0xd9, + 0xc4, 0xab, 0x11, 0x22, 0x38, 0x46, 0x94, 0x85, 0x9a, 0x6a, 0x68, 0x75, 0x72, 0xbd, 0x74, 0xc3, + 0xa8, 0x6f, 0x07, 0xe1, 0x61, 0xd6, 0x88, 0xa3, 0x13, 0xcb, 0x63, 0x8a, 0xd5, 0x80, 0x08, 0x35, + 0x95, 0x5d, 0x0d, 0x77, 0xc2, 0x81, 0x5e, 0x84, 0x59, 0x01, 0x5e, 0xb6, 0xef, 0x24, 0x28, 0xf0, + 0x88, 0x4e, 0xcc, 0x83, 0xb2, 0x92, 0x59, 0x0b, 0x77, 0xc0, 0x80, 0x2a, 0x70, 0xda, 0x25, 0x0e, + 0xbb, 0x13, 0xe4, 0xd1, 0x41, 0xd7, 0x5a, 0xa6, 0xe9, 0x32, 0xd5, 0x46, 0xbc, 0x19, 0xaa, 0x26, + 0xc1, 0x38, 0xad, 0x0d, 0x7a, 0x3c, 0x78, 0x57, 0xbc, 0x47, 0x0b, 0x9e, 0x5e, 0xab, 0x32, 0x4d, + 0x64, 0x80, 0x5b, 0x4a, 0x70, 0x14, 0x84, 0xe3, 0x75, 0xe9, 0x69, 0xee, 0x17, 0x2d, 0xb4, 0x1c, + 0xd7, 0x63, 0x71, 0x96, 0x06, 0xf8, 0x69, 0x8e, 0x65, 0x00, 0x8e, 0xd6, 0x43, 0x8f, 0xc1, 0x84, + 0x4b, 0x74, 0xdd, 0x6e, 0x34, 0x85, 0x9e, 0xcc, 0x82, 0x21, 0x0d, 0x8b, 0x15, 0x8c, 0x40, 0x70, + 0xac, 0x26, 0xda, 0x83, 0xd3, 0x41, 0x38, 0xe5, 0x65, 0xbb, 0xbe, 0xa2, 0xed, 0x32, 0xe1, 0xf8, + 0x6c, 0x9e, 0x18, 0x11, 0x7c, 0xba, 0x4a, 0x49, 0x74, 0x38, 0x8d, 0x06, 0x5a, 0x86, 0xe9, 0x58, + 0xf1, 0x92, 0x61, 0x12, 0x77, 0xe6, 0x1c, 0x1b, 0x36, 0x33, 0x76, 0x95, 0x52, 0xe0, 0x38, 0xb5, + 0x15, 0xba, 0x05, 0x67, 0x9a, 0x8e, 0xed, 0x11, 0xdd, 0xbb, 0x49, 0x05, 0x02, 0x53, 0x0c, 0xd0, + 0x9d, 0x99, 0x61, 0x73, 0xc1, 0xee, 0x43, 0xd7, 0xd2, 0x2a, 0xe0, 0xf4, 0x76, 0xe8, 0x33, 0x0a, + 0x5c, 0x76, 0x3d, 0x87, 0x68, 0x0d, 0xc3, 0xaa, 0x97, 0x6c, 0xcb, 0x22, 0x8c, 0x31, 0x55, 0x6a, + 0xe1, 0x93, 0xbb, 0xf3, 0xb9, 0x4e, 0x11, 0x75, 0xbf, 0x5d, 0xbc, 0x5c, 0xed, 0x88, 0x19, 0x1f, + 0x40, 0x19, 0xbd, 0x02, 0xd0, 0x20, 0x0d, 0xdb, 0xd9, 0xa3, 0x1c, 0x69, 0x66, 0x36, 0xbf, 0xbe, + 0xbb, 0x12, 0x60, 0xe1, 0x9f, 0x7f, 0xe4, 0x26, 0x37, 0x04, 0x62, 0x89, 0x1c, 0x9d, 0x6a, 0xca, + 0x6f, 0x45, 0x44, 0x33, 0xe9, 0xa3, 0xb9, 0xc0, 0x56, 0x8e, 0x4d, 0xf5, 0x4a, 0x5a, 0x05, 0x9c, + 0xde, 0x0e, 0xb9, 0x30, 0xc5, 0xbe, 0x50, 0x71, 0x92, 0x5f, 0x2f, 0xcd, 0xd7, 0xc9, 0xcc, 0xc5, + 0x5c, 0x93, 0x4b, 0x65, 0xff, 0xa9, 0x4a, 0x1c, 0x19, 0x4e, 0xe2, 0x0f, 0x89, 0x6a, 0xbb, 0x21, + 0xd1, 0x4b, 0xbd, 0x12, 0x95, 0x90, 0xe1, 0x24, 0x7e, 0xb5, 0x5d, 0x80, 0x33, 0xa9, 0xa7, 0x24, + 0x65, 0x1e, 0x7c, 0x8a, 0xe7, 0xfd, 0x5c, 0x70, 0xe2, 0xda, 0x93, 0x31, 0x8f, 0x95, 0x28, 0x08, + 0xc7, 0xeb, 0x52, 0x19, 0x96, 0x51, 0x5b, 0xaa, 0x86, 0xed, 0x0b, 0xa1, 0x0c, 0x5b, 0x89, 0xc1, + 0x70, 0xa2, 0x36, 0x2a, 0x89, 0xf9, 0x58, 0xaa, 0x56, 0xa8, 0x1a, 0xe8, 0x2e, 0x39, 0xc4, 0xd7, + 0x0e, 0xc2, 0xf1, 0xc9, 0x40, 0x9c, 0xac, 0x4f, 0x47, 0x41, 0x7f, 0xc8, 0xbd, 0xe8, 0x0f, 0x47, + 0xb1, 0x1a, 0x05, 0xe1, 0x78, 0x5d, 0x5f, 0x4f, 0x8f, 0x74, 0x61, 0x20, 0x1c, 0xc5, 0x6a, 0x0c, + 0x86, 0x13, 0xb5, 0xd5, 0x3f, 0xeb, 0x87, 0xfb, 0xba, 0x90, 0x2c, 0x51, 0x23, 0x7d, 0xba, 0x0f, + 0xcf, 0xf3, 0xba, 0x5b, 0x9e, 0x66, 0xc6, 0xf2, 0x1c, 0x9e, 0x5e, 0xb7, 0xcb, 0xe9, 0x66, 0x2d, + 0xe7, 0xe1, 0x49, 0x76, 0xbf, 0xfc, 0x8d, 0xf4, 0xe5, 0xcf, 0x39, 0xab, 0x07, 0x6e, 0x97, 0x66, + 0xc6, 0x76, 0xc9, 0x39, 0xab, 0x5d, 0x6c, 0xaf, 0x3f, 0xef, 0x87, 0x37, 0x75, 0x23, 0xe5, 0xe6, + 0xdc, 0x5f, 0x29, 0xbc, 0xe5, 0x58, 0xf7, 0x57, 0xd6, 0x83, 0xf0, 0x63, 0xdc, 0x5f, 0x1d, 0xd9, + 0xe7, 0xf1, 0xec, 0xaf, 0xac, 0x59, 0x3d, 0xae, 0xfd, 0x95, 0x35, 0xab, 0x5d, 0xec, 0xaf, 0xbf, + 0x89, 0x9f, 0x0f, 0x81, 0xa8, 0x5d, 0x81, 0x3e, 0xbd, 0xd9, 0xca, 0xc9, 0xa4, 0x98, 0x97, 0x61, + 0x69, 0x6d, 0x03, 0x53, 0x1c, 0x08, 0xc3, 0x20, 0xdf, 0x3f, 0x39, 0x59, 0x10, 0xf3, 0x1c, 0xe5, + 0x5b, 0x12, 0x0b, 0x4c, 0x74, 0xaa, 0x48, 0x73, 0x9b, 0x34, 0x88, 0xa3, 0x99, 0x55, 0xcf, 0x76, + 0xb4, 0x7a, 0x5e, 0x6e, 0xc3, 0x6f, 0x50, 0x62, 0xb8, 0x70, 0x02, 0x3b, 0x9d, 0x90, 0xa6, 0x51, + 0xcb, 0xc9, 0x5f, 0xd8, 0x84, 0xac, 0x55, 0xca, 0x98, 0xe2, 0x50, 0x7f, 0x7d, 0x18, 0xa4, 0x44, + 0x05, 0xe8, 0x13, 0x0a, 0x4c, 0xe9, 0xf1, 0xa0, 0x9e, 0xbd, 0xf8, 0x43, 0x25, 0x22, 0x84, 0xf2, + 0x2d, 0x9f, 0x28, 0xc6, 0x49, 0xb2, 0xe8, 0x43, 0x0a, 0x37, 0xf2, 0x05, 0xb7, 0x79, 0x62, 0x5a, + 0xaf, 0x1f, 0xd1, 0xbd, 0x77, 0x68, 0x2d, 0x0c, 0xaf, 0x58, 0xa3, 0x04, 0xd1, 0xab, 0x0a, 0x9c, + 0xb9, 0x9d, 0x76, 0x37, 0x21, 0x26, 0xff, 0x56, 0xde, 0xae, 0x64, 0x5c, 0x76, 0x70, 0x09, 0x32, + 0xb5, 0x02, 0x4e, 0xef, 0x48, 0x30, 0x4b, 0x81, 0xb9, 0x56, 0x7c, 0xa7, 0xb9, 0x67, 0x29, 0x66, + 0xf7, 0x0d, 0x67, 0x29, 0x00, 0xe0, 0x28, 0x41, 0xd4, 0x84, 0x91, 0xdb, 0xbe, 0x8d, 0x5c, 0xd8, + 0xc5, 0x4a, 0x79, 0xa9, 0x4b, 0x86, 0x76, 0xee, 0xef, 0x15, 0x14, 0xe2, 0x90, 0x08, 0xda, 0x86, + 0xa1, 0xdb, 0x9c, 0x57, 0x08, 0x7b, 0xd6, 0x7c, 0xcf, 0xda, 0x3f, 0x37, 0xab, 0x88, 0x22, 0xec, + 0xa3, 0x97, 0xdf, 0x12, 0x0c, 0x1f, 0xf0, 0xc4, 0xed, 0x33, 0x0a, 0x9c, 0xd9, 0x21, 0x8e, 0x67, + 0xe8, 0xf1, 0x9b, 0xa1, 0x91, 0xfc, 0x16, 0x8a, 0x67, 0xd2, 0x10, 0xf2, 0x6d, 0x92, 0x0a, 0xc2, + 0xe9, 0x5d, 0x40, 0xcf, 0x40, 0x3f, 0xf1, 0xf4, 0x9a, 0x08, 0xe6, 0xfd, 0xae, 0xbc, 0x6f, 0x7e, + 0xf9, 0xbb, 0x17, 0xfa, 0x1f, 0x66, 0xf8, 0xd4, 0xbf, 0x52, 0x20, 0x61, 0x9b, 0x46, 0x3f, 0xa0, + 0xc0, 0xd8, 0x16, 0xd1, 0xbc, 0x96, 0x43, 0xae, 0x0b, 0x9f, 0xd0, 0xbe, 0xab, 0xa3, 0x0f, 0x3d, + 0x73, 0x14, 0x26, 0xf1, 0xb9, 0x25, 0x09, 0x31, 0x77, 0x56, 0x09, 0x92, 0x8f, 0xc8, 0x20, 0x1c, + 0xe9, 0xc1, 0xec, 0x93, 0x30, 0x95, 0x68, 0x78, 0xa8, 0x6b, 0xca, 0x5f, 0x53, 0x20, 0x2d, 0xcf, + 0x3d, 0x7a, 0x11, 0x06, 0xb4, 0x5a, 0x2d, 0x48, 0x77, 0xfa, 0x68, 0x3e, 0xbf, 0xa9, 0x9a, 0x1c, + 0x6a, 0x88, 0xfd, 0xc4, 0x1c, 0x2d, 0x5a, 0x02, 0xa4, 0x45, 0xee, 0xa5, 0x57, 0xc2, 0x18, 0x17, + 0xec, 0x3a, 0x6d, 0x3e, 0x01, 0xc5, 0x29, 0x2d, 0xd4, 0x8f, 0x29, 0x80, 0x92, 0xe9, 0x6a, 0x90, + 0x03, 0xc3, 0x62, 0xff, 0xfa, 0xab, 0x54, 0xce, 0xf9, 0x9a, 0x2e, 0xf2, 0x34, 0x34, 0x74, 0xc2, + 0x13, 0x05, 0x2e, 0x0e, 0xe8, 0xa8, 0x7f, 0xa7, 0x40, 0x98, 0x12, 0x11, 0xbd, 0x03, 0x46, 0x6b, + 0xc4, 0xd5, 0x1d, 0xa3, 0xe9, 0x85, 0x0f, 0x49, 0x83, 0x07, 0x69, 0xe5, 0x10, 0x84, 0xe5, 0x7a, + 0x48, 0x85, 0x41, 0x4f, 0x73, 0x6f, 0x57, 0xca, 0x42, 0xd9, 0x63, 0x47, 0xf3, 0x3a, 0x2b, 0xc1, + 0x02, 0x12, 0xc6, 0x9e, 0xed, 0xeb, 0x22, 0xf6, 0x2c, 0xda, 0x3a, 0x82, 0x40, 0xbb, 0xe8, 0xe0, + 0x20, 0xbb, 0xea, 0xcf, 0x14, 0xe0, 0x14, 0xad, 0xb2, 0xa2, 0x19, 0x96, 0x47, 0x2c, 0xf6, 0x6c, + 0x2a, 0xe7, 0x24, 0xd4, 0x61, 0xdc, 0x8b, 0xbc, 0x7a, 0x3e, 0xfc, 0xa3, 0xda, 0xc0, 0xd3, 0x2b, + 0xfa, 0xd6, 0x39, 0x8a, 0x17, 0x3d, 0xea, 0xbf, 0x5b, 0xe3, 0x6a, 0xf1, 0x7d, 0xfe, 0x56, 0x65, + 0x8f, 0xd1, 0xee, 0x89, 0x27, 0xe4, 0x41, 0x1e, 0xcd, 0xc8, 0x13, 0xb5, 0x47, 0x60, 0x5c, 0xbc, + 0x90, 0xe0, 0x41, 0x84, 0x85, 0x5a, 0xcc, 0x8e, 0x95, 0x25, 0x19, 0x80, 0xa3, 0xf5, 0xd4, 0x3f, + 0x2c, 0x40, 0x34, 0x5b, 0x67, 0xde, 0x59, 0x4a, 0x46, 0x50, 0x2e, 0x1c, 0x5b, 0x04, 0x65, 0x9e, + 0xaf, 0x9e, 0xb9, 0x73, 0x8b, 0x7b, 0x76, 0x39, 0x57, 0x35, 0x2b, 0xc7, 0x41, 0x8d, 0x70, 0x5a, + 0xfb, 0x0f, 0x3d, 0xad, 0xef, 0x10, 0x9e, 0xbf, 0x03, 0x91, 0x38, 0xd6, 0xbe, 0xe7, 0xef, 0x54, + 0xa4, 0xa1, 0xf4, 0xca, 0x6e, 0x1e, 0x44, 0xbe, 0x1b, 0xba, 0x2e, 0x22, 0xc6, 0xb6, 0xbb, 0x6e, + 0x7b, 0x9a, 0xc9, 0xa6, 0x53, 0x18, 0x5d, 0x57, 0x64, 0x00, 0x8e, 0xd6, 0x53, 0x57, 0xe1, 0x8d, + 0xcb, 0xb6, 0x56, 0x5b, 0xd0, 0x4c, 0xba, 0x75, 0x1d, 0xe1, 0x96, 0xe7, 0xb2, 0x93, 0x79, 0xcd, + 0xb1, 0x3d, 0x5b, 0xb7, 0x4d, 0x7a, 0x6e, 0x6a, 0x41, 0xb0, 0x61, 0x76, 0x11, 0xe9, 0x9f, 0x9b, + 0x22, 0x58, 0x2f, 0xf6, 0xe1, 0xea, 0xef, 0x28, 0x30, 0x24, 0x72, 0xe0, 0x74, 0xf1, 0xb0, 0x74, + 0x0b, 0x06, 0x98, 0x76, 0xd4, 0x8b, 0x54, 0x5a, 0xdd, 0xb6, 0x6d, 0x2f, 0x92, 0x3c, 0x8b, 0xa7, + 0x9f, 0x63, 0x09, 0x43, 0x39, 0x7a, 0xe6, 0x8f, 0xea, 0xe8, 0xdb, 0x86, 0x47, 0x98, 0xdb, 0x8d, + 0xd8, 0xf8, 0xdc, 0x1f, 0x55, 0x2a, 0xc7, 0x91, 0x5a, 0xea, 0xff, 0x18, 0x80, 0x2b, 0x7e, 0x3e, + 0x9f, 0xb8, 0xa8, 0x16, 0xf0, 0xdc, 0xbd, 0x20, 0xc1, 0x10, 0x4b, 0x90, 0xe0, 0xdb, 0x54, 0xf3, + 0x69, 0xc9, 0xe7, 0xa4, 0x14, 0x42, 0x32, 0x3a, 0x9c, 0x46, 0x83, 0x87, 0x6f, 0x67, 0xc5, 0x37, + 0x88, 0x66, 0x7a, 0xdb, 0x3e, 0xed, 0x42, 0x2f, 0xe1, 0xdb, 0x93, 0xf8, 0x70, 0x2a, 0x15, 0xe6, + 0xd2, 0xe1, 0x27, 0x36, 0x72, 0x88, 0x26, 0xfb, 0x93, 0xf4, 0xf0, 0xdc, 0x68, 0x25, 0x15, 0x23, + 0xce, 0xa0, 0xc4, 0xcc, 0x8d, 0xda, 0x2e, 0xb3, 0x5e, 0x60, 0xe2, 0x39, 0x06, 0x4b, 0x82, 0x16, + 0xdc, 0x55, 0xac, 0x44, 0x41, 0x38, 0x5e, 0x17, 0x3d, 0x06, 0x13, 0xcc, 0x45, 0x26, 0x8c, 0x07, + 0x3a, 0x10, 0x46, 0x85, 0x5a, 0x8d, 0x40, 0x70, 0xac, 0x26, 0xfa, 0x2e, 0x05, 0xa6, 0x0d, 0xf9, + 0x31, 0x8d, 0x3f, 0xfa, 0xc1, 0x5e, 0xf2, 0x4a, 0xf0, 0x6d, 0x9c, 0x82, 0x16, 0xa7, 0x12, 0x43, + 0xcb, 0x30, 0x2d, 0x5e, 0x44, 0x47, 0xf7, 0x00, 0x0f, 0x16, 0xc7, 0xd6, 0xb4, 0x9c, 0x02, 0xc7, + 0xa9, 0xad, 0xd4, 0x0f, 0x17, 0x60, 0xec, 0x90, 0x09, 0x6d, 0x5b, 0x92, 0xcc, 0xd1, 0xc3, 0xbb, + 0x45, 0x99, 0x6a, 0x17, 0x62, 0x07, 0x7a, 0x0e, 0x26, 0x5a, 0x6c, 0x22, 0xfc, 0x50, 0x6a, 0xe2, + 0x9b, 0xfe, 0x16, 0xba, 0x72, 0x1b, 0x11, 0xc8, 0xbd, 0x76, 0x71, 0x56, 0x46, 0x1f, 0x85, 0xe2, + 0x18, 0x1e, 0xf5, 0x37, 0x0a, 0x80, 0xe4, 0xea, 0x4b, 0xa6, 0xb6, 0x63, 0x3b, 0xe8, 0x73, 0x0a, + 0x8c, 0xe9, 0x72, 0x0a, 0x31, 0x2e, 0x60, 0x3d, 0xd7, 0xeb, 0x60, 0x39, 0xfa, 0x39, 0x39, 0x37, + 0x18, 0x17, 0x84, 0xfd, 0xe7, 0xd6, 0x63, 0x32, 0xe8, 0x5e, 0xec, 0x37, 0x8e, 0xf4, 0x69, 0xf6, + 0xa3, 0x0a, 0x4c, 0x25, 0x30, 0xa5, 0x48, 0xc6, 0xcf, 0x47, 0x83, 0xa8, 0x1f, 0x49, 0xb2, 0x50, + 0x59, 0xbe, 0xfe, 0xc9, 0x01, 0x38, 0x9d, 0xb2, 0xa2, 0xcc, 0xc5, 0x86, 0xc4, 0xa4, 0xcb, 0x5e, + 0x5c, 0x6c, 0x12, 0x92, 0x6a, 0xe0, 0x62, 0x13, 0x87, 0xe0, 0x04, 0x5d, 0xf4, 0x0c, 0xf4, 0xe9, + 0x8e, 0x21, 0x36, 0xed, 0x23, 0xb9, 0xa6, 0x00, 0x57, 0x16, 0x46, 0x05, 0xc5, 0xbe, 0x12, 0xae, + 0x60, 0x8a, 0x90, 0x9e, 0xc5, 0xf2, 0x31, 0xe2, 0x0b, 0xac, 0xec, 0x2c, 0x96, 0x4f, 0x1b, 0x17, + 0x47, 0xeb, 0xa1, 0xe7, 0x60, 0x46, 0x68, 0xaa, 0x7e, 0x24, 0x13, 0xdb, 0x72, 0x3d, 0xca, 0xf1, + 0x3d, 0x21, 0x53, 0x30, 0xef, 0xd7, 0x9b, 0x19, 0x75, 0x70, 0x66, 0x6b, 0xf4, 0x9d, 0x30, 0x11, + 0xe1, 0x1e, 0x7e, 0x58, 0xb3, 0x9c, 0x4f, 0x66, 0x64, 0x4c, 0x9c, 0x57, 0x46, 0xcb, 0x70, 0x8c, + 0x1a, 0xfa, 0xc7, 0x0a, 0x4c, 0x05, 0x1b, 0x75, 0x8f, 0x6f, 0x79, 0x3f, 0x9b, 0xe3, 0xd2, 0xd1, + 0x7c, 0x41, 0x52, 0xaa, 0xdd, 0x38, 0x21, 0x9c, 0xa4, 0xad, 0xfe, 0xc8, 0x20, 0xc8, 0xd9, 0xbd, + 0xd1, 0x4a, 0x2f, 0x76, 0xc9, 0x70, 0x0f, 0xf8, 0xb6, 0xc9, 0x15, 0xe8, 0xab, 0x37, 0x5b, 0x39, + 0x0d, 0x93, 0x01, 0xba, 0xeb, 0x14, 0x5d, 0xbd, 0xd9, 0x42, 0xcf, 0x04, 0xa6, 0xce, 0x7c, 0xc6, + 0xc8, 0xe0, 0xa9, 0x64, 0xcc, 0xdc, 0xe9, 0xb3, 0xf7, 0xfe, 0x4c, 0xf6, 0xde, 0x80, 0x21, 0x57, + 0xd8, 0x41, 0x07, 0xf2, 0xc7, 0xa1, 0x94, 0x66, 0x5a, 0xd8, 0x3d, 0xb9, 0x85, 0xc6, 0x37, 0x8b, + 0xfa, 0x34, 0xa8, 0x22, 0xd8, 0x62, 0xe7, 0x92, 0x88, 0xb2, 0xc4, 0x14, 0xc1, 0x0d, 0x56, 0x82, + 0x05, 0x24, 0x21, 0xcc, 0x0d, 0x75, 0x23, 0xcc, 0xa1, 0xcf, 0xc6, 0xf9, 0xf7, 0x30, 0xdb, 0x7d, + 0x4f, 0xf7, 0x38, 0x9c, 0xff, 0xcf, 0x18, 0xf7, 0x3f, 0x0a, 0x0f, 0x3f, 0x69, 0xb1, 0xd0, 0x7d, + 0x30, 0xc0, 0xa2, 0x28, 0x09, 0x39, 0x20, 0x30, 0x6e, 0xb0, 0x38, 0x3a, 0x98, 0xc3, 0x50, 0x55, + 0xc4, 0x1e, 0xcc, 0xb7, 0xe9, 0x99, 0x6f, 0xa3, 0xa0, 0x27, 0x05, 0x2a, 0xbc, 0x12, 0x79, 0x13, + 0x99, 0xa6, 0x43, 0x6c, 0xc0, 0x50, 0xc3, 0xb0, 0x98, 0xbb, 0x47, 0x3e, 0x23, 0x3a, 0x77, 0xc1, + 0xe2, 0x28, 0xb0, 0x8f, 0x4b, 0xfd, 0xf3, 0x02, 0x65, 0x10, 0xa1, 0x52, 0xbf, 0xc7, 0x23, 0xe9, + 0x8a, 0xd7, 0xd7, 0x4a, 0x7e, 0x23, 0xa0, 0x84, 0x74, 0x3e, 0x40, 0xc8, 0x1d, 0x15, 0xc2, 0xdf, + 0x58, 0x22, 0x46, 0x49, 0x7b, 0x46, 0x83, 0x3c, 0x6b, 0x58, 0x35, 0xfb, 0x8e, 0x98, 0xde, 0x5e, + 0x49, 0xaf, 0x07, 0x08, 0x39, 0xe9, 0xf0, 0x37, 0x96, 0x88, 0xd1, 0x23, 0x89, 0x65, 0xf3, 0xb0, + 0x58, 0x56, 0x66, 0xd1, 0x37, 0x9e, 0x73, 0x5d, 0xf8, 0x1d, 0xb3, 0x23, 0xa9, 0x94, 0x51, 0x07, + 0x67, 0xb6, 0x56, 0x7f, 0x4e, 0x81, 0x33, 0xa9, 0x53, 0x81, 0xae, 0xc3, 0x54, 0xe8, 0x0e, 0x2b, + 0x0b, 0x09, 0xc3, 0x21, 0x8f, 0xbf, 0x19, 0xaf, 0x80, 0x93, 0x6d, 0x78, 0xae, 0xbb, 0x84, 0x10, + 0x22, 0x7c, 0x69, 0x65, 0x55, 0x4b, 0x06, 0xe3, 0xb4, 0x36, 0xea, 0x0b, 0x91, 0xce, 0x86, 0x93, + 0x45, 0xbf, 0x8c, 0x4d, 0x52, 0x0f, 0xde, 0xa4, 0x07, 0x5f, 0xc6, 0x02, 0x2d, 0xc4, 0x1c, 0x86, + 0x2e, 0xc9, 0xa1, 0x32, 0x02, 0xee, 0xee, 0x87, 0xcb, 0x50, 0x7f, 0x4d, 0x81, 0x73, 0x2b, 0x9a, + 0xd5, 0xd2, 0x4c, 0xfe, 0xf8, 0x7d, 0xcd, 0xb6, 0x4d, 0x31, 0x4d, 0x9d, 0x72, 0xef, 0x2b, 0xaf, + 0x95, 0xdc, 0xfb, 0xef, 0x83, 0x73, 0x19, 0xfe, 0x37, 0xa8, 0x0c, 0x63, 0xee, 0x1d, 0xad, 0xb9, + 0x40, 0xb6, 0xb5, 0x1d, 0x43, 0x04, 0xd6, 0xe2, 0x6e, 0xda, 0x63, 0x55, 0xa9, 0xfc, 0x5e, 0xec, + 0x37, 0x8e, 0xb4, 0x52, 0x3d, 0x00, 0xe1, 0xce, 0x6f, 0x58, 0x75, 0xb4, 0x05, 0xc3, 0x9a, 0x49, + 0x1c, 0x2f, 0x8c, 0x3e, 0xf9, 0xad, 0xb9, 0xec, 0xb4, 0x02, 0x07, 0x7f, 0xbe, 0xe6, 0xff, 0xc2, + 0x01, 0x6e, 0xf5, 0x9f, 0x29, 0x70, 0x36, 0x3d, 0x94, 0x52, 0x17, 0x6a, 0x51, 0x03, 0x46, 0x9d, + 0xb0, 0x99, 0xf8, 0x68, 0xdf, 0x29, 0xa7, 0xb8, 0x90, 0x1e, 0xde, 0x50, 0x45, 0xb0, 0xe4, 0xd8, + 0xae, 0xbf, 0x73, 0xe3, 0x59, 0x2f, 0x02, 0xab, 0x98, 0xd4, 0x13, 0x2c, 0xe3, 0x67, 0x19, 0x68, + 0x28, 0x75, 0xb7, 0xa9, 0xe9, 0xa4, 0x76, 0xc2, 0xf9, 0xf5, 0x8f, 0x20, 0xed, 0x43, 0x7a, 0xdf, + 0x8f, 0x37, 0x03, 0x4d, 0x06, 0xcd, 0x83, 0x33, 0xd0, 0xa4, 0x37, 0x7c, 0x9d, 0xa4, 0x46, 0x48, + 0xef, 0x7c, 0xc6, 0xc3, 0xf7, 0xff, 0x36, 0x98, 0x35, 0xda, 0x43, 0x26, 0xe9, 0xdf, 0x39, 0xc6, + 0x24, 0xfd, 0x13, 0xdf, 0x48, 0xd0, 0x9f, 0x92, 0xa0, 0x3f, 0x96, 0x34, 0x7e, 0xf0, 0x84, 0x92, + 0xc6, 0xbf, 0x0c, 0x83, 0x4d, 0xcd, 0x21, 0x96, 0x7f, 0x97, 0x5b, 0xc9, 0xe7, 0x68, 0x10, 0x6e, + 0xb4, 0x90, 0x0b, 0x06, 0x9f, 0xe4, 0x1a, 0x23, 0x80, 0x05, 0xa1, 0x94, 0xe0, 0x29, 0xc3, 0xc7, + 0x15, 0x9d, 0x30, 0x4c, 0x57, 0x3f, 0x72, 0x1c, 0xe9, 0xea, 0xd5, 0xbf, 0x55, 0xe0, 0x62, 0x27, + 0xb6, 0xc4, 0x0c, 0x28, 0x7a, 0xec, 0x33, 0xec, 0xc5, 0x80, 0x92, 0xe0, 0xb6, 0x81, 0x01, 0x25, + 0x0e, 0xc1, 0x09, 0xba, 0xe8, 0x3d, 0x80, 0xec, 0x4d, 0xee, 0xd7, 0x73, 0x9d, 0xd2, 0xe0, 0xaf, + 0x69, 0x0b, 0xec, 0xad, 0x42, 0x90, 0x18, 0xf3, 0x56, 0xa2, 0x06, 0x4e, 0x69, 0xa5, 0xfe, 0x4a, + 0x01, 0x60, 0x95, 0x78, 0x77, 0x6c, 0xe7, 0x36, 0x3d, 0xe3, 0x2f, 0x46, 0xae, 0x0e, 0x86, 0xbf, + 0x7e, 0xf1, 0x28, 0x2f, 0x42, 0x7f, 0xd3, 0xae, 0xb9, 0x42, 0xff, 0x60, 0x1d, 0x61, 0x4f, 0x35, + 0x58, 0x29, 0x2a, 0xc2, 0x00, 0x73, 0x7a, 0x12, 0x0a, 0x34, 0xbb, 0x78, 0x60, 0x97, 0x2b, 0x98, + 0x97, 0x53, 0x0e, 0x29, 0x62, 0x1a, 0xb8, 0xe2, 0x72, 0x67, 0x8c, 0x07, 0x21, 0xe7, 0x65, 0x38, + 0x80, 0xa2, 0xc7, 0x00, 0x8c, 0xe6, 0x92, 0xd6, 0x30, 0x4c, 0x43, 0x7c, 0xae, 0x23, 0xcc, 0x22, + 0x0e, 0x95, 0x35, 0xbf, 0xf4, 0x5e, 0xbb, 0x38, 0x2c, 0x7e, 0xed, 0x61, 0xa9, 0xb6, 0xfa, 0x05, + 0x05, 0x26, 0xc3, 0xc9, 0x13, 0x5b, 0xc5, 0xef, 0x39, 0x0f, 0x06, 0x9c, 0xd9, 0x73, 0x1e, 0xf7, + 0xbf, 0x73, 0xcf, 0xb9, 0x01, 0x2b, 0xab, 0xe7, 0x0f, 0xc2, 0x28, 0xe1, 0x21, 0x8f, 0x2a, 0x65, + 0xcc, 0x79, 0xdc, 0x08, 0x57, 0xe7, 0x16, 0xc3, 0x62, 0x2c, 0xd7, 0x51, 0xff, 0xbe, 0x0f, 0xc6, + 0x56, 0xeb, 0x86, 0xb5, 0xeb, 0xc7, 0x76, 0x0a, 0x2e, 0xde, 0x95, 0xe3, 0xb9, 0x78, 0x7f, 0x0e, + 0x66, 0x4c, 0xf9, 0x9a, 0x8b, 0x0b, 0x4e, 0x9a, 0x55, 0x0f, 0x66, 0x80, 0xe9, 0x31, 0xcb, 0x19, + 0x75, 0x70, 0x66, 0x6b, 0xe4, 0xc1, 0xa0, 0xee, 0x27, 0x68, 0xcc, 0x1d, 0xaf, 0x48, 0x9e, 0x8b, + 0x39, 0x39, 0x74, 0x47, 0xc0, 0xf3, 0xc4, 0xf6, 0x14, 0xb4, 0xd0, 0x47, 0x14, 0x38, 0x43, 0x76, + 0x79, 0xe8, 0x9a, 0x75, 0x47, 0xdb, 0xda, 0x32, 0x74, 0xf1, 0xe2, 0x8f, 0xef, 0xc4, 0xe5, 0xfd, + 0x76, 0xf1, 0xcc, 0x62, 0x5a, 0x85, 0x7b, 0xed, 0xe2, 0xb5, 0xd4, 0x48, 0x42, 0x6c, 0x35, 0x53, + 0x9b, 0xe0, 0x74, 0x52, 0xb3, 0x8f, 0xc2, 0xe8, 0x21, 0xde, 0x89, 0x47, 0xe2, 0x05, 0xfd, 0x6a, + 0x01, 0xc6, 0xe8, 0x76, 0x5b, 0xb6, 0x75, 0xcd, 0x2c, 0xaf, 0x56, 0xd1, 0xfd, 0xf1, 0x30, 0x89, + 0xc1, 0x15, 0x63, 0x22, 0x54, 0xe2, 0x32, 0x4c, 0x6f, 0xd9, 0x8e, 0x4e, 0xd6, 0x4b, 0x6b, 0xeb, + 0xb6, 0x70, 0x3e, 0x2b, 0xaf, 0x56, 0x85, 0x5e, 0xc7, 0xae, 0x3c, 0x96, 0x52, 0xe0, 0x38, 0xb5, + 0x15, 0xba, 0x05, 0x67, 0xc2, 0xf2, 0x8d, 0x26, 0x7f, 0xb0, 0x40, 0xd1, 0xf5, 0x85, 0x0f, 0x2e, + 0x96, 0xd2, 0x2a, 0xe0, 0xf4, 0x76, 0x48, 0x83, 0x0b, 0xe2, 0x6e, 0x65, 0xc9, 0x76, 0xee, 0x68, + 0x4e, 0x2d, 0x8a, 0xb6, 0x3f, 0xcc, 0x5b, 0x5f, 0xce, 0xae, 0x86, 0x3b, 0xe1, 0x50, 0xff, 0x4e, + 0x81, 0x68, 0x10, 0x4a, 0x74, 0x1e, 0xfa, 0x1c, 0x91, 0xff, 0x4f, 0x04, 0x63, 0xa4, 0x2a, 0x02, + 0x2d, 0x43, 0x73, 0x00, 0x4e, 0x18, 0x09, 0xb3, 0x10, 0xe6, 0xc5, 0x90, 0x62, 0x58, 0x4a, 0x35, + 0x28, 0x2a, 0x4f, 0xab, 0x0b, 0x86, 0xc7, 0x50, 0xad, 0x6b, 0x75, 0x4c, 0xcb, 0x58, 0x02, 0x14, + 0xa3, 0x4e, 0x5c, 0xdf, 0x1c, 0xcd, 0x13, 0xa0, 0xb0, 0x12, 0x2c, 0x20, 0x48, 0x83, 0xf1, 0x66, + 0xcb, 0x14, 0xf1, 0x94, 0xa8, 0xea, 0xc3, 0xcd, 0x86, 0x57, 0xd3, 0xb2, 0xfb, 0xb1, 0xd5, 0x4f, + 0x4d, 0xf1, 0xb7, 0x26, 0xa3, 0xc0, 0x51, 0x8c, 0xea, 0x8f, 0x0f, 0x82, 0x14, 0x5e, 0xe7, 0x10, + 0x52, 0xe8, 0x4f, 0x2b, 0x30, 0xad, 0x9b, 0x06, 0xb1, 0xbc, 0x58, 0xa4, 0x0a, 0x7e, 0x7c, 0x6c, + 0xe4, 0x8a, 0xfb, 0xd3, 0x24, 0x56, 0xa5, 0x2c, 0x9e, 0xb7, 0x94, 0x52, 0x90, 0x8b, 0x27, 0x40, + 0x29, 0x10, 0x9c, 0xda, 0x19, 0x36, 0x1e, 0x56, 0x5e, 0x29, 0xcb, 0xd1, 0x33, 0x4b, 0xa2, 0x0c, + 0x07, 0x50, 0xca, 0x79, 0xeb, 0x8e, 0xdd, 0x6a, 0xba, 0x25, 0xf6, 0x8a, 0x95, 0x2f, 0x0a, 0xe3, + 0xbc, 0xd7, 0xc3, 0x62, 0x2c, 0xd7, 0x41, 0x0f, 0xc3, 0x18, 0xff, 0xb9, 0xe6, 0x90, 0x2d, 0x63, + 0x57, 0x1c, 0x4a, 0xcc, 0x78, 0x7a, 0x5d, 0x2a, 0xc7, 0x91, 0x5a, 0x2c, 0x7e, 0x9b, 0xeb, 0xb6, + 0x88, 0xb3, 0x81, 0x97, 0x45, 0x92, 0x68, 0x1e, 0xbf, 0xcd, 0x2f, 0xc4, 0x21, 0x1c, 0xfd, 0x90, + 0x02, 0x13, 0x0e, 0x79, 0xb9, 0x65, 0x38, 0x54, 0x84, 0xd1, 0x8c, 0x86, 0x2b, 0x62, 0x1c, 0xe1, + 0xde, 0xe2, 0x2a, 0xcd, 0xe1, 0x08, 0x52, 0xce, 0x20, 0x03, 0x57, 0x8e, 0x28, 0x10, 0xc7, 0x7a, + 0x40, 0xa7, 0xca, 0x35, 0xea, 0x96, 0x61, 0xd5, 0xe7, 0xcd, 0xba, 0x2b, 0x32, 0x61, 0x71, 0x9b, + 0x63, 0x58, 0x8c, 0xe5, 0x3a, 0xe8, 0x11, 0x18, 0x6f, 0xb9, 0x94, 0xed, 0x35, 0x08, 0x9f, 0xdf, + 0x91, 0xd0, 0xd7, 0x65, 0x43, 0x06, 0xe0, 0x68, 0x3d, 0xf4, 0x18, 0x4c, 0xf8, 0x05, 0x62, 0x96, + 0x81, 0xa7, 0x63, 0x61, 0x77, 0x93, 0x11, 0x08, 0x8e, 0xd5, 0x9c, 0x9d, 0x87, 0xd3, 0x29, 0xc3, + 0x3c, 0x14, 0x6f, 0xfd, 0xbf, 0x0a, 0x9c, 0xe1, 0x52, 0x97, 0x9f, 0x5e, 0xda, 0x4f, 0x8e, 0x91, + 0x9e, 0x67, 0x42, 0x39, 0xd6, 0x3c, 0x13, 0x5f, 0x87, 0x7c, 0x1a, 0xea, 0xcf, 0x16, 0xe0, 0x8d, + 0x07, 0x7e, 0x97, 0xe8, 0x27, 0x14, 0x18, 0x25, 0xbb, 0x9e, 0xa3, 0x05, 0x4f, 0xfd, 0xe9, 0x26, + 0xdd, 0x3a, 0x16, 0x26, 0x30, 0xb7, 0x18, 0x12, 0xe2, 0x1b, 0x37, 0x50, 0xa5, 0x24, 0x08, 0x96, + 0xfb, 0x43, 0xb9, 0x2d, 0x4f, 0x57, 0x24, 0x3b, 0xc5, 0x09, 0x2e, 0x28, 0x20, 0xb3, 0x4f, 0xc0, + 0x64, 0x1c, 0xf3, 0xa1, 0xf6, 0xca, 0xe7, 0x15, 0x48, 0x0d, 0xc7, 0x89, 0x4a, 0x30, 0xa5, 0xb5, + 0x3c, 0x3b, 0x72, 0xaf, 0x27, 0x44, 0x49, 0xe6, 0xf4, 0x3d, 0x1f, 0x07, 0xe2, 0x64, 0x7d, 0x6e, + 0x80, 0xb5, 0x5a, 0x9a, 0x19, 0x45, 0xc3, 0x05, 0x2e, 0x61, 0x80, 0x4d, 0x80, 0x71, 0x5a, 0x1b, + 0xf5, 0x5f, 0x2a, 0x70, 0x26, 0xd5, 0x2c, 0xd9, 0x85, 0x2d, 0x2e, 0x7d, 0xdb, 0x17, 0x8e, 0x73, + 0xdb, 0xab, 0xbf, 0x5c, 0x80, 0xa1, 0x35, 0xc7, 0xa6, 0x47, 0xdb, 0x09, 0x58, 0xe0, 0xb4, 0x88, + 0x05, 0x2e, 0x97, 0x7d, 0x41, 0x74, 0x36, 0xd3, 0xe4, 0x66, 0xc4, 0x4c, 0x6e, 0xf3, 0xbd, 0x10, + 0xe9, 0x6c, 0x63, 0xfb, 0x92, 0x02, 0xa3, 0xa2, 0xe6, 0x09, 0x18, 0xd5, 0xbe, 0x23, 0x6a, 0x54, + 0x7b, 0x77, 0x0f, 0xe3, 0xca, 0xb0, 0xa2, 0x7d, 0x46, 0x81, 0x71, 0x51, 0x63, 0x85, 0x34, 0x36, + 0x89, 0x83, 0x96, 0x60, 0xc8, 0x6d, 0xb1, 0x85, 0x14, 0x03, 0xba, 0x20, 0x8b, 0x47, 0xce, 0xa6, + 0xa6, 0xd3, 0xee, 0x57, 0x79, 0x15, 0x29, 0x95, 0x32, 0x2f, 0xc0, 0x7e, 0x63, 0xba, 0xf7, 0x1d, + 0xdb, 0x4c, 0x84, 0xad, 0xc7, 0xb6, 0x49, 0x30, 0x83, 0x50, 0x3d, 0x8f, 0xfe, 0xf5, 0x75, 0x38, + 0xa6, 0xe7, 0x51, 0xb0, 0x8b, 0x79, 0xb9, 0xfa, 0x85, 0x81, 0x60, 0xb2, 0x99, 0x52, 0x7f, 0x03, + 0x46, 0x74, 0x87, 0x68, 0x1e, 0xa9, 0x2d, 0xec, 0x75, 0xd3, 0x39, 0x26, 0x07, 0x94, 0xfc, 0x16, + 0x38, 0x6c, 0x4c, 0x8f, 0x5c, 0xd9, 0xc1, 0xb3, 0x10, 0x4a, 0x27, 0x99, 0xce, 0x9d, 0xdf, 0x0a, + 0x03, 0xf6, 0x1d, 0x2b, 0x78, 0x1c, 0xd2, 0x91, 0x30, 0x1b, 0xca, 0x2d, 0x5a, 0x1b, 0xf3, 0x46, + 0x72, 0xda, 0x86, 0xfe, 0x0e, 0x69, 0x1b, 0x4c, 0x18, 0x6a, 0xb0, 0x65, 0xe8, 0x29, 0xb3, 0x6e, + 0x64, 0x41, 0xc3, 0x25, 0xe2, 0xbf, 0x5d, 0xec, 0x93, 0xa0, 0xa2, 0x93, 0xe5, 0x5b, 0x74, 0x64, + 0xd1, 0x29, 0x30, 0xf3, 0xe0, 0x10, 0x8e, 0xf6, 0xa2, 0xf9, 0x40, 0x86, 0xf2, 0xdb, 0x49, 0x45, + 0xf7, 0xa4, 0x14, 0x20, 0x7c, 0xea, 0xb3, 0x72, 0x82, 0xa0, 0xcf, 0x2b, 0x70, 0xae, 0x96, 0x9e, + 0xb1, 0x4d, 0x5c, 0x95, 0xe7, 0x7a, 0x98, 0x9d, 0x91, 0x04, 0x6e, 0xa1, 0x28, 0x26, 0x2c, 0x2b, + 0x4b, 0x1c, 0xce, 0xea, 0x8c, 0xfa, 0xbd, 0xfd, 0xc1, 0xd7, 0x24, 0x2c, 0x1d, 0xe9, 0x76, 0x28, + 0x25, 0x8f, 0x1d, 0x0a, 0xbd, 0xdd, 0xcf, 0x1f, 0xc6, 0xb7, 0xeb, 0xa5, 0x78, 0xfe, 0xb0, 0x31, + 0x41, 0x3a, 0x92, 0x33, 0xac, 0x05, 0xa7, 0x5d, 0x4f, 0x33, 0x49, 0xd5, 0x10, 0x97, 0x83, 0xae, + 0xa7, 0x35, 0x9a, 0x39, 0x12, 0x78, 0xf1, 0x40, 0x0d, 0x49, 0x54, 0x38, 0x0d, 0x3f, 0xfa, 0x6e, + 0x16, 0x7c, 0x4f, 0x33, 0xd9, 0xe5, 0x29, 0x4f, 0x62, 0x1a, 0x12, 0x3f, 0xbc, 0xbb, 0xbb, 0x08, + 0xad, 0x97, 0x8e, 0x0f, 0x67, 0x52, 0x42, 0xaf, 0xc0, 0x19, 0x7a, 0x00, 0xce, 0xeb, 0x9e, 0xb1, + 0x63, 0x78, 0x7b, 0x61, 0x17, 0x0e, 0x9f, 0xb5, 0x8b, 0x69, 0xdb, 0xcb, 0x69, 0xc8, 0x70, 0x3a, + 0x0d, 0xf5, 0x6f, 0x14, 0x40, 0xc9, 0xbd, 0x8e, 0x4c, 0x18, 0xae, 0xf9, 0x91, 0x13, 0x94, 0x23, + 0xc9, 0xaa, 0x13, 0x1c, 0x21, 0x41, 0xc0, 0x85, 0x80, 0x02, 0xb2, 0x61, 0xe4, 0xce, 0xb6, 0xe1, + 0x11, 0xd3, 0x70, 0xbd, 0x23, 0x4a, 0xe2, 0x13, 0xc4, 0x84, 0x7f, 0xd6, 0x47, 0x8c, 0x43, 0x1a, + 0xea, 0x27, 0xfb, 0x61, 0x38, 0xc8, 0xc6, 0x79, 0xb0, 0x9b, 0x75, 0x0b, 0x90, 0x2e, 0x05, 0x04, + 0xec, 0xc5, 0x66, 0xca, 0x64, 0xa0, 0x52, 0x02, 0x19, 0x4e, 0x21, 0x80, 0x5e, 0x81, 0x69, 0xc3, + 0xda, 0x72, 0xb4, 0x20, 0xdc, 0x61, 0xc9, 0x37, 0x94, 0xe5, 0x20, 0xcc, 0xb4, 0xe8, 0x4a, 0x0a, + 0x3a, 0x9c, 0x4a, 0x04, 0x11, 0x18, 0xe2, 0x09, 0x91, 0xfd, 0x5b, 0x97, 0x5c, 0xb6, 0x7a, 0x2e, + 0x6b, 0x86, 0xec, 0xdd, 0x97, 0x3d, 0x7d, 0xdc, 0x3c, 0x38, 0x2d, 0xff, 0xdf, 0xbf, 0x90, 0x12, + 0xfb, 0xbe, 0x94, 0x9f, 0x5e, 0x78, 0xb7, 0xc5, 0x83, 0xd3, 0x46, 0x0b, 0x71, 0x9c, 0xa0, 0xfa, + 0xbb, 0x0a, 0x0c, 0xf0, 0x18, 0x60, 0xc7, 0x2f, 0x6a, 0xbe, 0x2f, 0x22, 0x6a, 0x3e, 0x9e, 0x67, + 0x90, 0xac, 0xab, 0x99, 0x29, 0xfd, 0x7f, 0x47, 0x81, 0x11, 0x56, 0xe3, 0x04, 0x64, 0xbf, 0x17, + 0xa3, 0xb2, 0xdf, 0xa3, 0xb9, 0x47, 0x93, 0x21, 0xf9, 0xfd, 0x6e, 0x9f, 0x18, 0x0b, 0x13, 0xad, + 0x2a, 0x70, 0x5a, 0x3c, 0x8c, 0x5d, 0x36, 0xb6, 0x08, 0xdd, 0xe2, 0x65, 0x6d, 0xcf, 0x15, 0x4f, + 0x35, 0x78, 0xd0, 0x99, 0x24, 0x18, 0xa7, 0xb5, 0x41, 0xbf, 0xaa, 0x50, 0x21, 0xc6, 0x73, 0x0c, + 0xbd, 0xa7, 0xcb, 0xe0, 0xa0, 0x6f, 0x73, 0x2b, 0x1c, 0x19, 0xd7, 0x4d, 0x37, 0x42, 0x69, 0x86, + 0x95, 0xde, 0x6b, 0x17, 0x8b, 0x29, 0x36, 0xe3, 0x30, 0xad, 0xb5, 0xeb, 0x7d, 0xe4, 0x2f, 0x3a, + 0x56, 0x61, 0xda, 0x98, 0xdf, 0x63, 0x74, 0x03, 0x06, 0x5c, 0xdd, 0x6e, 0xfa, 0x4f, 0xab, 0xef, + 0x4b, 0xb3, 0x0d, 0xc6, 0xcd, 0x82, 0xc1, 0x04, 0x57, 0x69, 0x4b, 0xcc, 0x11, 0xcc, 0xbe, 0x04, + 0x63, 0x72, 0xcf, 0x53, 0x74, 0xdf, 0x72, 0xd4, 0x63, 0xee, 0x90, 0xce, 0x61, 0xb2, 0xae, 0xfc, + 0x47, 0x7d, 0x30, 0x88, 0x49, 0x5d, 0xa4, 0x4c, 0x3b, 0x40, 0xe7, 0x34, 0xfc, 0xfc, 0xc1, 0x85, + 0xfc, 0xef, 0xf0, 0xe4, 0xa4, 0x38, 0xcf, 0xdb, 0x96, 0x34, 0x07, 0x72, 0x0a, 0x61, 0x64, 0x05, + 0x89, 0xa4, 0xfa, 0xf2, 0x3b, 0xd4, 0xf2, 0x81, 0x75, 0x93, 0x3a, 0x0a, 0xfd, 0xa0, 0x02, 0x48, + 0xd3, 0x75, 0xe2, 0xba, 0x98, 0xb8, 0x74, 0xee, 0xb9, 0xb0, 0xca, 0xb9, 0x6c, 0xbe, 0xa8, 0xd8, + 0x71, 0x6c, 0xa1, 0xd8, 0x96, 0x00, 0xb9, 0x38, 0x85, 0x78, 0x2f, 0xe9, 0xac, 0x7e, 0x5f, 0x81, + 0xb1, 0x48, 0xb6, 0xb0, 0x46, 0x68, 0x4b, 0xcf, 0xef, 0xb2, 0xe3, 0xbf, 0xfe, 0xba, 0xd0, 0xa1, + 0x12, 0xb7, 0xcf, 0xdf, 0x0a, 0xd2, 0x5d, 0x1c, 0x4d, 0x62, 0x31, 0xf5, 0x53, 0x0a, 0x9c, 0xf5, + 0x07, 0x14, 0x8d, 0x6b, 0x8e, 0xae, 0xc2, 0xb0, 0xd6, 0x34, 0x98, 0xa1, 0x57, 0x36, 0x95, 0xcf, + 0xaf, 0x55, 0x58, 0x19, 0x0e, 0xa0, 0x91, 0x24, 0xcd, 0x85, 0x03, 0x93, 0x34, 0xbf, 0x59, 0x4a, + 0x3b, 0x3d, 0x10, 0xca, 0x2e, 0x01, 0x61, 0xee, 0xcc, 0xa9, 0xbe, 0x13, 0x46, 0xaa, 0xd5, 0x1b, + 0x7c, 0x49, 0x0f, 0x71, 0xe3, 0xa3, 0x7e, 0xbc, 0x0f, 0xc6, 0x45, 0x82, 0x06, 0x83, 0x99, 0x80, + 0x4e, 0xe0, 0x9c, 0x5b, 0x87, 0x11, 0x37, 0xb8, 0xc3, 0x28, 0x64, 0xf3, 0xa9, 0xe0, 0x5a, 0x22, + 0x9e, 0x65, 0x2f, 0xbc, 0xbe, 0x08, 0x11, 0xa1, 0x9b, 0x30, 0xf8, 0x32, 0xe5, 0xb9, 0xfe, 0xb7, + 0xda, 0x15, 0xeb, 0x0b, 0x3e, 0x44, 0xc6, 0xae, 0x5d, 0x2c, 0x50, 0x20, 0x97, 0x3d, 0x4f, 0x64, + 0x42, 0x60, 0x2f, 0xa1, 0x3a, 0x23, 0x33, 0x1b, 0x24, 0x9d, 0x1f, 0x13, 0xaf, 0x1c, 0xd9, 0x2f, + 0x1c, 0x10, 0x62, 0x29, 0x42, 0x23, 0x2d, 0x5e, 0x27, 0x29, 0x42, 0x23, 0x7d, 0xce, 0x38, 0xae, + 0x1f, 0x85, 0x33, 0xa9, 0x93, 0x71, 0xb0, 0x88, 0xad, 0xfe, 0xab, 0x02, 0xf4, 0x57, 0x09, 0xa9, + 0x9d, 0xc0, 0xce, 0x7c, 0x31, 0x22, 0x81, 0x7d, 0x6b, 0xee, 0x24, 0xa5, 0x59, 0x96, 0xbe, 0xad, + 0x98, 0xa5, 0xef, 0x89, 0xdc, 0x14, 0x3a, 0x9b, 0xf9, 0xfe, 0xaa, 0x00, 0x43, 0xb4, 0x5a, 0x79, + 0xb5, 0x8a, 0x1a, 0xd2, 0x56, 0x2e, 0xe4, 0x17, 0x9f, 0x05, 0xba, 0x83, 0x36, 0x31, 0xfd, 0x72, + 0x0c, 0x91, 0xf8, 0x45, 0x0c, 0xb2, 0x72, 0x04, 0xe4, 0x44, 0xec, 0x09, 0x46, 0xd4, 0xcf, 0x2b, + 0x83, 0x03, 0x42, 0xe8, 0x8e, 0xa4, 0x93, 0xf6, 0xe7, 0xf7, 0xdd, 0x4d, 0x27, 0xda, 0x41, 0x3d, + 0x55, 0x3f, 0xae, 0xc0, 0xa9, 0x58, 0xab, 0x2e, 0x94, 0xc6, 0x63, 0x61, 0x80, 0xea, 0xd7, 0x14, + 0xfa, 0x8d, 0xa5, 0x8c, 0xa0, 0x8b, 0x1e, 0x85, 0x09, 0xc0, 0x0a, 0x1d, 0x13, 0x80, 0x5d, 0x84, + 0x7e, 0x2a, 0x19, 0xc9, 0x3e, 0x3b, 0x54, 0x60, 0xc2, 0xac, 0x14, 0xe9, 0x30, 0xa1, 0x4b, 0x39, + 0xe0, 0xc9, 0x96, 0xe0, 0x9d, 0x5d, 0xb1, 0xe2, 0xe0, 0xba, 0xb1, 0x14, 0x41, 0x81, 0x63, 0x28, + 0xd5, 0xdf, 0x56, 0x60, 0x98, 0x0e, 0xf3, 0x04, 0x98, 0xe3, 0xb7, 0x47, 0x99, 0xe3, 0xbb, 0xf2, + 0xee, 0xa9, 0x0c, 0x9e, 0xf8, 0xd7, 0x05, 0x60, 0x19, 0x8c, 0x85, 0x7f, 0x91, 0xe4, 0x39, 0xa4, + 0x64, 0xf8, 0x3c, 0x5d, 0x11, 0x8e, 0x47, 0x31, 0xa3, 0xb4, 0xe4, 0x7c, 0xf4, 0xd6, 0x88, 0x6f, + 0x51, 0x44, 0xb8, 0x48, 0xf1, 0x2f, 0xba, 0x0b, 0xe3, 0xee, 0xb6, 0x6d, 0x7b, 0xe5, 0xf0, 0xe3, + 0xc9, 0x7d, 0x01, 0xc1, 0x1e, 0x8b, 0xfb, 0x43, 0xe1, 0x57, 0xb9, 0x55, 0x19, 0x37, 0x8e, 0x92, + 0x42, 0x73, 0x00, 0x9b, 0xa6, 0xad, 0xdf, 0xe6, 0xae, 0x4d, 0xfc, 0x71, 0x30, 0x73, 0x9e, 0x58, + 0x08, 0x4a, 0xb1, 0x54, 0xa3, 0x27, 0x2f, 0xae, 0xaf, 0x29, 0x7c, 0xa6, 0x0f, 0xf1, 0x8d, 0x9e, + 0xa0, 0x23, 0xdc, 0x5b, 0x60, 0xd0, 0x61, 0x3a, 0x82, 0x58, 0xb1, 0xe0, 0xe3, 0xe3, 0x9a, 0x03, + 0x16, 0x50, 0xba, 0x3d, 0xb8, 0xe2, 0xd3, 0x1f, 0x5e, 0x38, 0xc8, 0xea, 0x8a, 0xfa, 0xcb, 0x62, + 0x98, 0x41, 0x12, 0xec, 0x26, 0x8c, 0x33, 0xcd, 0x22, 0x96, 0x7d, 0xfb, 0xed, 0x5d, 0x7e, 0x23, + 0x72, 0xd3, 0xd0, 0x2d, 0x37, 0x52, 0x8c, 0xa3, 0x04, 0xd0, 0x23, 0x30, 0xee, 0x8f, 0x8e, 0x7b, + 0xc7, 0x16, 0xc2, 0x17, 0x9a, 0x6b, 0x32, 0x00, 0x47, 0xeb, 0xa9, 0x9f, 0x2e, 0xc0, 0x25, 0xde, + 0x77, 0x66, 0x79, 0x29, 0x93, 0x26, 0xb1, 0x6a, 0xc4, 0xd2, 0xf7, 0x98, 0x9c, 0x5d, 0xb3, 0xeb, + 0xe8, 0x15, 0x18, 0xbc, 0x43, 0x48, 0x2d, 0xb8, 0xc2, 0x78, 0x36, 0x7f, 0x0e, 0xf1, 0x0c, 0x12, + 0xcf, 0x32, 0xf4, 0xfc, 0x36, 0x98, 0xff, 0x8f, 0x05, 0x49, 0x4a, 0xbc, 0xe9, 0xd8, 0x9b, 0x81, + 0x38, 0x78, 0xf4, 0xc4, 0xd7, 0x18, 0x7a, 0x4e, 0x9c, 0xff, 0x8f, 0x05, 0x49, 0x75, 0x0d, 0xee, + 0xeb, 0xa2, 0xe9, 0x61, 0xc4, 0xfe, 0x83, 0x30, 0xf2, 0xd1, 0x1f, 0x06, 0xe3, 0x9f, 0x28, 0xf0, + 0x26, 0x09, 0xe5, 0xe2, 0x2e, 0xd5, 0x44, 0x4a, 0x5a, 0x53, 0xd3, 0xa9, 0xae, 0xcf, 0x62, 0xd4, + 0x1d, 0x2a, 0x6b, 0xef, 0xc7, 0x15, 0x18, 0xe2, 0x1e, 0x79, 0x3e, 0xfb, 0x7d, 0xb1, 0xc7, 0x29, + 0xcf, 0xec, 0x92, 0x9f, 0xcd, 0xcc, 0x1f, 0x1b, 0xff, 0xed, 0x62, 0x9f, 0xbe, 0xfa, 0x6f, 0x07, + 0xe0, 0x9b, 0xbb, 0x47, 0x84, 0xbe, 0xa6, 0xc8, 0xd9, 0xc6, 0xb9, 0x8d, 0xbc, 0x71, 0xbc, 0x9d, + 0x0f, 0xac, 0x41, 0xc2, 0xc0, 0xf0, 0x6c, 0x22, 0x21, 0xf9, 0x11, 0x19, 0x9a, 0xc2, 0x81, 0xa1, + 0x7f, 0xae, 0xc0, 0x18, 0x3d, 0x96, 0x02, 0xe6, 0xc2, 0x97, 0xa9, 0x79, 0xcc, 0x23, 0x5d, 0x95, + 0x48, 0xc6, 0xe2, 0x5a, 0xc9, 0x20, 0x1c, 0xe9, 0x1b, 0xda, 0x88, 0x5e, 0xff, 0x71, 0x15, 0xf1, + 0x72, 0x9a, 0x5c, 0x72, 0x98, 0x74, 0xff, 0xb3, 0x26, 0x4c, 0x44, 0x67, 0xfe, 0x38, 0xcd, 0x64, + 0xb3, 0x4f, 0xc2, 0x54, 0x62, 0xf4, 0x87, 0x32, 0xc8, 0x7c, 0x69, 0x10, 0x8a, 0xd2, 0x54, 0xa7, + 0x85, 0xa7, 0x41, 0x9f, 0x55, 0x60, 0x54, 0xb3, 0x2c, 0xe1, 0xd8, 0xe4, 0xef, 0xdf, 0x5a, 0x8f, + 0xab, 0x9a, 0x46, 0x6a, 0x6e, 0x3e, 0x24, 0x13, 0xf3, 0xdc, 0x91, 0x20, 0x58, 0xee, 0x4d, 0x07, + 0xef, 0xdc, 0xc2, 0x89, 0x79, 0xe7, 0xa2, 0x0f, 0xf8, 0x07, 0x71, 0x5f, 0xfe, 0x40, 0x15, 0x07, + 0xcc, 0x0d, 0x3b, 0xd7, 0x33, 0xac, 0x92, 0xdf, 0xaf, 0xb0, 0x43, 0x36, 0x8c, 0x22, 0x24, 0xce, + 0xa4, 0x5c, 0x4e, 0x96, 0x07, 0x86, 0x28, 0x0a, 0xce, 0xee, 0xb0, 0x08, 0x47, 0xc9, 0xa3, 0x1f, + 0x55, 0x60, 0xec, 0xae, 0x6d, 0x69, 0xa6, 0xf0, 0xb3, 0x16, 0xd7, 0x34, 0xef, 0x3b, 0x9e, 0x79, + 0x09, 0xc8, 0x70, 0xdf, 0x4a, 0xb9, 0x04, 0x47, 0xba, 0xc1, 0x32, 0xe6, 0xb3, 0xb7, 0xd5, 0x83, + 0xa1, 0x3c, 0x2d, 0xbf, 0xab, 0x9e, 0x7d, 0x02, 0x26, 0xe3, 0x7b, 0xf0, 0x50, 0xdf, 0xd3, 0x3a, + 0x3c, 0x70, 0x88, 0xfe, 0x76, 0x79, 0xf4, 0xa9, 0xbf, 0xd1, 0x1f, 0x39, 0x4a, 0x33, 0xb7, 0x47, + 0x17, 0xb6, 0xf2, 0x57, 0x63, 0x1f, 0x33, 0x67, 0xd1, 0xc6, 0x71, 0x6d, 0xd8, 0xa3, 0xfd, 0xa2, + 0xfb, 0x4e, 0xee, 0x8b, 0x7e, 0xad, 0x7d, 0x52, 0x3d, 0xef, 0xcc, 0x05, 0x6e, 0x0b, 0x10, 0x0b, + 0x16, 0x26, 0x6b, 0x62, 0x81, 0x3a, 0x0d, 0xd7, 0xf0, 0xc3, 0x4d, 0x4b, 0x22, 0xdd, 0x33, 0xbc, + 0x18, 0xfb, 0x70, 0x75, 0x39, 0x72, 0x58, 0xac, 0xdb, 0x4d, 0xdb, 0xb4, 0xeb, 0x7b, 0xf3, 0x77, + 0x34, 0x87, 0x60, 0xbb, 0xe5, 0x09, 0x6c, 0xdd, 0x0a, 0x88, 0x1f, 0x1e, 0x80, 0x2b, 0x12, 0xba, + 0xd4, 0xa8, 0x9c, 0x87, 0x79, 0xab, 0xf0, 0x6a, 0x3c, 0x78, 0x66, 0x21, 0xbf, 0x93, 0xe9, 0x41, + 0xfd, 0xca, 0x17, 0x4c, 0x13, 0xb5, 0x15, 0x80, 0x86, 0xb6, 0x2b, 0x22, 0xb9, 0x89, 0xd3, 0xa2, + 0x76, 0x2c, 0x1d, 0x5c, 0x09, 0xc8, 0xf0, 0xee, 0x3d, 0xe7, 0xdb, 0x39, 0x43, 0xc0, 0x11, 0x49, + 0x80, 0xd2, 0x88, 0x7a, 0x8e, 0x16, 0x3a, 0xdb, 0x80, 0x53, 0xb1, 0x9e, 0x1f, 0xeb, 0x3d, 0xe3, + 0x97, 0x86, 0x7c, 0x05, 0x59, 0x84, 0x98, 0xfb, 0x25, 0x05, 0xce, 0x93, 0x2c, 0x81, 0x53, 0x68, + 0xcb, 0xcf, 0x1d, 0x97, 0x40, 0x2b, 0x12, 0x6e, 0x65, 0x81, 0x71, 0x76, 0xcf, 0xd0, 0x1e, 0x80, + 0x1b, 0x7c, 0xd3, 0xbd, 0xc4, 0xad, 0x48, 0x65, 0x12, 0xdc, 0x0e, 0x13, 0xfe, 0xc6, 0x12, 0x31, + 0xf4, 0x53, 0x0a, 0x4c, 0x9b, 0x29, 0x1c, 0x4e, 0x70, 0xcc, 0xea, 0x31, 0x9c, 0x2d, 0xdc, 0x43, + 0x25, 0x0d, 0x82, 0x53, 0xbb, 0x82, 0x7e, 0x26, 0x33, 0xc4, 0x30, 0x97, 0x4c, 0xd6, 0x8f, 0xe3, + 0x1b, 0xcc, 0x11, 0x6d, 0xf8, 0xd3, 0x0a, 0xa0, 0x5a, 0x42, 0xf9, 0x16, 0xce, 0x89, 0x4f, 0x1f, + 0xb9, 0x89, 0x81, 0xbb, 0x18, 0x25, 0xcb, 0x71, 0x4a, 0x27, 0xd8, 0x3a, 0x7b, 0x29, 0x3c, 0x5f, + 0x3c, 0x01, 0xee, 0x75, 0x9d, 0xd3, 0x8e, 0x13, 0xbe, 0xce, 0x69, 0x10, 0x9c, 0xda, 0x15, 0xf5, + 0xb7, 0x46, 0xb8, 0x2d, 0x98, 0xf9, 0x80, 0xbc, 0x08, 0x83, 0x9b, 0x9a, 0x7e, 0x5b, 0xdc, 0xbf, + 0xe6, 0xf4, 0x4a, 0x5a, 0x60, 0x18, 0xb8, 0x15, 0x86, 0xff, 0x8f, 0x05, 0x56, 0xf4, 0x3c, 0xf4, + 0xd5, 0x2c, 0xff, 0x85, 0xfd, 0xbb, 0x7b, 0xb8, 0x5f, 0x08, 0xe3, 0x94, 0x94, 0x57, 0xab, 0x98, + 0x22, 0x45, 0x16, 0x0c, 0x5b, 0xc2, 0x74, 0x2a, 0xac, 0x5b, 0x4f, 0xe5, 0x25, 0x10, 0x98, 0x60, + 0x03, 0xc3, 0xaf, 0x5f, 0x82, 0x03, 0x1a, 0x94, 0x5e, 0xec, 0x7e, 0x33, 0x37, 0xbd, 0xe0, 0x46, + 0x48, 0x0e, 0xe0, 0x1a, 0xbf, 0x15, 0x22, 0x30, 0xe8, 0x69, 0x86, 0xe5, 0xf9, 0xaf, 0xe5, 0x1f, + 0xcf, 0x4b, 0x6d, 0x9d, 0x62, 0x09, 0x2d, 0xa4, 0xec, 0xa7, 0x8b, 0x05, 0x72, 0xb4, 0x09, 0x83, + 0xfc, 0xc5, 0xbc, 0xf8, 0x84, 0x72, 0xdf, 0xaf, 0xf1, 0x47, 0xf8, 0x7c, 0x1b, 0xf0, 0xff, 0xb1, + 0xc0, 0x8c, 0x5e, 0x82, 0x61, 0xd7, 0x77, 0x47, 0x1b, 0xee, 0x6d, 0xea, 0x02, 0x5f, 0x34, 0xf1, + 0xfe, 0x57, 0x38, 0xa1, 0x05, 0xf8, 0xd1, 0x26, 0x0c, 0x19, 0x42, 0xa5, 0x1a, 0xc9, 0xbf, 0xed, + 0x7c, 0x75, 0x89, 0x69, 0x1b, 0xbe, 0xa6, 0xe4, 0x23, 0xce, 0xf2, 0x39, 0x81, 0xaf, 0xa3, 0xcf, + 0x09, 0x7a, 0x19, 0x80, 0xf8, 0x26, 0x74, 0x77, 0x66, 0x34, 0xff, 0x96, 0x91, 0x0c, 0xf1, 0xbe, + 0x40, 0x14, 0x14, 0xb9, 0x58, 0x22, 0x82, 0x5e, 0x91, 0x6d, 0x76, 0x63, 0xbd, 0x05, 0x01, 0x49, + 0x86, 0xb5, 0x09, 0xaf, 0xee, 0x02, 0x73, 0x91, 0x64, 0x4a, 0x53, 0xbf, 0x04, 0x00, 0xe1, 0xb5, + 0x2e, 0xda, 0x82, 0x61, 0x9f, 0x4a, 0x2f, 0x61, 0x78, 0xae, 0x0b, 0x30, 0xdf, 0x5e, 0xfe, 0x2f, + 0x1c, 0xe0, 0x46, 0xa5, 0xb4, 0x70, 0x50, 0x61, 0x86, 0xdc, 0xee, 0x42, 0x41, 0xbd, 0x0c, 0xa0, + 0x87, 0x41, 0x5e, 0xfb, 0xf2, 0xaf, 0x55, 0x10, 0x00, 0x36, 0x5c, 0x2b, 0x29, 0x46, 0xac, 0x44, + 0x24, 0xc3, 0x2b, 0xbd, 0x3f, 0x97, 0x57, 0xfa, 0xe3, 0x70, 0x4a, 0x78, 0x01, 0x56, 0xd8, 0x2d, + 0xa3, 0xb7, 0x27, 0x1e, 0x6e, 0x32, 0xff, 0xd0, 0x52, 0x14, 0x84, 0xe3, 0x75, 0xd1, 0xbf, 0x51, + 0x60, 0x58, 0x17, 0x02, 0x9a, 0xe0, 0x6d, 0xcb, 0xbd, 0x5d, 0xea, 0xcf, 0xf9, 0xf2, 0x1e, 0x17, + 0xe4, 0x9f, 0xf1, 0xb9, 0xaa, 0x5f, 0x7c, 0x44, 0x62, 0x7c, 0xd0, 0x6b, 0xf4, 0x7b, 0x0a, 0x8c, + 0x6a, 0xa6, 0x69, 0xeb, 0x9a, 0xc7, 0xc2, 0x03, 0xf2, 0x17, 0xa5, 0xb7, 0x7a, 0x1c, 0xc5, 0x7c, + 0x88, 0x91, 0x0f, 0xe4, 0xbd, 0x81, 0x25, 0x20, 0x84, 0x1c, 0xd1, 0x58, 0xe4, 0xee, 0xa3, 0x7f, + 0xaa, 0xc0, 0x9b, 0xf8, 0x33, 0xde, 0x12, 0x95, 0xb9, 0xb6, 0x0c, 0x5d, 0xf3, 0x08, 0x8f, 0x59, + 0xea, 0x3f, 0x21, 0xe3, 0x3e, 0xf4, 0xc3, 0x87, 0xf6, 0xa1, 0xbf, 0xba, 0xdf, 0x2e, 0xbe, 0xa9, + 0xd4, 0x05, 0x6e, 0xdc, 0x55, 0x0f, 0xd0, 0x5d, 0x18, 0x37, 0xe5, 0xf8, 0xe3, 0x82, 0xc9, 0xe7, + 0xba, 0x7e, 0x8d, 0x04, 0x32, 0xe7, 0x06, 0x86, 0x48, 0x11, 0x8e, 0x92, 0x9a, 0xbd, 0x0d, 0xe3, + 0x91, 0x8d, 0x76, 0xac, 0x86, 0x6b, 0x0b, 0x26, 0xe3, 0xfb, 0xe1, 0x58, 0xf5, 0xbc, 0x9b, 0x30, + 0x12, 0x08, 0x0b, 0xe8, 0x92, 0x44, 0x28, 0x14, 0xbd, 0x6e, 0x92, 0x3d, 0x4e, 0xb5, 0x18, 0xb1, + 0xa1, 0x70, 0x23, 0x21, 0x8b, 0xde, 0x28, 0x10, 0xaa, 0x7f, 0x20, 0x6e, 0x55, 0xd7, 0x49, 0xa3, + 0x69, 0x6a, 0x1e, 0x79, 0xfd, 0xfb, 0x21, 0xa9, 0xff, 0x59, 0xe1, 0xe7, 0x0d, 0x17, 0x6d, 0x90, + 0x06, 0xa3, 0x0d, 0x9e, 0xfc, 0x8e, 0xc5, 0x7a, 0x54, 0xf2, 0x47, 0x99, 0x5c, 0x09, 0xd1, 0x60, + 0x19, 0x27, 0xba, 0x03, 0x23, 0xbe, 0x30, 0xe8, 0x5b, 0x6a, 0x96, 0x7a, 0x13, 0xce, 0x02, 0xb9, + 0x33, 0x38, 0x5a, 0xfd, 0x12, 0x17, 0x87, 0xb4, 0x54, 0x0d, 0x50, 0xb2, 0x0d, 0xba, 0x3f, 0x7c, + 0xcf, 0xa6, 0x44, 0xd3, 0xd5, 0x24, 0xde, 0xb4, 0xf9, 0x46, 0xd6, 0x42, 0x96, 0x91, 0x55, 0xfd, + 0xcd, 0x02, 0x4c, 0x0b, 0xd5, 0x73, 0x5e, 0xd7, 0xed, 0x96, 0xe5, 0x85, 0xcf, 0x8c, 0xf9, 0xdb, + 0x7d, 0x41, 0x84, 0x89, 0x93, 0xfc, 0x61, 0x3f, 0x16, 0x10, 0x74, 0x8b, 0x5b, 0x3f, 0xad, 0xda, + 0xba, 0x7d, 0x9b, 0x58, 0x21, 0x97, 0x90, 0x83, 0x64, 0x2c, 0xa6, 0x55, 0xc0, 0xe9, 0xed, 0xd0, + 0x0e, 0xa0, 0x86, 0xb6, 0x1b, 0xc7, 0x96, 0x2f, 0x09, 0x1a, 0xd3, 0x17, 0x57, 0x12, 0xd8, 0x70, + 0x0a, 0x05, 0x7a, 0x90, 0x52, 0x49, 0xae, 0xe9, 0x91, 0x1a, 0x1f, 0xa2, 0xef, 0xd4, 0xc1, 0x0e, + 0xd2, 0xf9, 0x28, 0x08, 0xc7, 0xeb, 0xaa, 0x1f, 0x1d, 0x84, 0xf3, 0xd1, 0x49, 0xa4, 0x5f, 0xa8, + 0xff, 0xbc, 0xfe, 0x49, 0xff, 0xed, 0x18, 0x9f, 0xc8, 0xfb, 0xe3, 0x6f, 0xc7, 0x66, 0x64, 0xa7, + 0x21, 0xd1, 0x28, 0xf2, 0x8e, 0xec, 0xeb, 0xf0, 0x56, 0x3e, 0xe3, 0x71, 0x74, 0xdf, 0xb1, 0xc6, + 0x04, 0xf8, 0x84, 0x02, 0xb3, 0xd1, 0xe2, 0x25, 0xc3, 0x32, 0xdc, 0x6d, 0x91, 0xf7, 0xe4, 0xf0, + 0x4f, 0xd7, 0x58, 0xe6, 0xe4, 0xe5, 0x4c, 0x8c, 0xb8, 0x03, 0x35, 0xf4, 0x7d, 0x0a, 0x5c, 0x88, + 0xcd, 0x4b, 0x24, 0x0b, 0xcb, 0xe1, 0x5f, 0xb1, 0xb1, 0xe0, 0x2e, 0xcb, 0xd9, 0x28, 0x71, 0x27, + 0x7a, 0x9d, 0x22, 0x7e, 0x0e, 0xbe, 0x56, 0x22, 0x7e, 0xfe, 0x62, 0x01, 0x06, 0x98, 0xdf, 0xd4, + 0xeb, 0xe3, 0xc1, 0x11, 0xeb, 0x6a, 0xa6, 0xbf, 0x6b, 0x3d, 0xe6, 0xef, 0xfa, 0x64, 0x7e, 0x12, + 0x9d, 0x1d, 0x5e, 0xdf, 0x0b, 0x67, 0x59, 0xb5, 0xf9, 0x1a, 0x33, 0xb4, 0xb9, 0xa4, 0x36, 0x5f, + 0xab, 0x31, 0xf5, 0xf6, 0xe0, 0x3b, 0xb2, 0x4b, 0xd0, 0xd7, 0x72, 0xcc, 0x78, 0x08, 0xd9, 0x0d, + 0xbc, 0x8c, 0x69, 0xb9, 0xfa, 0xc3, 0x05, 0x98, 0x64, 0xb8, 0x25, 0x16, 0x83, 0x76, 0x60, 0xd8, + 0x11, 0x6c, 0x46, 0xac, 0xcd, 0x72, 0xee, 0xa1, 0xa5, 0xb0, 0x2e, 0xae, 0xb1, 0xf9, 0xbf, 0x70, + 0x40, 0x0b, 0x7d, 0x8f, 0x02, 0x93, 0xc4, 0xd2, 0x9d, 0x3d, 0xf6, 0xa8, 0x7b, 0xde, 0xa3, 0x5a, + 0x73, 0x2f, 0x01, 0xab, 0x17, 0x63, 0xb8, 0x44, 0x6e, 0xc5, 0x58, 0x29, 0x4e, 0xd0, 0x54, 0xff, + 0x74, 0x00, 0x66, 0xb2, 0x7a, 0x8f, 0x7e, 0x48, 0x81, 0xb3, 0x7a, 0x28, 0xfa, 0xce, 0xb7, 0xbc, + 0x6d, 0xdb, 0xf1, 0x53, 0x3b, 0xe4, 0xb6, 0xcb, 0x94, 0xe6, 0x83, 0xe9, 0x61, 0xf9, 0x4c, 0x4a, + 0xa9, 0x14, 0x70, 0x06, 0x65, 0xf4, 0x0a, 0x80, 0xeb, 0x6e, 0xdf, 0x24, 0x7b, 0x4d, 0xcd, 0xf0, + 0x5d, 0xc8, 0x6e, 0xe6, 0xdf, 0x8f, 0xd5, 0x1b, 0x02, 0x55, 0xd0, 0x29, 0x6e, 0x34, 0x0f, 0xcb, + 0x25, 0x72, 0xe8, 0x23, 0x0a, 0x8c, 0xdb, 0x72, 0xe0, 0x98, 0x5e, 0x5e, 0x15, 0xa4, 0x46, 0xa0, + 0xe1, 0x22, 0x7f, 0x14, 0x14, 0x25, 0x49, 0x97, 0x65, 0xca, 0x8d, 0x1f, 0xb1, 0x82, 0x09, 0xaf, + 0xe4, 0x13, 0xc6, 0x32, 0xce, 0x6b, 0x6e, 0x3e, 0x48, 0x82, 0x93, 0xe4, 0x59, 0xa7, 0x88, 0xa7, + 0xd7, 0xc2, 0x3d, 0x47, 0x3b, 0x35, 0x98, 0xbf, 0x53, 0x8b, 0xeb, 0xa5, 0x72, 0x04, 0x59, 0xb4, + 0x53, 0x49, 0x70, 0x92, 0xbc, 0xfa, 0xe1, 0x02, 0x9c, 0x63, 0xcb, 0x7c, 0xb3, 0xb5, 0x49, 0xb8, + 0x17, 0xd8, 0x3f, 0xb8, 0x48, 0x3f, 0xbf, 0xa3, 0xc0, 0x08, 0x9b, 0x83, 0xd7, 0xc9, 0x63, 0x51, + 0xd6, 0xd7, 0x0c, 0x4f, 0xeb, 0xdf, 0x56, 0x60, 0x2a, 0x91, 0xcc, 0xaa, 0xab, 0xa7, 0x86, 0x27, + 0xe6, 0x04, 0xfc, 0xe6, 0x30, 0x81, 0x66, 0x5f, 0x18, 0x61, 0x23, 0x9e, 0x3c, 0x53, 0x7d, 0x16, + 0xc6, 0x23, 0x8e, 0xd6, 0x52, 0x2c, 0xca, 0xb4, 0x28, 0x9a, 0x72, 0xa8, 0xc9, 0x42, 0xa7, 0x20, + 0x99, 0xe1, 0x96, 0x4f, 0x72, 0xb6, 0x7f, 0x38, 0x5b, 0x1e, 0x89, 0x2d, 0xef, 0xdf, 0x27, 0xb1, + 0x08, 0x97, 0x6e, 0x2f, 0xf7, 0x49, 0x2c, 0x54, 0xa6, 0x88, 0x48, 0xcb, 0xff, 0xc7, 0x02, 0x2b, + 0x7a, 0x2a, 0x1a, 0x6f, 0x76, 0x35, 0x54, 0x32, 0xa7, 0xe3, 0x51, 0x62, 0xd9, 0x96, 0x4c, 0xd4, + 0x46, 0x98, 0xdf, 0x48, 0xf1, 0xb3, 0x2c, 0x57, 0x9a, 0x9d, 0xf2, 0x6a, 0x95, 0x07, 0x22, 0x0c, + 0x6e, 0xa2, 0xa2, 0xa6, 0xf7, 0xfe, 0x93, 0x30, 0xbd, 0x3b, 0x30, 0xba, 0x6d, 0x6c, 0x12, 0xc7, + 0xe2, 0xf2, 0xd4, 0x40, 0x7e, 0x51, 0xf1, 0x46, 0x88, 0x86, 0xdb, 0x23, 0xa4, 0x02, 0x2c, 0x13, + 0x41, 0x4e, 0x24, 0x6a, 0xf6, 0x60, 0x7e, 0xa9, 0x24, 0xb4, 0x91, 0x87, 0xe3, 0xcc, 0x88, 0x98, + 0x6d, 0x01, 0x58, 0x41, 0x28, 0xd9, 0x5e, 0x6e, 0xa8, 0xc2, 0x80, 0xb4, 0x5c, 0xe8, 0x08, 0x7f, + 0x63, 0x89, 0x02, 0x9d, 0xd7, 0x46, 0x98, 0x59, 0x41, 0xd8, 0x3b, 0x9f, 0xec, 0x31, 0xbb, 0x85, + 0xb0, 0xf3, 0x84, 0x05, 0x58, 0x26, 0x42, 0xc7, 0xd8, 0x08, 0xf2, 0x09, 0x08, 0x7b, 0x66, 0xae, + 0x31, 0x86, 0x59, 0x09, 0xf8, 0x18, 0xc3, 0xdf, 0x58, 0xa2, 0x80, 0x5e, 0x92, 0x2e, 0x32, 0x21, + 0xbf, 0xb5, 0xac, 0xab, 0x4b, 0xcc, 0x77, 0x84, 0x46, 0xa3, 0x51, 0xf6, 0x9d, 0x5e, 0x90, 0x0c, + 0x46, 0x2c, 0xcf, 0x02, 0xe5, 0x1d, 0x09, 0x03, 0x52, 0xf8, 0xbc, 0x63, 0xac, 0xe3, 0xf3, 0x8e, + 0x12, 0x95, 0xce, 0xa4, 0x27, 0x92, 0x8c, 0x21, 0x8c, 0x87, 0xb7, 0x31, 0xd5, 0x38, 0x10, 0x27, + 0xeb, 0x73, 0x86, 0x4f, 0x6a, 0xac, 0xed, 0x84, 0xcc, 0xf0, 0x79, 0x19, 0x0e, 0xa0, 0x68, 0x07, + 0xc6, 0x5c, 0xe9, 0xad, 0xc8, 0xcc, 0xa9, 0x5e, 0xef, 0x32, 0xc5, 0x3b, 0x11, 0xe6, 0x94, 0x29, + 0x97, 0xe0, 0x08, 0x9d, 0xe8, 0x45, 0xdb, 0xe4, 0xc9, 0x5e, 0xb4, 0xa1, 0x56, 0xd4, 0x0d, 0x7c, + 0xea, 0x48, 0x02, 0xca, 0x1c, 0xe8, 0x26, 0x4e, 0x97, 0x96, 0xec, 0x36, 0x6d, 0xb7, 0xe5, 0x10, + 0xe6, 0x7f, 0xca, 0x96, 0x07, 0x85, 0x4b, 0xbb, 0x18, 0x07, 0xe2, 0x64, 0x7d, 0xa6, 0xfb, 0xb9, + 0x7b, 0xae, 0x47, 0x1a, 0xf4, 0xd8, 0xb2, 0x2d, 0x62, 0x79, 0xee, 0xcc, 0xe9, 0xfc, 0xba, 0x5f, + 0x35, 0x86, 0x8b, 0x1f, 0x3b, 0xf1, 0x52, 0x9c, 0xa0, 0x49, 0x77, 0x8e, 0x1c, 0x92, 0x66, 0x66, + 0x3a, 0xff, 0xce, 0x91, 0xc3, 0xdd, 0xf0, 0x9d, 0x23, 0x97, 0xe0, 0x08, 0x1d, 0xf4, 0x08, 0x8c, + 0xbb, 0x7e, 0x32, 0x75, 0x36, 0x83, 0x67, 0xc2, 0xa8, 0xa1, 0x55, 0x19, 0x80, 0xa3, 0xf5, 0xd0, + 0x07, 0x61, 0x4c, 0x3e, 0x3b, 0x67, 0xce, 0x1e, 0x75, 0xfc, 0x7c, 0xde, 0x73, 0x19, 0x14, 0x21, + 0x88, 0x30, 0x9c, 0x95, 0x5e, 0x31, 0xca, 0xdf, 0xf7, 0x39, 0x36, 0x04, 0xae, 0xcf, 0xa6, 0xd6, + 0xc0, 0x19, 0x2d, 0xd1, 0x8f, 0xa7, 0xdf, 0xdb, 0xcf, 0xb0, 0x2d, 0xbd, 0x76, 0x24, 0xf7, 0xf6, + 0xcf, 0x1a, 0xde, 0xf6, 0xad, 0x26, 0x0f, 0x71, 0x76, 0xc8, 0x2b, 0x7c, 0xf5, 0x8f, 0x14, 0x80, + 0xc0, 0x72, 0x73, 0x12, 0x77, 0x26, 0xb5, 0x88, 0x31, 0x6b, 0xa1, 0x27, 0x4b, 0x53, 0x66, 0x7a, + 0x14, 0xf5, 0xcb, 0x0a, 0x4c, 0x84, 0xd5, 0x4e, 0x40, 0x35, 0xd2, 0xa3, 0xaa, 0xd1, 0x13, 0xbd, + 0x8d, 0x2b, 0x43, 0x3f, 0xfa, 0x3f, 0x05, 0x79, 0x54, 0x4c, 0xfa, 0xdd, 0x89, 0xf8, 0x20, 0x50, + 0xd2, 0x37, 0x7a, 0xf1, 0x41, 0x90, 0xc3, 0x80, 0x84, 0xe3, 0x4d, 0xf1, 0x49, 0xf8, 0xce, 0x88, + 0xfc, 0xd9, 0x43, 0x00, 0x9e, 0x40, 0xd8, 0xf4, 0x49, 0xf3, 0x09, 0x38, 0x48, 0x18, 0x7d, 0x59, + 0x3e, 0x9e, 0x7a, 0x48, 0x69, 0x12, 0x19, 0x70, 0x67, 0xef, 0x8f, 0x5f, 0x9c, 0x82, 0x51, 0xc9, + 0xc8, 0x19, 0xf3, 0xa8, 0x50, 0x4e, 0xc2, 0xa3, 0xc2, 0x83, 0x51, 0x3d, 0xc8, 0x69, 0xe9, 0x4f, + 0x7b, 0x8f, 0x34, 0x83, 0x63, 0x31, 0xcc, 0x96, 0xe9, 0x62, 0x99, 0x0c, 0x15, 0xde, 0x82, 0x3d, + 0xd6, 0x77, 0x04, 0x7e, 0x2e, 0x9d, 0xf6, 0xd5, 0xc3, 0x00, 0xbe, 0xfc, 0x4f, 0x6a, 0x22, 0x54, + 0x7c, 0xe0, 0xc3, 0x5d, 0x71, 0x6f, 0x04, 0x30, 0x2c, 0xd5, 0x4b, 0xde, 0xd0, 0x0f, 0x9c, 0xd8, + 0x0d, 0x3d, 0xdd, 0x06, 0xa6, 0x9f, 0xfd, 0xbf, 0x27, 0xbf, 0xb9, 0x65, 0x1f, 0x4b, 0xb8, 0x0d, + 0x82, 0x22, 0x17, 0x4b, 0x44, 0x32, 0x1c, 0x6b, 0x86, 0x72, 0x39, 0xd6, 0xb4, 0xe0, 0xb4, 0x43, + 0x3c, 0x67, 0xaf, 0xb4, 0xa7, 0xb3, 0x1c, 0x2b, 0x8e, 0xc7, 0x34, 0xf8, 0xe1, 0x7c, 0x91, 0x1b, + 0x71, 0x12, 0x15, 0x4e, 0xc3, 0x1f, 0x11, 0x80, 0x47, 0x3a, 0x0a, 0xc0, 0xef, 0x80, 0x51, 0x8f, + 0xe8, 0xdb, 0x96, 0xa1, 0x6b, 0x66, 0xa5, 0x2c, 0x02, 0x89, 0x87, 0xb2, 0x5c, 0x08, 0xc2, 0x72, + 0x3d, 0xb4, 0x00, 0x7d, 0x2d, 0xa3, 0x26, 0x34, 0x80, 0x6f, 0x09, 0xae, 0x0b, 0x2a, 0xe5, 0x7b, + 0xed, 0xe2, 0x1b, 0x43, 0x4f, 0x95, 0x60, 0x54, 0xd7, 0x9a, 0xb7, 0xeb, 0xd7, 0xbc, 0xbd, 0x26, + 0x71, 0xe7, 0x36, 0x2a, 0x65, 0x4c, 0x1b, 0xa7, 0x39, 0x1d, 0x8d, 0x1d, 0xc2, 0xe9, 0xe8, 0xd3, + 0x0a, 0x9c, 0xd6, 0xe2, 0x37, 0x1d, 0xc4, 0x9d, 0x19, 0xcf, 0xcf, 0x2d, 0xd3, 0x6f, 0x4f, 0x16, + 0x2e, 0x88, 0xf1, 0x9d, 0x9e, 0x4f, 0x92, 0xc3, 0x69, 0x7d, 0x40, 0x0e, 0xa0, 0x86, 0x51, 0x0f, + 0x12, 0xf1, 0x8b, 0x55, 0x9f, 0xc8, 0x67, 0xb7, 0x59, 0x49, 0x60, 0xc2, 0x29, 0xd8, 0xd1, 0x1d, + 0x18, 0x95, 0x84, 0x24, 0xa1, 0xc9, 0x94, 0x8f, 0xe2, 0x42, 0x86, 0x6b, 0xbb, 0xf2, 0x1d, 0x87, + 0x4c, 0x29, 0xb8, 0x6d, 0x95, 0xcc, 0x0c, 0xe2, 0xc6, 0x91, 0x8d, 0x7a, 0x32, 0xff, 0x6d, 0x6b, + 0x3a, 0x46, 0xdc, 0x81, 0x1a, 0x8b, 0x97, 0x48, 0xc1, 0x92, 0x6e, 0x3e, 0x33, 0x95, 0x3f, 0xe0, + 0xcb, 0x72, 0x14, 0x15, 0xdf, 0x9a, 0xb1, 0x42, 0x1c, 0x27, 0x88, 0x96, 0x00, 0x89, 0xbb, 0xa2, + 0x50, 0x39, 0x73, 0x67, 0x10, 0x73, 0x04, 0x60, 0x4b, 0xba, 0x98, 0x80, 0xe2, 0x94, 0x16, 0xc8, + 0x8b, 0xd8, 0x4a, 0x7a, 0xd0, 0x72, 0xe2, 0xc9, 0x7b, 0x3a, 0x5a, 0x4c, 0xbe, 0x5b, 0x49, 0x64, + 0x73, 0xe6, 0xca, 0xcd, 0x8d, 0xde, 0xb3, 0x39, 0x0b, 0xf2, 0xdd, 0xe4, 0x74, 0xfe, 0x9c, 0x02, + 0xe7, 0x1a, 0xe9, 0x59, 0x2b, 0x99, 0xce, 0x93, 0xf3, 0xe2, 0x2a, 0x23, 0x11, 0x26, 0x33, 0x63, + 0x64, 0x65, 0xc9, 0xc4, 0x59, 0x1d, 0x51, 0xff, 0x50, 0x11, 0xa6, 0xe8, 0x13, 0xf4, 0x8b, 0x3a, + 0xee, 0x0b, 0x6b, 0xf5, 0x59, 0x98, 0xa9, 0xfa, 0xd1, 0x4e, 0x6b, 0xb1, 0xa4, 0x06, 0xef, 0x86, + 0x71, 0x7e, 0x15, 0xb4, 0xa2, 0x35, 0x57, 0xc3, 0x7b, 0x83, 0x20, 0x2e, 0x46, 0x49, 0x06, 0xe2, + 0x68, 0x5d, 0xf5, 0xab, 0x0a, 0x9c, 0x8b, 0x62, 0xb6, 0x1d, 0xe3, 0x6e, 0xef, 0x88, 0xd1, 0xc7, + 0x14, 0x18, 0xbd, 0x1d, 0xdc, 0x52, 0xf9, 0xa2, 0x5b, 0xae, 0xb7, 0x23, 0x7e, 0xaf, 0x88, 0x23, + 0x5d, 0x7b, 0x25, 0x33, 0x65, 0x86, 0x40, 0x17, 0xcb, 0xa4, 0xd5, 0xff, 0xae, 0x40, 0xc2, 0x7c, + 0x80, 0x36, 0x61, 0x88, 0x12, 0x29, 0xaf, 0x56, 0xc5, 0x9e, 0x78, 0x77, 0x3e, 0xa9, 0x92, 0xa1, + 0xe0, 0x97, 0x22, 0xe2, 0x07, 0xf6, 0x11, 0xa3, 0x1d, 0x1e, 0x88, 0xc0, 0xcf, 0x78, 0x24, 0xb6, + 0x47, 0x2e, 0xb1, 0x5d, 0xce, 0x9c, 0xc4, 0xd5, 0x7a, 0xb9, 0x04, 0x47, 0xe8, 0xa8, 0xcb, 0x00, + 0xa1, 0xc9, 0xa7, 0x67, 0x3f, 0xc3, 0xff, 0x70, 0x1a, 0xce, 0xf4, 0xfc, 0x2a, 0xf2, 0x23, 0x0a, + 0x9c, 0x25, 0x3b, 0x86, 0xee, 0xcd, 0x6f, 0x79, 0xc4, 0xb9, 0x75, 0x6b, 0x65, 0x7d, 0xdb, 0x21, + 0xee, 0xb6, 0x6d, 0xd6, 0xba, 0xf1, 0xaa, 0x4c, 0x71, 0x01, 0x63, 0xa6, 0x89, 0xc5, 0x54, 0x8c, + 0x38, 0x83, 0x12, 0x33, 0x77, 0xed, 0x70, 0x43, 0x00, 0xa6, 0x3a, 0x57, 0xcb, 0x71, 0x3d, 0x11, + 0xc0, 0x90, 0x9b, 0xbb, 0xe2, 0x40, 0x9c, 0xac, 0x1f, 0x47, 0xc2, 0xd2, 0xf9, 0x31, 0xb9, 0x5d, + 0x49, 0x22, 0x61, 0x40, 0x9c, 0xac, 0x2f, 0x23, 0xe1, 0x2b, 0x45, 0x0f, 0xc5, 0x81, 0x24, 0x92, + 0x00, 0x88, 0x93, 0xf5, 0x51, 0x0d, 0x2e, 0x3a, 0x44, 0xb7, 0x1b, 0x0d, 0x62, 0xd5, 0xd8, 0xa4, + 0xac, 0x68, 0x4e, 0xdd, 0xb0, 0x96, 0x1c, 0x8d, 0x55, 0x64, 0xb7, 0x07, 0x0a, 0xcb, 0xbc, 0x7b, + 0x11, 0x77, 0xa8, 0x87, 0x3b, 0x62, 0x41, 0x0d, 0x38, 0xd5, 0x62, 0xfc, 0xdf, 0x61, 0x01, 0xce, + 0x76, 0x34, 0x53, 0x5c, 0x11, 0x1c, 0x76, 0xc5, 0xd8, 0x41, 0xbd, 0x11, 0x45, 0x85, 0xe3, 0xb8, + 0xd1, 0x1e, 0x15, 0xcf, 0x45, 0x77, 0x24, 0x92, 0xc3, 0xb9, 0x48, 0x0a, 0x11, 0x3d, 0x81, 0x0e, + 0xa7, 0xd1, 0x40, 0x15, 0x38, 0xed, 0x69, 0x4e, 0x9d, 0x78, 0xa5, 0xb5, 0x8d, 0x35, 0xe2, 0xe8, + 0x94, 0xc7, 0x9a, 0x5c, 0x5a, 0x57, 0x38, 0xaa, 0xf5, 0x24, 0x18, 0xa7, 0xb5, 0x41, 0x1f, 0x84, + 0x37, 0x47, 0x27, 0x75, 0xd9, 0xbe, 0x43, 0x9c, 0x05, 0xbb, 0x65, 0xd5, 0xa2, 0xc8, 0x81, 0x21, + 0xbf, 0x7f, 0xbf, 0x5d, 0x7c, 0x33, 0xee, 0xa6, 0x01, 0xee, 0x0e, 0x6f, 0xb2, 0x03, 0x1b, 0xcd, + 0x66, 0x6a, 0x07, 0x46, 0xb3, 0x3a, 0x90, 0xd1, 0x00, 0x77, 0x87, 0x17, 0x61, 0x38, 0xcb, 0x27, + 0x86, 0xe7, 0x89, 0x96, 0x28, 0x8e, 0x31, 0x8a, 0xec, 0xfb, 0x5d, 0x4f, 0xad, 0x81, 0x33, 0x5a, + 0xd2, 0x33, 0xe5, 0x6a, 0xd6, 0xf0, 0x13, 0x64, 0xc6, 0x19, 0x99, 0xb7, 0xee, 0xb7, 0x8b, 0x57, + 0x71, 0x97, 0x6d, 0x70, 0xd7, 0xd8, 0x53, 0xba, 0x12, 0x4e, 0x44, 0xa2, 0x2b, 0x13, 0x59, 0x5d, + 0xc9, 0x6e, 0x83, 0xbb, 0xc6, 0x8e, 0xbe, 0x57, 0x81, 0xf3, 0x7a, 0xb3, 0x75, 0xc3, 0x70, 0x3d, + 0xbb, 0xee, 0x68, 0x8d, 0x32, 0xd1, 0xb5, 0xbd, 0x1b, 0x9a, 0xb9, 0xb5, 0x6c, 0x6c, 0x11, 0xa1, + 0x74, 0x1c, 0xf6, 0xc3, 0x61, 0x2f, 0x80, 0x4b, 0x6b, 0x1b, 0xe9, 0x48, 0x71, 0x36, 0x3d, 0xf4, + 0x23, 0x0a, 0x5c, 0x6c, 0xb0, 0x2e, 0x66, 0x74, 0x68, 0x32, 0x57, 0x87, 0x18, 0x17, 0x5b, 0xe9, + 0x80, 0x17, 0x77, 0xa4, 0xca, 0x26, 0x89, 0x57, 0x98, 0xaf, 0xd7, 0x1d, 0x52, 0x67, 0x58, 0x03, + 0xee, 0x32, 0x95, 0x7f, 0x92, 0x56, 0xb2, 0x90, 0xe2, 0x6c, 0x7a, 0xe8, 0x25, 0xb8, 0x9c, 0x09, + 0x2c, 0xd9, 0x2d, 0xcb, 0x63, 0x97, 0x30, 0x7d, 0x0b, 0xea, 0x7e, 0xbb, 0x78, 0x79, 0xa5, 0x63, + 0x4d, 0x7c, 0x00, 0x26, 0x16, 0x04, 0x25, 0x12, 0x8f, 0xe0, 0x34, 0x93, 0xc4, 0x5e, 0xc8, 0x95, + 0x17, 0xf8, 0x08, 0x83, 0x10, 0x7c, 0x39, 0x1a, 0x84, 0x60, 0x9a, 0xf5, 0xea, 0xbd, 0x47, 0xd7, + 0xab, 0x6f, 0x44, 0x1e, 0xe8, 0xe6, 0x45, 0xca, 0xa7, 0x15, 0x10, 0x0f, 0x49, 0xd1, 0xc5, 0x88, + 0xdb, 0xd1, 0x70, 0xcc, 0xe5, 0xc8, 0x4f, 0xcf, 0x5b, 0x48, 0x4d, 0xcf, 0xfb, 0x16, 0x29, 0x1a, + 0xf4, 0x48, 0xa8, 0x2c, 0x71, 0xcc, 0x61, 0x38, 0x68, 0xf4, 0x00, 0x8c, 0x04, 0x4a, 0xb5, 0x30, + 0x76, 0xb2, 0xd4, 0x38, 0xa1, 0xf6, 0x1d, 0xc2, 0xd5, 0xdf, 0x57, 0x00, 0xc2, 0xac, 0xd3, 0xe8, + 0x3e, 0x3f, 0x6a, 0x0e, 0xef, 0x60, 0x70, 0x63, 0x20, 0x47, 0xce, 0x39, 0xf8, 0x55, 0x04, 0x52, + 0x61, 0xb0, 0xc5, 0xf2, 0x6c, 0x8a, 0x97, 0x0c, 0xcc, 0x05, 0x66, 0x83, 0x95, 0x60, 0x01, 0x41, + 0x1b, 0x30, 0xd4, 0x30, 0x2c, 0xf6, 0xe8, 0xa4, 0x3f, 0xd7, 0xa3, 0x13, 0xa6, 0x0f, 0xac, 0x70, + 0x14, 0xd8, 0xc7, 0xa5, 0xfe, 0x92, 0x02, 0xa7, 0xa2, 0xe1, 0xb9, 0x59, 0xec, 0x1d, 0x11, 0xb5, + 0x55, 0x64, 0x05, 0x60, 0x4d, 0x45, 0x34, 0x4a, 0xec, 0xc3, 0xa2, 0xb7, 0xd3, 0x3d, 0xdc, 0x3e, + 0xa4, 0x47, 0x09, 0x3f, 0xe0, 0x22, 0xe0, 0xc7, 0x4e, 0xc3, 0x20, 0x57, 0xb4, 0xa9, 0x1c, 0x9f, + 0x12, 0x85, 0xeb, 0x66, 0xfe, 0xc4, 0x17, 0x79, 0x42, 0xf3, 0xc8, 0xe9, 0x3f, 0x0b, 0x1d, 0xd3, + 0x7f, 0x62, 0xe8, 0xd3, 0x1d, 0xa3, 0x17, 0x4f, 0xa4, 0x12, 0xae, 0x70, 0x4f, 0xa4, 0x12, 0xae, + 0x60, 0x8a, 0x0c, 0x79, 0x11, 0x17, 0x9d, 0xfe, 0xfc, 0x26, 0x20, 0x3e, 0x01, 0x92, 0xa3, 0xce, + 0x44, 0x47, 0x27, 0x1d, 0x3f, 0xe4, 0xff, 0x40, 0xfe, 0x57, 0x4a, 0x62, 0xca, 0xbb, 0x09, 0xf9, + 0xef, 0x7f, 0x48, 0x83, 0x99, 0x1f, 0xd2, 0x16, 0x0c, 0x89, 0x4f, 0x41, 0x28, 0x04, 0xef, 0xee, + 0x21, 0xc9, 0xbd, 0x94, 0x4e, 0x8b, 0x17, 0x60, 0x1f, 0x39, 0xd5, 0x32, 0x1b, 0xda, 0xae, 0xd1, + 0x68, 0x35, 0x98, 0x16, 0x30, 0x20, 0x57, 0x65, 0xc5, 0xd8, 0x87, 0xb3, 0xaa, 0xfc, 0x71, 0x17, + 0x93, 0xda, 0xe5, 0xaa, 0xbc, 0x18, 0xfb, 0x70, 0xf4, 0x3c, 0x0c, 0x37, 0xb4, 0xdd, 0x6a, 0xcb, + 0xa9, 0x13, 0xe1, 0xa0, 0x93, 0x6d, 0x14, 0x6a, 0x79, 0x86, 0x39, 0x67, 0x58, 0x9e, 0xeb, 0x39, + 0x73, 0x15, 0xcb, 0xbb, 0xe5, 0x54, 0x3d, 0xe6, 0x00, 0xc4, 0x76, 0xdd, 0x8a, 0xc0, 0x82, 0x03, + 0x7c, 0xc8, 0x84, 0x89, 0x86, 0xb6, 0xbb, 0x61, 0x69, 0x3c, 0x9b, 0x83, 0x90, 0xb2, 0xf3, 0x50, + 0x60, 0x56, 0xb9, 0x95, 0x08, 0x2e, 0x1c, 0xc3, 0x9d, 0xe2, 0x0c, 0x3a, 0x76, 0x5c, 0xce, 0xa0, + 0xf3, 0x41, 0xb8, 0x04, 0x6e, 0xd2, 0x3f, 0x9f, 0x1a, 0xa8, 0xb0, 0x63, 0x28, 0x84, 0x17, 0x83, + 0x50, 0x08, 0x13, 0xf9, 0xbd, 0x17, 0x3b, 0x84, 0x41, 0x68, 0xc1, 0x68, 0x4d, 0xf3, 0x34, 0x5e, + 0xea, 0xce, 0x9c, 0xca, 0x7f, 0x3b, 0x5d, 0x0e, 0xd0, 0x84, 0x2c, 0x29, 0x2c, 0x73, 0xb1, 0x4c, + 0x07, 0xdd, 0x82, 0x33, 0xf4, 0x63, 0x35, 0x89, 0x17, 0x56, 0x61, 0x36, 0xb3, 0x49, 0xf6, 0xfd, + 0xb0, 0xe7, 0x72, 0x37, 0xd3, 0x2a, 0xe0, 0xf4, 0x76, 0x61, 0x50, 0xdd, 0xa9, 0xf4, 0xa0, 0xba, + 0xe8, 0x93, 0x69, 0x6e, 0x37, 0x88, 0xcd, 0xe9, 0x7b, 0xf2, 0xf3, 0x86, 0xdc, 0xce, 0x37, 0xff, + 0x5a, 0x81, 0x19, 0xb1, 0xcb, 0x84, 0xab, 0x8c, 0x49, 0x9c, 0x15, 0xcd, 0xd2, 0xea, 0xc4, 0x11, + 0x76, 0xf2, 0xf5, 0x1e, 0xf8, 0x43, 0x02, 0x67, 0x10, 0xa3, 0xe2, 0x4d, 0xfb, 0xed, 0xe2, 0x95, + 0x83, 0x6a, 0xe1, 0xcc, 0xbe, 0x21, 0x07, 0x86, 0xdc, 0x3d, 0x57, 0xf7, 0x4c, 0x57, 0xc8, 0xa0, + 0xd7, 0x7b, 0xe0, 0xac, 0x55, 0x8e, 0x89, 0xb3, 0xd6, 0x30, 0x89, 0x23, 0x2f, 0xc5, 0x3e, 0x21, + 0xf4, 0x83, 0x0a, 0x4c, 0x89, 0xcb, 0x33, 0x29, 0x06, 0xd0, 0x99, 0xfc, 0x0f, 0x76, 0x4a, 0x71, + 0x64, 0xbe, 0x7b, 0x0c, 0xb3, 0x26, 0x25, 0xa0, 0x38, 0x49, 0x9d, 0x1e, 0xaa, 0x4d, 0xc7, 0xb0, + 0x1d, 0xc3, 0xdb, 0x63, 0x8e, 0x48, 0x03, 0x7e, 0x28, 0x7d, 0x5e, 0x86, 0x03, 0x28, 0xaa, 0xc2, + 0x04, 0xb7, 0xda, 0x54, 0x3d, 0x47, 0xf3, 0x48, 0x7d, 0x4f, 0x78, 0x0b, 0x3d, 0xc0, 0x92, 0x1d, + 0x47, 0x20, 0xf7, 0xda, 0xc5, 0x33, 0x62, 0x6d, 0xa2, 0x00, 0x1c, 0x43, 0x81, 0xde, 0x1f, 0x73, + 0xde, 0x9a, 0xc9, 0x9f, 0x08, 0x91, 0xaf, 0xc5, 0x61, 0x5c, 0xb8, 0x7a, 0x0d, 0x6b, 0xd7, 0x43, + 0x32, 0x9a, 0xd9, 0xc7, 0x60, 0x4c, 0xde, 0x35, 0x87, 0x8a, 0xa6, 0xe7, 0x01, 0x4a, 0x0e, 0xf6, + 0xb8, 0x03, 0x0e, 0xa9, 0x3f, 0xad, 0xc0, 0x64, 0x5c, 0x76, 0x41, 0xdb, 0x30, 0x24, 0x18, 0x99, + 0xa0, 0x3a, 0x9f, 0xd7, 0x6b, 0xd9, 0x24, 0xe2, 0x9d, 0x32, 0x17, 0x85, 0x45, 0x11, 0xf6, 0xd1, + 0xcb, 0x2f, 0x12, 0x0a, 0x1d, 0x5e, 0x24, 0x3c, 0x0e, 0x67, 0xd3, 0x59, 0x1a, 0x55, 0x24, 0x34, + 0xaa, 0x5f, 0x09, 0xa3, 0x75, 0xa0, 0x48, 0x30, 0xa5, 0x0b, 0x73, 0x98, 0xfa, 0x01, 0x88, 0x27, + 0x61, 0x43, 0x2f, 0xc1, 0x88, 0xeb, 0x6e, 0x73, 0x3f, 0x33, 0x31, 0xc8, 0x7c, 0x57, 0x3d, 0x7e, + 0x42, 0x1c, 0xae, 0xfb, 0x04, 0x3f, 0x71, 0x88, 0x7e, 0xe1, 0xb9, 0x2f, 0x7e, 0xf5, 0xf2, 0x1b, + 0xfe, 0xe0, 0xab, 0x97, 0xdf, 0xf0, 0x95, 0xaf, 0x5e, 0x7e, 0xc3, 0x87, 0xf6, 0x2f, 0x2b, 0x5f, + 0xdc, 0xbf, 0xac, 0xfc, 0xc1, 0xfe, 0x65, 0xe5, 0x2b, 0xfb, 0x97, 0x95, 0xbf, 0xdc, 0xbf, 0xac, + 0xfc, 0xc0, 0x7f, 0xbc, 0xfc, 0x86, 0xe7, 0x1f, 0x0a, 0xa9, 0x5f, 0xf3, 0x89, 0x86, 0xff, 0x34, + 0x6f, 0xd7, 0xa9, 0xd2, 0xea, 0xfa, 0x5a, 0x2b, 0xa3, 0xfe, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, + 0xd6, 0x36, 0x3d, 0x82, 0xf8, 0x27, 0x01, 0x00, } func (m *APIServerLogging) Marshal() (dAtA []byte, err error) { @@ -10198,6 +10229,38 @@ func (m *ETCDEncryptionKeyRotation) MarshalToSizedBuffer(dAtA []byte) (int, erro return len(dAtA) - i, nil } +func (m *EncryptionAtRest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *EncryptionAtRest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *EncryptionAtRest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Resources) > 0 { + for iNdEx := len(m.Resources) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Resources[iNdEx]) + copy(dAtA[i:], m.Resources[iNdEx]) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Resources[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *EncryptionConfig) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -17139,6 +17202,18 @@ func (m *ShootCredentials) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.EncryptionAtRest != nil { + { + size, err := m.EncryptionAtRest.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } if m.Rotation != nil { { size, err := m.Rotation.MarshalToSizedBuffer(dAtA[:i]) @@ -20459,6 +20534,21 @@ func (m *ETCDEncryptionKeyRotation) Size() (n int) { return n } +func (m *EncryptionAtRest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Resources) > 0 { + for _, s := range m.Resources { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } + return n +} + func (m *EncryptionConfig) Size() (n int) { if m == nil { return 0 @@ -23055,6 +23145,10 @@ func (m *ShootCredentials) Size() (n int) { l = m.Rotation.Size() n += 1 + l + sovGenerated(uint64(l)) } + if m.EncryptionAtRest != nil { + l = m.EncryptionAtRest.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -24670,6 +24764,16 @@ func (this *ETCDEncryptionKeyRotation) String() string { }, "") return s } +func (this *EncryptionAtRest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&EncryptionAtRest{`, + `Resources:` + fmt.Sprintf("%v", this.Resources) + `,`, + `}`, + }, "") + return s +} func (this *EncryptionConfig) String() string { if this == nil { return "nil" @@ -26503,6 +26607,7 @@ func (this *ShootCredentials) String() string { } s := strings.Join([]string{`&ShootCredentials{`, `Rotation:` + strings.Replace(this.Rotation.String(), "ShootCredentialsRotation", "ShootCredentialsRotation", 1) + `,`, + `EncryptionAtRest:` + strings.Replace(this.EncryptionAtRest.String(), "EncryptionAtRest", "EncryptionAtRest", 1) + `,`, `}`, }, "") return s @@ -36475,6 +36580,88 @@ func (m *ETCDEncryptionKeyRotation) Unmarshal(dAtA []byte) error { } return nil } +func (m *EncryptionAtRest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: EncryptionAtRest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: EncryptionAtRest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Resources", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Resources = append(m.Resources, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *EncryptionConfig) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -57339,6 +57526,42 @@ func (m *ShootCredentials) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EncryptionAtRest", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.EncryptionAtRest == nil { + m.EncryptionAtRest = &EncryptionAtRest{} + } + if err := m.EncryptionAtRest.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/core/v1beta1/generated.proto b/pkg/apis/core/v1beta1/generated.proto index a71d51c072f..7453b6cc1ac 100644 --- a/pkg/apis/core/v1beta1/generated.proto +++ b/pkg/apis/core/v1beta1/generated.proto @@ -1013,6 +1013,15 @@ message ETCDEncryptionKeyRotation { optional bool autoCompleteAfterPrepared = 6; } +// EncryptionAtRest contains information about Shoot data encryption at rest. +message EncryptionAtRest { + // Resources is the list of resources in the Shoot which are currently encrypted. + // Secrets are encrypted by default and are not part of the list. + // See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details. + // +optional + repeated string resources = 1; +} + // EncryptionConfig contains customizable encryption configuration of the API server. message EncryptionConfig { // Resources contains the list of resources that shall be encrypted in addition to secrets. @@ -3187,6 +3196,10 @@ message ShootCredentials { // Rotation contains information about the credential rotations. // +optional optional ShootCredentialsRotation rotation = 1; + + // EncryptionAtRest contains information about Shoot data encryption at rest. + // +optional + optional EncryptionAtRest encryptionAtRest = 2; } // ShootCredentialsRotation contains information about the rotation of credentials. @@ -3503,6 +3516,9 @@ message ShootStatus { // EncryptedResources is the list of resources in the Shoot which are currently encrypted. // Secrets are encrypted by default and are not part of the list. // See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details. + // + // Deprecated: This field is deprecated and will be removed with release v1.138. + // This field will be removed in favor of `status.credentials.encryptionAtRest.resources`. // +optional repeated string encryptedResources = 18; diff --git a/pkg/apis/core/v1beta1/helper/shoot.go b/pkg/apis/core/v1beta1/helper/shoot.go index a18b5a2fc53..7f23df97ede 100644 --- a/pkg/apis/core/v1beta1/helper/shoot.go +++ b/pkg/apis/core/v1beta1/helper/shoot.go @@ -748,3 +748,12 @@ func IsOneWorkerPoolLowerKubernetes134(controlPlaneVersion *semver.Version, work } return atLeastOnePoolLowerKubernetes134, nil } + +// GetShootEncryptedResourcesInStatus returns the encrypted resources from the shoot status. +func GetShootEncryptedResourcesInStatus(shootStatus gardencorev1beta1.ShootStatus) []string { + if shootStatus.Credentials != nil && shootStatus.Credentials.EncryptionAtRest != nil { + return shootStatus.Credentials.EncryptionAtRest.Resources + } + + return nil +} diff --git a/pkg/apis/core/v1beta1/helper/shoot_test.go b/pkg/apis/core/v1beta1/helper/shoot_test.go index 04886592f3a..aa2ed3772df 100644 --- a/pkg/apis/core/v1beta1/helper/shoot_test.go +++ b/pkg/apis/core/v1beta1/helper/shoot_test.go @@ -1629,4 +1629,13 @@ var _ = Describe("Helper", func() { Entry("with control plane version 1.34 and one worker pool with lower version", semver.MustParse("1.34.0"), []gardencorev1beta1.Worker{{Kubernetes: &gardencorev1beta1.WorkerKubernetes{Version: ptr.To("1.31.0")}}}, true), Entry("with control plane version 1.34 and one worker pool with lower version", semver.MustParse("1.34.0"), []gardencorev1beta1.Worker{{Kubernetes: &gardencorev1beta1.WorkerKubernetes{Version: ptr.To("1.34.0")}}}, false), ) + + DescribeTable("#GetShootEncryptedResourcesInStatus", + func(status gardencorev1beta1.ShootStatus, expected []string) { + Expect(GetShootEncryptedResourcesInStatus(status)).To(Equal(expected)) + }, + Entry("no credentials field", gardencorev1beta1.ShootStatus{}, nil), + Entry("without resources", gardencorev1beta1.ShootStatus{Credentials: &gardencorev1beta1.ShootCredentials{}}, nil), + Entry("with resources", gardencorev1beta1.ShootStatus{Credentials: &gardencorev1beta1.ShootCredentials{EncryptionAtRest: &gardencorev1beta1.EncryptionAtRest{Resources: []string{"configmaps", "shoots.core.gardener.cloud"}}}}, []string{"configmaps", "shoots.core.gardener.cloud"}), + ) }) diff --git a/pkg/apis/core/v1beta1/types_shoot.go b/pkg/apis/core/v1beta1/types_shoot.go index aa8dd751b6c..349a593ba68 100644 --- a/pkg/apis/core/v1beta1/types_shoot.go +++ b/pkg/apis/core/v1beta1/types_shoot.go @@ -214,6 +214,9 @@ type ShootStatus struct { // EncryptedResources is the list of resources in the Shoot which are currently encrypted. // Secrets are encrypted by default and are not part of the list. // See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details. + // + // Deprecated: This field is deprecated and will be removed with release v1.138. + // This field will be removed in favor of `status.credentials.encryptionAtRest.resources`. // +optional EncryptedResources []string `json:"encryptedResources,omitempty" protobuf:"bytes,18,rep,name=encryptedResources"` // Networking contains information about cluster networking such as CIDRs. @@ -280,6 +283,9 @@ type ShootCredentials struct { // Rotation contains information about the credential rotations. // +optional Rotation *ShootCredentialsRotation `json:"rotation,omitempty" protobuf:"bytes,1,opt,name=rotation"` + // EncryptionAtRest contains information about Shoot data encryption at rest. + // +optional + EncryptionAtRest *EncryptionAtRest `json:"encryptionAtRest,omitempty" protobuf:"bytes,2,opt,name=encryptionAtRest"` } // ShootCredentialsRotation contains information about the rotation of credentials. @@ -305,6 +311,15 @@ type ShootCredentialsRotation struct { ETCDEncryptionKey *ETCDEncryptionKeyRotation `json:"etcdEncryptionKey,omitempty" protobuf:"bytes,6,opt,name=etcdEncryptionKey"` } +// EncryptionAtRest contains information about Shoot data encryption at rest. +type EncryptionAtRest struct { + // Resources is the list of resources in the Shoot which are currently encrypted. + // Secrets are encrypted by default and are not part of the list. + // See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details. + // +optional + Resources []string `json:"resources,omitempty" protobuf:"bytes,1,rep,name=resources"` +} + // CARotation contains information about the certificate authority credential rotation. type CARotation struct { // Phase describes the phase of the certificate authority credential rotation. diff --git a/pkg/apis/core/v1beta1/zz_generated.conversion.go b/pkg/apis/core/v1beta1/zz_generated.conversion.go index c95a329cc51..5bbc3d4c47b 100644 --- a/pkg/apis/core/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/core/v1beta1/zz_generated.conversion.go @@ -641,6 +641,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*EncryptionAtRest)(nil), (*core.EncryptionAtRest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_EncryptionAtRest_To_core_EncryptionAtRest(a.(*EncryptionAtRest), b.(*core.EncryptionAtRest), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*core.EncryptionAtRest)(nil), (*EncryptionAtRest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_core_EncryptionAtRest_To_v1beta1_EncryptionAtRest(a.(*core.EncryptionAtRest), b.(*EncryptionAtRest), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*EncryptionConfig)(nil), (*core.EncryptionConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_EncryptionConfig_To_core_EncryptionConfig(a.(*EncryptionConfig), b.(*core.EncryptionConfig), scope) }); err != nil { @@ -3609,6 +3619,26 @@ func Convert_core_ETCDEncryptionKeyRotation_To_v1beta1_ETCDEncryptionKeyRotation return autoConvert_core_ETCDEncryptionKeyRotation_To_v1beta1_ETCDEncryptionKeyRotation(in, out, s) } +func autoConvert_v1beta1_EncryptionAtRest_To_core_EncryptionAtRest(in *EncryptionAtRest, out *core.EncryptionAtRest, s conversion.Scope) error { + out.Resources = *(*[]string)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1beta1_EncryptionAtRest_To_core_EncryptionAtRest is an autogenerated conversion function. +func Convert_v1beta1_EncryptionAtRest_To_core_EncryptionAtRest(in *EncryptionAtRest, out *core.EncryptionAtRest, s conversion.Scope) error { + return autoConvert_v1beta1_EncryptionAtRest_To_core_EncryptionAtRest(in, out, s) +} + +func autoConvert_core_EncryptionAtRest_To_v1beta1_EncryptionAtRest(in *core.EncryptionAtRest, out *EncryptionAtRest, s conversion.Scope) error { + out.Resources = *(*[]string)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_core_EncryptionAtRest_To_v1beta1_EncryptionAtRest is an autogenerated conversion function. +func Convert_core_EncryptionAtRest_To_v1beta1_EncryptionAtRest(in *core.EncryptionAtRest, out *EncryptionAtRest, s conversion.Scope) error { + return autoConvert_core_EncryptionAtRest_To_v1beta1_EncryptionAtRest(in, out, s) +} + func autoConvert_v1beta1_EncryptionConfig_To_core_EncryptionConfig(in *EncryptionConfig, out *core.EncryptionConfig, s conversion.Scope) error { out.Resources = *(*[]string)(unsafe.Pointer(&in.Resources)) return nil @@ -6732,6 +6762,7 @@ func Convert_core_ShootAdvertisedAddress_To_v1beta1_ShootAdvertisedAddress(in *c func autoConvert_v1beta1_ShootCredentials_To_core_ShootCredentials(in *ShootCredentials, out *core.ShootCredentials, s conversion.Scope) error { out.Rotation = (*core.ShootCredentialsRotation)(unsafe.Pointer(in.Rotation)) + out.EncryptionAtRest = (*core.EncryptionAtRest)(unsafe.Pointer(in.EncryptionAtRest)) return nil } @@ -6742,6 +6773,7 @@ func Convert_v1beta1_ShootCredentials_To_core_ShootCredentials(in *ShootCredenti func autoConvert_core_ShootCredentials_To_v1beta1_ShootCredentials(in *core.ShootCredentials, out *ShootCredentials, s conversion.Scope) error { out.Rotation = (*ShootCredentialsRotation)(unsafe.Pointer(in.Rotation)) + out.EncryptionAtRest = (*EncryptionAtRest)(unsafe.Pointer(in.EncryptionAtRest)) return nil } diff --git a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go index f7c3b8e4a74..06547e654cb 100644 --- a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go @@ -1859,6 +1859,27 @@ func (in *ETCDEncryptionKeyRotation) DeepCopy() *ETCDEncryptionKeyRotation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionAtRest) DeepCopyInto(out *EncryptionAtRest) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionAtRest. +func (in *EncryptionAtRest) DeepCopy() *EncryptionAtRest { + if in == nil { + return nil + } + out := new(EncryptionAtRest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EncryptionConfig) DeepCopyInto(out *EncryptionConfig) { *out = *in @@ -5452,6 +5473,11 @@ func (in *ShootCredentials) DeepCopyInto(out *ShootCredentials) { *out = new(ShootCredentialsRotation) (*in).DeepCopyInto(*out) } + if in.EncryptionAtRest != nil { + in, out := &in.EncryptionAtRest, &out.EncryptionAtRest + *out = new(EncryptionAtRest) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index 78351172dc6..3f7572092b0 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -193,8 +193,10 @@ func ValidateShootUpdate(newShoot, oldShoot *core.Shoot) field.ErrorList { if newShoot.Spec.Hibernation != nil { hibernationEnabled = ptr.Deref(newShoot.Spec.Hibernation.Enabled, false) } - for _, er := range newShoot.Status.EncryptedResources { - encryptedResources.Insert(schema.ParseGroupResource(er)) + if newShoot.Status.Credentials != nil && newShoot.Status.Credentials.EncryptionAtRest != nil { + for _, er := range newShoot.Status.Credentials.EncryptionAtRest.Resources { + encryptedResources.Insert(schema.ParseGroupResource(er)) + } } allErrs = append(allErrs, ValidateEncryptionConfigUpdate(newEncryptionConfig, oldEncryptionConfig, encryptedResources, etcdEncryptionKeyRotation, hibernationEnabled, field.NewPath("spec", "kubernetes", "kubeAPIServer", "encryptionConfig"))...) @@ -1519,8 +1521,10 @@ func validateHibernationUpdate(new, old *core.Shoot) field.ErrorList { } } - for _, r := range old.Status.EncryptedResources { - encryptedResourcesInStatus.Insert(schema.ParseGroupResource(r)) + if old.Status.Credentials != nil && old.Status.Credentials.EncryptionAtRest != nil { + for _, er := range old.Status.Credentials.EncryptionAtRest.Resources { + encryptedResourcesInStatus.Insert(schema.ParseGroupResource(er)) + } } if !hibernationEnabledInOld && hibernationEnabledInNew { @@ -1536,7 +1540,7 @@ func validateHibernationUpdate(new, old *core.Shoot) field.ErrorList { } if !encryptedResourcesInOldSpec.Equal(encryptedResourcesInStatus) { - allErrs = append(allErrs, field.Forbidden(fldPath, "shoot cannot be hibernated when spec.kubernetes.kubeAPIServer.encryptionConfig.resources and status.encryptedResources are not equal")) + allErrs = append(allErrs, field.Forbidden(fldPath, "shoot cannot be hibernated when spec.kubernetes.kubeAPIServer.encryptionConfig.resources and status.credentials.encryptionAtRest.resources are not equal")) } } @@ -2923,8 +2927,10 @@ func validateShootOperation(operation, maintenanceOperation string, shoot *core. return allErrs } - for _, res := range shoot.Status.EncryptedResources { - encryptedResources.Insert(schema.ParseGroupResource(res)) + if shoot.Status.Credentials != nil && shoot.Status.Credentials.EncryptionAtRest != nil { + for _, er := range shoot.Status.Credentials.EncryptionAtRest.Resources { + encryptedResources.Insert(schema.ParseGroupResource(er)) + } } fldPathOp := fldPath.Key(v1beta1constants.GardenerOperation) diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index 8b0826af77a..f4dd32a7eaf 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -261,6 +261,11 @@ var _ = Describe("Shoot Validation Tests", func() { {Key: "foo"}, }, }, + Status: core.ShootStatus{ + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{}, + }, + }, } }) @@ -2547,7 +2552,7 @@ var _ = Describe("Shoot Validation Tests", func() { shoot.Spec.Kubernetes.KubeAPIServer.EncryptionConfig = &core.EncryptionConfig{ Resources: []string{"configmaps", "deployments.apps"}, } - shoot.Status.EncryptedResources = []string{"deployments.apps", "configmaps"} + shoot.Status.Credentials.EncryptionAtRest.Resources = []string{"deployments.apps", "configmaps"} newShoot := prepareShootForUpdate(shoot) newShoot.Spec.Kubernetes.KubeAPIServer.EncryptionConfig.Resources = []string{"configmaps", "new.fancyresource.io"} @@ -2560,7 +2565,7 @@ var _ = Describe("Shoot Validation Tests", func() { shoot.Spec.Kubernetes.KubeAPIServer.EncryptionConfig = &core.EncryptionConfig{ Resources: resources, } - shoot.Status.EncryptedResources = resources + shoot.Status.Credentials.EncryptionAtRest.Resources = resources shoot.Spec.Hibernation = &core.Hibernation{Enabled: ptr.To(true)} newShoot := prepareShootForUpdate(shoot) @@ -2580,7 +2585,7 @@ var _ = Describe("Shoot Validation Tests", func() { shoot.Spec.Kubernetes.KubeAPIServer.EncryptionConfig = &core.EncryptionConfig{ Resources: resources, } - shoot.Status.EncryptedResources = resources + shoot.Status.Credentials.EncryptionAtRest.Resources = resources shoot.Status.IsHibernated = true newShoot := prepareShootForUpdate(shoot) @@ -2595,15 +2600,13 @@ var _ = Describe("Shoot Validation Tests", func() { shoot.Spec.Kubernetes.KubeAPIServer.EncryptionConfig = &core.EncryptionConfig{ Resources: resources, } - shoot.Status.EncryptedResources = resources + shoot.Status.Credentials.EncryptionAtRest.Resources = resources newShoot := prepareShootForUpdate(shoot) newShoot.Spec.Kubernetes.KubeAPIServer.EncryptionConfig.Resources = []string{"configmaps", "new.fancyresource.io"} - newShoot.Status.Credentials = &core.ShootCredentials{ - Rotation: &core.ShootCredentialsRotation{ - ETCDEncryptionKey: &core.ETCDEncryptionKeyRotation{ - Phase: core.RotationPreparing, - }, + newShoot.Status.Credentials.Rotation = &core.ShootCredentialsRotation{ + ETCDEncryptionKey: &core.ETCDEncryptionKeyRotation{ + Phase: core.RotationPreparing, }, } @@ -2621,19 +2624,17 @@ var _ = Describe("Shoot Validation Tests", func() { shoot.Spec.Kubernetes.KubeAPIServer.EncryptionConfig = &core.EncryptionConfig{ Resources: resources, } - shoot.Status.EncryptedResources = resources + shoot.Status.Credentials.EncryptionAtRest.Resources = resources newShoot := prepareShootForUpdate(shoot) newShoot.Spec.Kubernetes.KubeAPIServer.EncryptionConfig.Resources = []string{"deployments.apps", "newresource.fancyresource.io"} - newShoot.Status.Credentials = nil + newShoot.Status.Credentials.Rotation = nil Expect(ValidateShootUpdate(newShoot, shoot)).To(BeEmpty()) - newShoot.Status.Credentials = &core.ShootCredentials{ - Rotation: &core.ShootCredentialsRotation{ - ETCDEncryptionKey: &core.ETCDEncryptionKeyRotation{ - Phase: core.RotationCompleted, - }, + newShoot.Status.Credentials.Rotation = &core.ShootCredentialsRotation{ + ETCDEncryptionKey: &core.ETCDEncryptionKeyRotation{ + Phase: core.RotationCompleted, }, } @@ -5199,7 +5200,11 @@ var _ = Describe("Shoot Validation Tests", func() { LastOperation: &core.LastOperation{ Type: core.LastOperationTypeReconcile, }, - EncryptedResources: []string{"configmaps"}, + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps"}, + }, + }, }), Entry("when AutoInPlaceUpdate workers rollout is pending", false, core.ShootStatus{ LastOperation: &core.LastOperation{ @@ -5970,28 +5975,44 @@ var _ = Describe("Shoot Validation Tests", func() { LastOperation: &core.LastOperation{ Type: core.LastOperationTypeReconcile, }, - EncryptedResources: []string{"configmaps"}, + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps"}, + }, + }, }), Entry("when shoot spec encrypted resources and status encrypted resources are not equal", false, []string{"pods"}, core.ShootStatus{ LastOperation: &core.LastOperation{ Type: core.LastOperationTypeReconcile, }, - EncryptedResources: []string{"configmaps"}, + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps"}, + }, + }, }), Entry("when shoot spec encrypted resources and status encrypted resources are equal", true, []string{"configmaps"}, core.ShootStatus{ LastOperation: &core.LastOperation{ Type: core.LastOperationTypeReconcile, }, - EncryptedResources: []string{"configmaps"}, + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps"}, + }, + }, }), Entry("when shoot spec encrypted resources and status encrypted resources are equal", true, []string{"configmaps"}, core.ShootStatus{ LastOperation: &core.LastOperation{ Type: core.LastOperationTypeReconcile, }, - EncryptedResources: []string{"configmaps."}, + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps."}, + }, + }, }), ) @@ -6109,28 +6130,44 @@ var _ = Describe("Shoot Validation Tests", func() { LastOperation: &core.LastOperation{ Type: core.LastOperationTypeReconcile, }, - EncryptedResources: []string{"configmaps"}, + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps"}, + }, + }, }), Entry("when shoot spec encrypted resources and status encrypted resources are not equal", false, []string{"pods"}, core.ShootStatus{ LastOperation: &core.LastOperation{ Type: core.LastOperationTypeReconcile, }, - EncryptedResources: []string{"configmaps"}, + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps"}, + }, + }, }), Entry("when shoot spec encrypted resources and status encrypted resources are equal", true, []string{"configmaps"}, core.ShootStatus{ LastOperation: &core.LastOperation{ Type: core.LastOperationTypeReconcile, }, - EncryptedResources: []string{"configmaps"}, + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps"}, + }, + }, }), Entry("when shoot spec encrypted resources and status encrypted resources are equal", true, []string{"configmaps"}, core.ShootStatus{ LastOperation: &core.LastOperation{ Type: core.LastOperationTypeReconcile, }, - EncryptedResources: []string{"configmaps."}, + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps."}, + }, + }, }), ) @@ -6622,7 +6659,7 @@ var _ = Describe("Shoot Validation Tests", func() { Resources: []string{"events", "configmaps"}, }, } - shoot.Status.EncryptedResources = []string{"events"} + shoot.Status.Credentials.EncryptionAtRest.Resources = []string{"events"} newShoot := prepareShootForUpdate(shoot) newShoot.Spec.Hibernation = &core.Hibernation{Enabled: ptr.To(true)} @@ -6630,7 +6667,7 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShootUpdate(newShoot, shoot)).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.hibernation.enabled"), - "Detail": ContainSubstring("when spec.kubernetes.kubeAPIServer.encryptionConfig.resources and status.encryptedResources are not equal"), + "Detail": ContainSubstring("when spec.kubernetes.kubeAPIServer.encryptionConfig.resources and status.credentials.encryptionAtRest.resources are not equal"), })))) }) @@ -6641,7 +6678,7 @@ var _ = Describe("Shoot Validation Tests", func() { Resources: []string{"events", "configmaps"}, }, } - shoot.Status.EncryptedResources = []string{"configmaps.", "events"} + shoot.Status.Credentials.EncryptionAtRest.Resources = []string{"configmaps.", "events"} newShoot := prepareShootForUpdate(shoot) newShoot.Spec.Hibernation = &core.Hibernation{Enabled: ptr.To(true)} diff --git a/pkg/apis/core/zz_generated.deepcopy.go b/pkg/apis/core/zz_generated.deepcopy.go index 43c931263cb..a8507f4622a 100644 --- a/pkg/apis/core/zz_generated.deepcopy.go +++ b/pkg/apis/core/zz_generated.deepcopy.go @@ -1866,6 +1866,27 @@ func (in *ETCDEncryptionKeyRotation) DeepCopy() *ETCDEncryptionKeyRotation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionAtRest) DeepCopyInto(out *EncryptionAtRest) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionAtRest. +func (in *EncryptionAtRest) DeepCopy() *EncryptionAtRest { + if in == nil { + return nil + } + out := new(EncryptionAtRest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EncryptionConfig) DeepCopyInto(out *EncryptionConfig) { *out = *in @@ -5452,6 +5473,11 @@ func (in *ShootCredentials) DeepCopyInto(out *ShootCredentials) { *out = new(ShootCredentialsRotation) (*in).DeepCopyInto(*out) } + if in.EncryptionAtRest != nil { + in, out := &in.EncryptionAtRest, &out.EncryptionAtRest + *out = new(EncryptionAtRest) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/operator/v1alpha1/helper/garden.go b/pkg/apis/operator/v1alpha1/helper/garden.go index f58558d67c0..7f02484d118 100644 --- a/pkg/apis/operator/v1alpha1/helper/garden.go +++ b/pkg/apis/operator/v1alpha1/helper/garden.go @@ -242,3 +242,12 @@ func GetAPIServerSNIDomains(domains []string, sni operatorv1alpha1.SNI) []string return sniDomains } + +// GetEncryptedResourcesInStatus returns the encrypted resources from the garden status. +func GetEncryptedResourcesInStatus(gardenStatus operatorv1alpha1.GardenStatus) []string { + if gardenStatus.Credentials != nil && gardenStatus.Credentials.EncryptionAtRest != nil { + return gardenStatus.Credentials.EncryptionAtRest.Resources + } + + return nil +} diff --git a/pkg/apis/operator/v1alpha1/helper/garden_test.go b/pkg/apis/operator/v1alpha1/helper/garden_test.go index 74ca5438cc1..72fc0297519 100644 --- a/pkg/apis/operator/v1alpha1/helper/garden_test.go +++ b/pkg/apis/operator/v1alpha1/helper/garden_test.go @@ -276,4 +276,13 @@ var _ = Describe("helper", func() { Expect(GetAPIServerSNIDomains(domains, sni)).To(Equal([]string{"api.bar", "bar.foo.bar", "foo.foo.bar"})) }) }) + + DescribeTable("#GetEncryptedResourcesInStatus", + func(status operatorv1alpha1.GardenStatus, expected []string) { + Expect(GetEncryptedResourcesInStatus(status)).To(Equal(expected)) + }, + Entry("no credentials field", operatorv1alpha1.GardenStatus{}, nil), + Entry("without resources", operatorv1alpha1.GardenStatus{Credentials: &operatorv1alpha1.Credentials{}}, nil), + Entry("with resources", operatorv1alpha1.GardenStatus{Credentials: &operatorv1alpha1.Credentials{EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{Resources: []string{"configmaps", "shoots.core.gardener.cloud"}}}}, []string{"configmaps", "shoots.core.gardener.cloud"}), + ) }) diff --git a/pkg/apis/operator/v1alpha1/types_garden.go b/pkg/apis/operator/v1alpha1/types_garden.go index bb90a3b81ac..3a4983b99c3 100644 --- a/pkg/apis/operator/v1alpha1/types_garden.go +++ b/pkg/apis/operator/v1alpha1/types_garden.go @@ -764,6 +764,9 @@ type GardenStatus struct { // EncryptedResources is the list of resources which are currently encrypted in the virtual garden by the virtual kube-apiserver. // Resources which are encrypted by default will not appear here. // See https://github.com/gardener/gardener/blob/master/docs/concepts/operator.md#etcd-encryption-config for more details. + // + // Deprecated: This field is deprecated and will be removed with release v1.138. + // This field will be removed in favor of `status.credentials.encryptionAtRest.resources`. // +optional EncryptedResources []string `json:"encryptedResources,omitempty"` } @@ -773,6 +776,9 @@ type Credentials struct { // Rotation contains information about the credential rotations. // +optional Rotation *CredentialsRotation `json:"rotation,omitempty"` + // EncryptionAtRest contains information about garden data encryption at rest. + // +optional + EncryptionAtRest *EncryptionAtRest `json:"encryptionAtRest,omitempty"` } // CredentialsRotation contains information about the rotation of credentials. @@ -794,6 +800,15 @@ type CredentialsRotation struct { WorkloadIdentityKey *WorkloadIdentityKeyRotation `json:"workloadIdentityKey,omitempty"` } +// EncryptionAtRest contains information about virtual garden data encryption at rest. +type EncryptionAtRest struct { + // Resources is the list of resources which are currently encrypted in the virtual garden by the virtual kube-apiserver. + // Resources which are encrypted by default will not appear here. + // See https://github.com/gardener/gardener/blob/master/docs/concepts/operator.md#etcd-encryption-config for more details. + // +optional + Resources []string `json:"resources,omitempty"` +} + // WorkloadIdentityKeyRotation contains information about the workload identity key credential rotation. type WorkloadIdentityKeyRotation struct { // Phase describes the phase of the workload identity key credential rotation. diff --git a/pkg/apis/operator/v1alpha1/validation/garden.go b/pkg/apis/operator/v1alpha1/validation/garden.go index b0456025552..c88f3d80cb8 100644 --- a/pkg/apis/operator/v1alpha1/validation/garden.go +++ b/pkg/apis/operator/v1alpha1/validation/garden.go @@ -653,7 +653,7 @@ func validateOperationContext(operation string, garden *operatorv1alpha1.Garden, gardenerEncryptionConfig = garden.Spec.VirtualCluster.Gardener.APIServer.EncryptionConfig } - for _, r := range garden.Status.EncryptedResources { + for _, r := range helper.GetEncryptedResourcesInStatus(garden.Status) { encryptedResources.Insert(schema.ParseGroupResource(r)) } resourcesToEncrypt.Insert(sharedcomponent.GetResourcesForEncryptionFromConfig(encryptionConfig)...) @@ -814,12 +814,12 @@ func validateEncryptionConfigUpdate(oldGarden, newGarden *operatorv1alpha1.Garde } } - for _, r := range utils.FilterEntriesByFilterFn(newGarden.Status.EncryptedResources, operator.IsServedByKubeAPIServer) { + for _, r := range utils.FilterEntriesByFilterFn(helper.GetEncryptedResourcesInStatus(newGarden.Status), operator.IsServedByKubeAPIServer) { currentEncryptedKubernetesResources.Insert(schema.ParseGroupResource(r)) } allErrs = append(allErrs, gardencorevalidation.ValidateEncryptionConfigUpdate(newKubeAPIServerEncryptionConfig, oldKubeAPIServerEncryptionConfig, currentEncryptedKubernetesResources, etcdEncryptionKeyRotation, false, kubeAPIServerEncryptionConfigFldPath)...) - for _, r := range utils.FilterEntriesByFilterFn(newGarden.Status.EncryptedResources, operator.IsServedByGardenerAPIServer) { + for _, r := range utils.FilterEntriesByFilterFn(helper.GetEncryptedResourcesInStatus(newGarden.Status), operator.IsServedByGardenerAPIServer) { currentEncryptedGardenerResources.Insert(schema.ParseGroupResource(r)) } allErrs = append(allErrs, gardencorevalidation.ValidateEncryptionConfigUpdate(newGAPIServerEncryptionConfig, oldGAPIServerEncryptionConfig, currentEncryptedGardenerResources, etcdEncryptionKeyRotation, false, gAPIServerEncryptionConfigFldPath)...) diff --git a/pkg/apis/operator/v1alpha1/validation/garden_test.go b/pkg/apis/operator/v1alpha1/validation/garden_test.go index 8ff23745eb2..dc7ba2c7ce1 100644 --- a/pkg/apis/operator/v1alpha1/validation/garden_test.go +++ b/pkg/apis/operator/v1alpha1/validation/garden_test.go @@ -302,7 +302,11 @@ var _ = Describe("Validation Tests", func() { }, nil, nil), Entry("when spec encrypted resources and status encrypted resources are not equal", false, operatorv1alpha1.GardenStatus{ - EncryptedResources: []string{"configmaps", "projects.core.gardener.cloud"}, + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{ + Resources: []string{"configmaps", "projects.core.gardener.cloud"}, + }, + }, }, &gardencorev1beta1.EncryptionConfig{Resources: []string{"deployments.apps"}}, &gardencorev1beta1.EncryptionConfig{Resources: []string{"shoots.core.gardener.cloud"}}, @@ -314,7 +318,11 @@ var _ = Describe("Validation Tests", func() { ), Entry("when spec encrypted resources and status encrypted resources are equal", true, operatorv1alpha1.GardenStatus{ - EncryptedResources: []string{"configmaps", "daemonsets.apps", "projects.core.gardener.cloud", "shoots.core.gardener.cloud"}, + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{ + Resources: []string{"configmaps", "daemonsets.apps", "projects.core.gardener.cloud", "shoots.core.gardener.cloud"}, + }, + }, }, &gardencorev1beta1.EncryptionConfig{Resources: []string{"daemonsets.apps", "configmaps"}}, &gardencorev1beta1.EncryptionConfig{Resources: []string{"shoots.core.gardener.cloud", "projects.core.gardener.cloud"}}, @@ -897,7 +905,11 @@ var _ = Describe("Validation Tests", func() { }, nil, nil), Entry("when spec encrypted resources and status encrypted resources are not equal", false, operatorv1alpha1.GardenStatus{ - EncryptedResources: []string{"configmaps", "projects.core.gardener.cloud"}, + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{ + Resources: []string{"configmaps", "projects.core.gardener.cloud"}, + }, + }, }, &gardencorev1beta1.EncryptionConfig{Resources: []string{"deployments.apps"}}, &gardencorev1beta1.EncryptionConfig{Resources: []string{"shoots.core.gardener.cloud"}}, @@ -909,7 +921,11 @@ var _ = Describe("Validation Tests", func() { ), Entry("when spec encrypted resources and status encrypted resources are equal", true, operatorv1alpha1.GardenStatus{ - EncryptedResources: []string{"configmaps.", "daemonsets.apps", "projects.core.gardener.cloud", "shoots.core.gardener.cloud"}, + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{ + Resources: []string{"configmaps.", "daemonsets.apps", "projects.core.gardener.cloud", "shoots.core.gardener.cloud"}, + }, + }, }, &gardencorev1beta1.EncryptionConfig{Resources: []string{"daemonsets.apps", "configmaps"}}, &gardencorev1beta1.EncryptionConfig{Resources: []string{"shoots.core.gardener.cloud", "projects.core.gardener.cloud"}}, @@ -980,7 +996,11 @@ var _ = Describe("Validation Tests", func() { }, nil, nil), Entry("when spec encrypted resources and status encrypted resources are not equal", false, operatorv1alpha1.GardenStatus{ - EncryptedResources: []string{"configmaps", "projects.core.gardener.cloud"}, + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{ + Resources: []string{"configmaps", "projects.core.gardener.cloud"}, + }, + }, }, &gardencorev1beta1.EncryptionConfig{Resources: []string{"deployments.apps"}}, &gardencorev1beta1.EncryptionConfig{Resources: []string{"shoots.core.gardener.cloud"}}, @@ -992,7 +1012,11 @@ var _ = Describe("Validation Tests", func() { ), Entry("when spec encrypted resources and status encrypted resources are equal", true, operatorv1alpha1.GardenStatus{ - EncryptedResources: []string{"configmaps.", "daemonsets.apps", "projects.core.gardener.cloud", "shoots.core.gardener.cloud"}, + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{ + Resources: []string{"configmaps.", "daemonsets.apps", "projects.core.gardener.cloud", "shoots.core.gardener.cloud"}, + }, + }, }, &gardencorev1beta1.EncryptionConfig{Resources: []string{"daemonsets.apps", "configmaps"}}, &gardencorev1beta1.EncryptionConfig{Resources: []string{"shoots.core.gardener.cloud", "projects.core.gardener.cloud"}}, @@ -2573,6 +2597,11 @@ var _ = Describe("Validation Tests", func() { }, }, }, + Status: operatorv1alpha1.GardenStatus{ + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{}, + }, + }, } newGarden = oldGarden.DeepCopy() @@ -2946,16 +2975,14 @@ var _ = Describe("Validation Tests", func() { }, }, } - newGarden.Status.EncryptedResources = append(oldResources, oldGardenerResources...) + newGarden.Status.Credentials.EncryptionAtRest.Resources = append(oldResources, oldGardenerResources...) newGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.EncryptionConfig.Resources = []string{"deployments.apps", "newresource.fancyresource.io"} newGarden.Spec.VirtualCluster.Gardener.APIServer.EncryptionConfig.Resources = []string{"shoots.core.gardener.cloud"} - newGarden.Status.Credentials = &operatorv1alpha1.Credentials{ - Rotation: &operatorv1alpha1.CredentialsRotation{ - ETCDEncryptionKey: &gardencorev1beta1.ETCDEncryptionKeyRotation{ - Phase: gardencorev1beta1.RotationPreparing, - }, + newGarden.Status.Credentials.Rotation = &operatorv1alpha1.CredentialsRotation{ + ETCDEncryptionKey: &gardencorev1beta1.ETCDEncryptionKeyRotation{ + Phase: gardencorev1beta1.RotationPreparing, }, } @@ -2986,16 +3013,14 @@ var _ = Describe("Validation Tests", func() { }, }, } - newGarden.Status.EncryptedResources = append(oldResources, oldGardenerResources...) + newGarden.Status.Credentials.EncryptionAtRest.Resources = append(oldResources, oldGardenerResources...) newGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.EncryptionConfig.Resources = []string{"configmaps", "resource.custom.io"} newGarden.Spec.VirtualCluster.Gardener.APIServer.EncryptionConfig.Resources = []string{"shoots.core.gardener.cloud", "bastions.operations.gardener.cloud"} - newGarden.Status.Credentials = &operatorv1alpha1.Credentials{ - Rotation: &operatorv1alpha1.CredentialsRotation{ - ETCDEncryptionKey: &gardencorev1beta1.ETCDEncryptionKeyRotation{ - Phase: gardencorev1beta1.RotationPreparing, - }, + newGarden.Status.Credentials.Rotation = &operatorv1alpha1.CredentialsRotation{ + ETCDEncryptionKey: &gardencorev1beta1.ETCDEncryptionKeyRotation{ + Phase: gardencorev1beta1.RotationPreparing, }, } @@ -3015,19 +3040,17 @@ var _ = Describe("Validation Tests", func() { }, }, } - newGarden.Status.EncryptedResources = append(oldResources, oldGardenerResources...) + newGarden.Status.Credentials.EncryptionAtRest.Resources = append(oldResources, oldGardenerResources...) newGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.EncryptionConfig.Resources = []string{"deployments.apps", "newresource.fancyresource.io"} newGarden.Spec.VirtualCluster.Gardener.APIServer.EncryptionConfig.Resources = []string{"shoots.core.gardener.cloud"} - newGarden.Status.Credentials = nil + newGarden.Status.Credentials.Rotation = nil Expect(ValidateGardenUpdate(oldGarden, newGarden, extensions)).To(BeEmpty()) - newGarden.Status.Credentials = &operatorv1alpha1.Credentials{ - Rotation: &operatorv1alpha1.CredentialsRotation{ - ETCDEncryptionKey: &gardencorev1beta1.ETCDEncryptionKeyRotation{ - Phase: gardencorev1beta1.RotationCompleted, - }, + newGarden.Status.Credentials.Rotation = &operatorv1alpha1.CredentialsRotation{ + ETCDEncryptionKey: &gardencorev1beta1.ETCDEncryptionKeyRotation{ + Phase: gardencorev1beta1.RotationCompleted, }, } diff --git a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index df655a86277..dd1528a9554 100644 --- a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -184,6 +184,11 @@ func (in *Credentials) DeepCopyInto(out *Credentials) { *out = new(CredentialsRotation) (*in).DeepCopyInto(*out) } + if in.EncryptionAtRest != nil { + in, out := &in.EncryptionAtRest, &out.EncryptionAtRest + *out = new(EncryptionAtRest) + (*in).DeepCopyInto(*out) + } return } @@ -585,6 +590,27 @@ func (in *ETCDMain) DeepCopy() *ETCDMain { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionAtRest) DeepCopyInto(out *EncryptionAtRest) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionAtRest. +func (in *EncryptionAtRest) DeepCopy() *EncryptionAtRest { + if in == nil { + return nil + } + out := new(EncryptionAtRest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Extension) DeepCopyInto(out *Extension) { *out = *in diff --git a/pkg/apiserver/openapi/api_violations.report b/pkg/apiserver/openapi/api_violations.report index 302d97ecf80..06d3f9f33a1 100644 --- a/pkg/apiserver/openapi/api_violations.report +++ b/pkg/apiserver/openapi/api_violations.report @@ -21,6 +21,7 @@ API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,DNS,Providers API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,DNSIncludeExclude,Exclude API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,DNSIncludeExclude,Include +API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,EncryptionAtRest,Resources API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,EncryptionConfig,Resources API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,ExposureClassScheduling,Tolerations API rule violation: list_type_missing,github.com/gardener/gardener/pkg/apis/core/v1beta1,ExtensionResourceState,Resources diff --git a/pkg/apiserver/openapi/openapi_generated.go b/pkg/apiserver/openapi/openapi_generated.go index 9d11d1fa681..a94c857a368 100644 --- a/pkg/apiserver/openapi/openapi_generated.go +++ b/pkg/apiserver/openapi/openapi_generated.go @@ -93,6 +93,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/gardener/gardener/pkg/apis/core/v1beta1.ETCD": schema_pkg_apis_core_v1beta1_ETCD(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.ETCDConfig": schema_pkg_apis_core_v1beta1_ETCDConfig(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.ETCDEncryptionKeyRotation": schema_pkg_apis_core_v1beta1_ETCDEncryptionKeyRotation(ref), + "github.com/gardener/gardener/pkg/apis/core/v1beta1.EncryptionAtRest": schema_pkg_apis_core_v1beta1_EncryptionAtRest(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.EncryptionConfig": schema_pkg_apis_core_v1beta1_EncryptionConfig(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.ExpirableVersion": schema_pkg_apis_core_v1beta1_ExpirableVersion(ref), "github.com/gardener/gardener/pkg/apis/core/v1beta1.ExposureClass": schema_pkg_apis_core_v1beta1_ExposureClass(ref), @@ -3722,6 +3723,34 @@ func schema_pkg_apis_core_v1beta1_ETCDEncryptionKeyRotation(ref common.Reference } } +func schema_pkg_apis_core_v1beta1_EncryptionAtRest(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "EncryptionAtRest contains information about Shoot data encryption at rest.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "resources": { + SchemaProps: spec.SchemaProps{ + Description: "Resources is the list of resources in the Shoot which are currently encrypted. Secrets are encrypted by default and are not part of the list. See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + }, + }, + } +} + func schema_pkg_apis_core_v1beta1_EncryptionConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -9249,11 +9278,17 @@ func schema_pkg_apis_core_v1beta1_ShootCredentials(ref common.ReferenceCallback) Ref: ref("github.com/gardener/gardener/pkg/apis/core/v1beta1.ShootCredentialsRotation"), }, }, + "encryptionAtRest": { + SchemaProps: spec.SchemaProps{ + Description: "EncryptionAtRest contains information about Shoot data encryption at rest.", + Ref: ref("github.com/gardener/gardener/pkg/apis/core/v1beta1.EncryptionAtRest"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/gardener/gardener/pkg/apis/core/v1beta1.ShootCredentialsRotation"}, + "github.com/gardener/gardener/pkg/apis/core/v1beta1.EncryptionAtRest", "github.com/gardener/gardener/pkg/apis/core/v1beta1.ShootCredentialsRotation"}, } } @@ -10008,7 +10043,7 @@ func schema_pkg_apis_core_v1beta1_ShootStatus(ref common.ReferenceCallback) comm }, "encryptedResources": { SchemaProps: spec.SchemaProps{ - Description: "EncryptedResources is the list of resources in the Shoot which are currently encrypted. Secrets are encrypted by default and are not part of the list. See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details.", + Description: "EncryptedResources is the list of resources in the Shoot which are currently encrypted. Secrets are encrypted by default and are not part of the list. See https://github.com/gardener/gardener/blob/master/docs/usage/security/etcd_encryption_config.md for more details.\n\nDeprecated: This field is deprecated and will be removed with release v1.138. This field will be removed in favor of `status.credentials.encryptionAtRest.resources`.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ diff --git a/pkg/apiserver/registry/core/shoot/storage/storage.go b/pkg/apiserver/registry/core/shoot/storage/storage.go index 42ce37fa0af..378c931dd5c 100644 --- a/pkg/apiserver/registry/core/shoot/storage/storage.go +++ b/pkg/apiserver/registry/core/shoot/storage/storage.go @@ -75,6 +75,7 @@ func NewREST(optsGetter generic.RESTOptionsGetter, credentialsRotationInterval t DeleteStrategy: shootStrategy, TableConvertor: newTableConvertor(), + Decorator: defaultOnRead, } options = &generic.StoreOptions{ RESTOptions: optsGetter, @@ -173,3 +174,29 @@ var _ rest.ShortNamesProvider = &REST{} func (r *REST) ShortNames() []string { return []string{} } + +// defaultOnRead ensures the shoot.status.credentials.encryptionAtRest.resources field is set on read requests. +// TODO(AleksandarSavchev): Remove this function after v1.135 has been released. +func defaultOnRead(obj runtime.Object) { + switch s := obj.(type) { + case *core.Shoot: + defaultOnReadShoot(s) + case *core.ShootList: + defaultOnReadShoots(s) + default: + } +} + +func defaultOnReadShoot(s *core.Shoot) { + shoot.SyncEncryptedResourcesStatus(s) +} + +func defaultOnReadShoots(shootList *core.ShootList) { + if shootList == nil { + return + } + + for i := range shootList.Items { + defaultOnReadShoot(&shootList.Items[i]) + } +} diff --git a/pkg/apiserver/registry/core/shoot/strategy.go b/pkg/apiserver/registry/core/shoot/strategy.go index a2e1a0ea42f..b1220723b12 100644 --- a/pkg/apiserver/registry/core/shoot/strategy.go +++ b/pkg/apiserver/registry/core/shoot/strategy.go @@ -80,6 +80,9 @@ func (shootStrategy) PrepareForUpdate(_ context.Context, obj, old runtime.Object if oldShoot.Spec.CredentialsBindingName == nil && !utilfeature.DefaultFeatureGate.Enabled(features.ShootCredentialsBinding) { newShoot.Spec.CredentialsBindingName = nil } + + // Ensure that encrypted resources are synced from `.status.encryptedResources` to `status.credentials.encryptionAtRest.resources`. + SyncEncryptedResourcesStatus(newShoot) } func mustIncreaseGeneration(oldShoot, newShoot *core.Shoot) bool { @@ -252,6 +255,9 @@ func (shootStatusStrategy) PrepareForUpdate(_ context.Context, obj, old runtime. gardencorehelper.ShouldETCDEncryptionKeyRotationBeAutoCompleteAfterPrepared(newShoot.Status.Credentials) { newShoot.Generation = oldShoot.Generation + 1 } + + // Ensure that encrypted resources are synced from `.status.encryptedResources` to `status.credentials.encryptionAtRest.resources`. + SyncEncryptedResourcesStatus(newShoot) } func (shootStatusStrategy) ValidateUpdate(_ context.Context, obj, old runtime.Object) field.ErrorList { @@ -362,3 +368,21 @@ func getStatusSeedName(shoot *core.Shoot) string { } return *shoot.Status.SeedName } + +// SyncEncryptedResourcesStatus ensures the status fields shoot.status.encryptedResources and +// shoot.status.credentials.encryptionAtRest.resources are in sync. +// TODO(AleksandarSavchev): Remove this function after v1.135 has been released. +func SyncEncryptedResourcesStatus(shoot *core.Shoot) { + if len(shoot.Status.EncryptedResources) > 0 { + if shoot.Status.Credentials == nil { + shoot.Status.Credentials = &core.ShootCredentials{} + } + if shoot.Status.Credentials.EncryptionAtRest == nil { + shoot.Status.Credentials.EncryptionAtRest = &core.EncryptionAtRest{} + } + + shoot.Status.Credentials.EncryptionAtRest.Resources = shoot.Status.EncryptedResources + } else if shoot.Status.Credentials != nil && shoot.Status.Credentials.EncryptionAtRest != nil { + shoot.Status.Credentials.EncryptionAtRest.Resources = nil + } +} diff --git a/pkg/apiserver/registry/core/shoot/strategy_test.go b/pkg/apiserver/registry/core/shoot/strategy_test.go index 2593781f88a..7c5044100b4 100644 --- a/pkg/apiserver/registry/core/shoot/strategy_test.go +++ b/pkg/apiserver/registry/core/shoot/strategy_test.go @@ -252,6 +252,48 @@ var _ = Describe("Strategy", func() { }) }) + DescribeTable("should sync encrypted resources from status.encryptedResources to status.credentials.encryptionAtRest.resources", + func(status core.ShootStatus, expected core.ShootStatus) { + oldShoot.Status = status + newShoot.Status = status + + strategy.PrepareForUpdate(ctx, newShoot, oldShoot) + + Expect(newShoot.Status).To(Equal(expected)) + }, + Entry("no encrypted resources", core.ShootStatus{}, core.ShootStatus{}), + Entry("with encrypted resources", + core.ShootStatus{ + EncryptedResources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + core.ShootStatus{ + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + }, + EncryptedResources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + ), + Entry("should overwrite", core.ShootStatus{ + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + }, + EncryptedResources: []string{"configmaps"}, + }, + core.ShootStatus{ + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps"}, + }, + }, + EncryptedResources: []string{"configmaps"}, + }, + ), + ) + Context("seedName change", func() { BeforeEach(func() { oldShoot = &core.Shoot{ @@ -790,6 +832,51 @@ var _ = Describe("Strategy", func() { Entry("rotation phase is not prepared", nil, &core.ETCDEncryptionKeyRotation{Phase: core.RotationCompleting, AutoCompleteAfterPrepared: ptr.To(true)}, false), ) }) + + DescribeTable("should sync encrypted resources from status.encryptedResources to status.credentials.encryptionAtRest.resources", + func(status core.ShootStatus, expected core.ShootStatus) { + oldShoot := &core.Shoot{ + Spec: core.ShootSpec{}, + Status: status, + } + newShoot := oldShoot.DeepCopy() + + strategy.PrepareForUpdate(ctx, newShoot, oldShoot) + + Expect(newShoot.Status).To(Equal(expected)) + }, + Entry("no encrypted resources", core.ShootStatus{}, core.ShootStatus{}), + Entry("with encrypted resources", + core.ShootStatus{ + EncryptedResources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + core.ShootStatus{ + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + }, + EncryptedResources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + ), + Entry("should overwrite", core.ShootStatus{ + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + }, + EncryptedResources: []string{"configmaps"}, + }, + core.ShootStatus{ + Credentials: &core.ShootCredentials{ + EncryptionAtRest: &core.EncryptionAtRest{ + Resources: []string{"configmaps"}, + }, + }, + EncryptedResources: []string{"configmaps"}, + }, + ), + ) }) }) diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go index 5e0afacff4c..a082d44a5c0 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go @@ -492,7 +492,22 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope encryptedResources = shared.StringifyGroupResources(shared.GetResourcesForEncryptionFromConfig(o.Shoot.GetInfo().Spec.Kubernetes.KubeAPIServer.EncryptionConfig)) } + // TODO(AleksandarSavchev): Stop setting the shoot.Status.EncryptedResources after v1.135 has been released. shoot.Status.EncryptedResources = encryptedResources + + if len(encryptedResources) > 0 { + if shoot.Status.Credentials == nil { + shoot.Status.Credentials = &gardencorev1beta1.ShootCredentials{} + } + if shoot.Status.Credentials.EncryptionAtRest == nil { + shoot.Status.Credentials.EncryptionAtRest = &gardencorev1beta1.EncryptionAtRest{} + } + + shoot.Status.Credentials.EncryptionAtRest.Resources = encryptedResources + } else if shoot.Status.Credentials != nil && shoot.Status.Credentials.EncryptionAtRest != nil { + shoot.Status.Credentials.EncryptionAtRest.Resources = nil + } + return nil }); err != nil { return err diff --git a/pkg/gardenlet/operation/shoot/shoot.go b/pkg/gardenlet/operation/shoot/shoot.go index 56c49395f13..7679c21b3e0 100644 --- a/pkg/gardenlet/operation/shoot/shoot.go +++ b/pkg/gardenlet/operation/shoot/shoot.go @@ -286,9 +286,8 @@ func (b *Builder) Build(ctx context.Context, c client.Reader) (*Shoot, error) { if shoot.GetInfo().Spec.Kubernetes.KubeAPIServer != nil { shoot.ResourcesToEncrypt = sharedcomponent.StringifyGroupResources(sharedcomponent.GetResourcesForEncryptionFromConfig(shoot.GetInfo().Spec.Kubernetes.KubeAPIServer.EncryptionConfig)) } - if len(shoot.GetInfo().Status.EncryptedResources) > 0 { - shoot.EncryptedResources = sharedcomponent.NormalizeResources(shoot.GetInfo().Status.EncryptedResources) - } + + shoot.EncryptedResources = v1beta1helper.GetShootEncryptedResourcesInStatus(shoot.GetInfo().Status) if b.seed != nil { shoot.TopologyAwareRoutingEnabled = v1beta1helper.IsTopologyAwareRoutingForShootControlPlaneEnabled(b.seed, shootObject) diff --git a/pkg/operator/controller/garden/garden/reconciler_reconcile.go b/pkg/operator/controller/garden/garden/reconciler_reconcile.go index 8e68ca12efa..e67bde6f843 100644 --- a/pkg/operator/controller/garden/garden/reconciler_reconcile.go +++ b/pkg/operator/controller/garden/garden/reconciler_reconcile.go @@ -180,7 +180,7 @@ func (r *Reconciler) reconcile( virtualClusterClient client.Client defaultEncryptedGVKs = append(gardenerutils.DefaultGardenerGVKsForEncryption(), gardenerutils.DefaultGVKsForEncryption()...) resourcesToEncrypt = append(shared.StringifyGroupResources(getKubernetesResourcesForEncryption(garden)), shared.StringifyGroupResources(getGardenerResourcesForEncryption(garden))...) - encryptedResources = shared.NormalizeResources(garden.Status.EncryptedResources) + encryptedResources = shared.NormalizeResources(helper.GetEncryptedResourcesInStatus(garden.Status)) g = flow.NewGraph("Garden reconciliation") generateGenericTokenKubeconfig = g.Add(flow.Task{ @@ -531,7 +531,23 @@ func (r *Reconciler) reconcile( encryptedResources := append(getKubernetesResourcesForEncryption(garden), getGardenerResourcesForEncryption(garden)...) patch := client.MergeFrom(garden.DeepCopy()) + + // TODO(AleksandarSavchev): Stop setting the shoot.Status.EncryptedResources after v1.135 has been released. garden.Status.EncryptedResources = shared.StringifyGroupResources(encryptedResources) + + if len(encryptedResources) > 0 { + if garden.Status.Credentials == nil { + garden.Status.Credentials = &operatorv1alpha1.Credentials{} + } + if garden.Status.Credentials.EncryptionAtRest == nil { + garden.Status.Credentials.EncryptionAtRest = &operatorv1alpha1.EncryptionAtRest{} + } + + garden.Status.Credentials.EncryptionAtRest.Resources = shared.StringifyGroupResources(encryptedResources) + } else if garden.Status.Credentials != nil && garden.Status.Credentials.EncryptionAtRest != nil { + garden.Status.Credentials.EncryptionAtRest.Resources = nil + } + if err := r.RuntimeClientSet.Client().Status().Patch(ctx, garden, patch); err != nil { return fmt.Errorf("error patching Garden status after snapshotting ETCD: %w", err) } @@ -954,7 +970,7 @@ func (r *Reconciler) deployKubeAPIServerFunc(garden *operatorv1alpha1.Garden, ku nil, nil, shared.StringifyGroupResources(getKubernetesResourcesForEncryption(garden)), - utils.FilterEntriesByFilterFn(shared.NormalizeResources(garden.Status.EncryptedResources), operator.IsServedByKubeAPIServer), + utils.FilterEntriesByFilterFn(shared.NormalizeResources(helper.GetEncryptedResourcesInStatus(garden.Status)), operator.IsServedByKubeAPIServer), helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials), false, ) @@ -991,7 +1007,7 @@ func (r *Reconciler) deployGardenerAPIServerFunc(garden *operatorv1alpha1.Garden r.GardenNamespace, gardenerAPIServer, shared.StringifyGroupResources(getGardenerResourcesForEncryption(garden)), - utils.FilterEntriesByFilterFn(garden.Status.EncryptedResources, operator.IsServedByGardenerAPIServer), + utils.FilterEntriesByFilterFn(helper.GetEncryptedResourcesInStatus(garden.Status), operator.IsServedByGardenerAPIServer), helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials), helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials), ) diff --git a/pkg/operator/webhook/add.go b/pkg/operator/webhook/add.go index eb86e41b7c9..fcafcf6f290 100644 --- a/pkg/operator/webhook/add.go +++ b/pkg/operator/webhook/add.go @@ -137,7 +137,8 @@ func GetMutatingWebhookConfiguration(mode, url string) *admissionregistrationv1. Rule: admissionregistrationv1.Rule{ APIGroups: []string{operatorv1alpha1.SchemeGroupVersion.Group}, APIVersions: []string{operatorv1alpha1.SchemeGroupVersion.Version}, - Resources: []string{"gardens"}, + // TODO(AleksandarSavchev): Remove gardens/status after v1.135 has been released. + Resources: []string{"gardens", "gardens/status"}, }, Operations: []admissionregistrationv1.OperationType{ admissionregistrationv1.Create, diff --git a/pkg/operator/webhook/defaulting/garden/handler.go b/pkg/operator/webhook/defaulting/garden/handler.go index 8cfa2033f7d..105bc8ef575 100644 --- a/pkg/operator/webhook/defaulting/garden/handler.go +++ b/pkg/operator/webhook/defaulting/garden/handler.go @@ -44,5 +44,20 @@ func (h *Handler) Default(_ context.Context, obj runtime.Object) error { garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager.KubeControllerManagerConfig = &gardencorev1beta1.KubeControllerManagerConfig{} } + // Defaulting used for migration from `.status.encryptedResources` to `status.credentials.encryptionAtRest.resources`. + // TODO(AleksandarSavchev): Remove this block after v1.135 has been released. + if len(garden.Status.EncryptedResources) > 0 { + if garden.Status.Credentials == nil { + garden.Status.Credentials = &operatorv1alpha1.Credentials{} + } + if garden.Status.Credentials.EncryptionAtRest == nil { + garden.Status.Credentials.EncryptionAtRest = &operatorv1alpha1.EncryptionAtRest{} + } + + if len(garden.Status.Credentials.EncryptionAtRest.Resources) == 0 { + garden.Status.Credentials.EncryptionAtRest.Resources = garden.Status.EncryptedResources + } + } + return nil } diff --git a/pkg/operator/webhook/defaulting/garden/handler_test.go b/pkg/operator/webhook/defaulting/garden/handler_test.go index 0d05a420599..04e684dae83 100644 --- a/pkg/operator/webhook/defaulting/garden/handler_test.go +++ b/pkg/operator/webhook/defaulting/garden/handler_test.go @@ -111,5 +111,47 @@ var _ = Describe("Handler", func() { Expect(handler.Default(ctx, garden)).To(Succeed()) Expect(garden.Spec.RuntimeCluster.Networking.IPFamilies).To(Equal([]gardencorev1beta1.IPFamily{"foo"})) }) + + DescribeTable("should sync encrypted resources from status.encryptedResources to status.credentials.encryptionAtRest.resources", + func(status operatorv1alpha1.GardenStatus, expected operatorv1alpha1.GardenStatus) { + garden.Status = status + + Expect(handler.Default(ctx, garden)).To(Succeed()) + + Expect(garden.Status).To(Equal(expected)) + }, + Entry("no encrypted resources", operatorv1alpha1.GardenStatus{}, operatorv1alpha1.GardenStatus{}), + Entry("with encrypted resources", + operatorv1alpha1.GardenStatus{ + EncryptedResources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + operatorv1alpha1.GardenStatus{ + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{ + Resources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + }, + EncryptedResources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + ), + Entry("should not overwrite", + operatorv1alpha1.GardenStatus{ + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{ + Resources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + }, + EncryptedResources: []string{"configmaps"}, + }, + operatorv1alpha1.GardenStatus{ + Credentials: &operatorv1alpha1.Credentials{ + EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{ + Resources: []string{"configmaps", "shoots.core.gardener.cloud"}, + }, + }, + EncryptedResources: []string{"configmaps"}, + }, + ), + ) }) }) From 0a597ce2c140c5602d04acea13583d42c3d943a1 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 14 Nov 2025 10:39:31 +0100 Subject: [PATCH 113/176] Update dependency gardener/logging to v0.69.0 (#13450) --- imagevector/containers.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 4cc482ebc64..6d39c7731f9 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -613,7 +613,7 @@ images: name: fluent-bit-to-vali sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/fluent-bit-to-vali - tag: "v0.68.0" + tag: "v0.69.0" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -648,7 +648,7 @@ images: - name: vali-curator sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/vali-curator - tag: "v0.68.0" + tag: "v0.69.0" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -704,7 +704,7 @@ images: name: telegraf-iptables sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/telegraf-iptables - tag: "v0.68.0" + tag: "v0.69.0" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -724,7 +724,7 @@ images: - name: event-logger sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/event-logger - tag: "v0.68.0" + tag: "v0.69.0" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -741,7 +741,7 @@ images: - name: tune2fs sourceRepository: github.com/gardener/logging repository: europe-docker.pkg.dev/gardener-project/releases/gardener/tune2fs - tag: "v0.68.0" + tag: "v0.69.0" labels: - name: 'gardener.cloud/cve-categorisation' value: From 1ef836bb6a50852abc069eff75c99b87c4e332e7 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 14 Nov 2025 16:27:26 +0100 Subject: [PATCH 114/176] Update dependency golangci/golangci-lint to v2.6.2 (#13457) --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 5b58af0a1c9..49af4ce7c33 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -54,7 +54,7 @@ TYPOS := $(TOOLS_BIN_DIR)/typos # default tool versions # renovate: datasource=github-releases depName=golangci/golangci-lint -GOLANGCI_LINT_VERSION ?= v2.6.1 +GOLANGCI_LINT_VERSION ?= v2.6.2 # renovate: datasource=github-releases depName=securego/gosec GOSEC_VERSION ?= v2.22.10 # renovate: datasource=github-releases depName=joelanford/go-apidiff From 5f712044c5f0c15ca2bce8f1d6cb3cfe36d4ea30 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 14 Nov 2025 19:28:26 +0100 Subject: [PATCH 115/176] Update golang.org/x/exp digest to e25ba8c (#13452) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 44167fb70e0..b3f9e818ba8 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( go.yaml.in/yaml/v2 v2.4.3 go.yaml.in/yaml/v4 v4.0.0-rc.2 golang.org/x/crypto v0.44.0 - golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 + golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 golang.org/x/text v0.31.0 golang.org/x/time v0.14.0 golang.org/x/tools v0.39.0 diff --git a/go.sum b/go.sum index eee5d4f5dc9..babe54b39f1 100644 --- a/go.sum +++ b/go.sum @@ -860,8 +860,8 @@ golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= -golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= +golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 h1:zfMcR1Cs4KNuomFFgGefv5N0czO2XZpUbxGUy8i8ug0= +golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From 100191c88f47d47cc52fa21f3cd4e1b3f62537ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20G=C3=B6tz?= <47362717+oliver-goetz@users.noreply.github.com> Date: Sat, 15 Nov 2025 00:19:26 +0100 Subject: [PATCH 116/176] Set `useClientProtocol` in destination rules to `true` when `IstioTLSTermination` is enabled (#13459) --- pkg/component/kubernetes/apiserverexposure/sni_test.go | 9 +++++++++ pkg/component/test/istiocomponent.go | 1 + pkg/utils/istio/destinationrule.go | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/pkg/component/kubernetes/apiserverexposure/sni_test.go b/pkg/component/kubernetes/apiserverexposure/sni_test.go index 0c989f798e1..507ed19c1de 100644 --- a/pkg/component/kubernetes/apiserverexposure/sni_test.go +++ b/pkg/component/kubernetes/apiserverexposure/sni_test.go @@ -479,6 +479,9 @@ var _ = Describe("#SNI", func() { BeforeEach(func() { istioTLSTermination = true + expectedDestinationRule.Spec.TrafficPolicy.ConnectionPool.Http = &istioapinetworkingv1beta1.ConnectionPoolSettings_HTTPSettings{ + UseClientProtocol: true, + } expectedDestinationRule.Spec.TrafficPolicy.LoadBalancer = &istioapinetworkingv1beta1.LoadBalancerSettings{ LbPolicy: &istioapinetworkingv1beta1.LoadBalancerSettings_Simple{ Simple: istioapinetworkingv1beta1.LoadBalancerSettings_LEAST_REQUEST, @@ -544,6 +547,9 @@ var _ = Describe("#SNI", func() { Hosts: wildcardHosts, } + expectedDestinationRule.Spec.TrafficPolicy.ConnectionPool.Http = &istioapinetworkingv1beta1.ConnectionPoolSettings_HTTPSettings{ + UseClientProtocol: true, + } expectedDestinationRule.Spec.TrafficPolicy.LoadBalancer = &istioapinetworkingv1beta1.LoadBalancerSettings{ LbPolicy: &istioapinetworkingv1beta1.LoadBalancerSettings_Simple{ Simple: istioapinetworkingv1beta1.LoadBalancerSettings_LEAST_REQUEST, @@ -625,6 +631,9 @@ var _ = Describe("#SNI", func() { Hosts: wildcardHosts, } + expectedDestinationRule.Spec.TrafficPolicy.ConnectionPool.Http = &istioapinetworkingv1beta1.ConnectionPoolSettings_HTTPSettings{ + UseClientProtocol: true, + } expectedDestinationRule.Spec.TrafficPolicy.LoadBalancer = &istioapinetworkingv1beta1.LoadBalancerSettings{ LbPolicy: &istioapinetworkingv1beta1.LoadBalancerSettings_Simple{ Simple: istioapinetworkingv1beta1.LoadBalancerSettings_LEAST_REQUEST, diff --git a/pkg/component/test/istiocomponent.go b/pkg/component/test/istiocomponent.go index 0c8db7b103a..e94f6691df1 100644 --- a/pkg/component/test/istiocomponent.go +++ b/pkg/component/test/istiocomponent.go @@ -19,6 +19,7 @@ func CmpOptsForDestinationRule() cmp.Option { istioapinetworkingv1beta1.DestinationRule{}, istioapinetworkingv1beta1.TrafficPolicy{}, istioapinetworkingv1beta1.ConnectionPoolSettings{}, + istioapinetworkingv1beta1.ConnectionPoolSettings_HTTPSettings{}, istioapinetworkingv1beta1.ConnectionPoolSettings_TCPSettings{}, istioapinetworkingv1beta1.ConnectionPoolSettings_TCPSettings_TcpKeepalive{}, istioapinetworkingv1beta1.LoadBalancerSettings{}, diff --git a/pkg/utils/istio/destinationrule.go b/pkg/utils/istio/destinationrule.go index 7625a85598b..2b36e2d5535 100644 --- a/pkg/utils/istio/destinationrule.go +++ b/pkg/utils/istio/destinationrule.go @@ -35,6 +35,9 @@ func DestinationRuleWithTLSTermination(destinationRule *istionetworkingv1beta1.D CredentialName: caSecret, Sni: sniHost, }, + &istioapinetworkingv1beta1.ConnectionPoolSettings_HTTPSettings{ + UseClientProtocol: true, + }, ) } @@ -55,6 +58,7 @@ func DestinationRuleWithLocalityPreferenceAndTLS(destinationRule *istionetworkin MinHealthPercent: 0, }, tls, + nil, ) } @@ -65,6 +69,7 @@ func destinationRuleWithTrafficPolicy( loadbalancer *istioapinetworkingv1beta1.LoadBalancerSettings, outlierDetection *istioapinetworkingv1beta1.OutlierDetection, tls *istioapinetworkingv1beta1.ClientTLSSettings, + httpConnectionPool *istioapinetworkingv1beta1.ConnectionPoolSettings_HTTPSettings, ) func() error { return func() error { destinationRule.Labels = labels @@ -80,6 +85,7 @@ func destinationRuleWithTrafficPolicy( Interval: &durationpb.Duration{Seconds: 75}, }, }, + Http: httpConnectionPool, }, LoadBalancer: loadbalancer, OutlierDetection: outlierDetection, From b39c89d131e3c656e54b3382a337b219f93398da Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Sat, 15 Nov 2025 00:19:33 +0100 Subject: [PATCH 117/176] Update module github.com/docker/cli to v29.0.1+incompatible (#13462) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b3f9e818ba8..7fb8b91f953 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/containerd/errdefs v1.0.0 github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 - github.com/docker/cli v29.0.0+incompatible + github.com/docker/cli v29.0.1+incompatible github.com/fluent/fluent-operator/v3 v3.5.0 github.com/gardener/cert-management v0.19.0 github.com/gardener/dependency-watchdog v1.6.0 diff --git a/go.sum b/go.sum index babe54b39f1..6125c0e5c32 100644 --- a/go.sum +++ b/go.sum @@ -188,8 +188,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/cli v29.0.0+incompatible h1:KgsN2RUFMNM8wChxryicn4p46BdQWpXOA1XLGBGPGAw= -github.com/docker/cli v29.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.0.1+incompatible h1:EnvMEAR9Ro5xQEKbMitlabj5vCDY0vwcDyY/Lsow7FQ= +github.com/docker/cli v29.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= From c27c96600ce9db9e7e4a19422843b995c473099d Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Sat, 15 Nov 2025 02:09:19 +0100 Subject: [PATCH 118/176] Update dependency GoogleContainerTools/skaffold to v2.17.0 (#13453) --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 49af4ce7c33..09d89973e0a 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -77,7 +77,7 @@ PROMTOOL_VERSION ?= 3.7.3 # renovate: datasource=github-releases depName=protocolbuffers/protobuf PROTOC_VERSION ?= v33.1 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold -SKAFFOLD_VERSION ?= v2.16.1 +SKAFFOLD_VERSION ?= v2.17.0 # renovate: datasource=github-releases depName=mikefarah/yq YQ_VERSION ?= v4.48.1 # renovate: datasource=github-releases depName=ironcore-dev/vgopath From c8cc4df3b56f416d5018d32f8e07a8048a4fd3be Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Sat, 15 Nov 2025 02:09:29 +0100 Subject: [PATCH 119/176] Update dependency projectcalico/calico to v3.31.1 (#13463) --- dev-setup/kind/calico/base/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-setup/kind/calico/base/kustomization.yaml b/dev-setup/kind/calico/base/kustomization.yaml index 6c983c31733..8f36aadf8da 100644 --- a/dev-setup/kind/calico/base/kustomization.yaml +++ b/dev-setup/kind/calico/base/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- https://raw.githubusercontent.com/projectcalico/calico/v3.31.0/manifests/calico.yaml +- https://raw.githubusercontent.com/projectcalico/calico/v3.31.1/manifests/calico.yaml images: - name: docker.io/calico/cni From d8ce263975cbc9dfe67434c34f82adf823d78ec9 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Sat, 15 Nov 2025 06:56:19 +0100 Subject: [PATCH 120/176] Update dependency mikefarah/yq to v4.48.2 (#13468) --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 09d89973e0a..8a1eed9140d 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -79,7 +79,7 @@ PROTOC_VERSION ?= v33.1 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold SKAFFOLD_VERSION ?= v2.17.0 # renovate: datasource=github-releases depName=mikefarah/yq -YQ_VERSION ?= v4.48.1 +YQ_VERSION ?= v4.48.2 # renovate: datasource=github-releases depName=ironcore-dev/vgopath VGOPATH_VERSION ?= v0.1.9 # renovate: datasource=github-releases depName=crate-ci/typos From fa004f89419e927a4a48c16c1bfdb83dbe407ad1 Mon Sep 17 00:00:00 2001 From: Erik Schubert Date: Mon, 17 Nov 2025 11:29:46 +0100 Subject: [PATCH 121/176] Use --skip-metadata for `ctr images pull` in `gardener-node-agent`'s `init` script (#13265) * Use --skip-metadata for ctr images pull in node-agent init script * Fix tests * Remove alphabetic characters from ctr version in init script --- .../operatingsystemconfig/nodeinit/gardenadm_test.go | 7 ++++++- .../operatingsystemconfig/nodeinit/nodeinit_test.go | 7 ++++++- .../nodeinit/templates/scripts/init.tpl.sh | 12 +++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pkg/component/extensions/operatingsystemconfig/nodeinit/gardenadm_test.go b/pkg/component/extensions/operatingsystemconfig/nodeinit/gardenadm_test.go index 1565b5a56a3..fb7d0eb8c4e 100644 --- a/pkg/component/extensions/operatingsystemconfig/nodeinit/gardenadm_test.go +++ b/pkg/component/extensions/operatingsystemconfig/nodeinit/gardenadm_test.go @@ -85,7 +85,12 @@ unmount() { trap unmount EXIT echo "> Pull gardenadm image and mount it to the temporary directory" -ctr images pull --hosts-dir "/etc/containerd/certs.d" "` + image + `" +CTR_MAJOR=$(ctr version | grep Version | tail -n1 | awk '{print $2}' | cut -d '.' -f 1 | sed 's/[a-zA-Z]//g') +CTR_EXTRA_ARGS="" +if [ "$CTR_MAJOR" -gt 1 ]; then + CTR_EXTRA_ARGS="--skip-metadata" +fi +ctr images pull $CTR_EXTRA_ARGS --hosts-dir "/etc/containerd/certs.d" "` + image + `" ctr images mount "` + image + `" "$tmp_dir" echo "> Copy gardenadm binary to host (/opt/bin) and make it executable" diff --git a/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go b/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go index 4ac0bdcc9da..3462ea2661c 100644 --- a/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go +++ b/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go @@ -127,7 +127,12 @@ unmount() { trap unmount EXIT echo "> Pull gardener-node-agent image and mount it to the temporary directory" -ctr images pull --hosts-dir "/etc/containerd/certs.d" "` + image + `" +CTR_MAJOR=$(ctr version | grep Version | tail -n1 | awk '{print $2}' | cut -d '.' -f 1 | sed 's/[a-zA-Z]//g') +CTR_EXTRA_ARGS="" +if [ "$CTR_MAJOR" -gt 1 ]; then + CTR_EXTRA_ARGS="--skip-metadata" +fi +ctr images pull $CTR_EXTRA_ARGS --hosts-dir "/etc/containerd/certs.d" "` + image + `" ctr images mount "` + image + `" "$tmp_dir" echo "> Copy gardener-node-agent binary to host (/opt/bin) and make it executable" diff --git a/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh b/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh index adcac154337..3ccc021633e 100644 --- a/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh +++ b/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh @@ -12,7 +12,17 @@ unmount() { trap unmount EXIT echo "> Pull {{ .binaryName }} image and mount it to the temporary directory" -ctr images pull --hosts-dir "/etc/containerd/certs.d" "{{ .image }}" +{{- /* +ctr v2 pulls manifests for all platforms of a multi-arch image by default. Mirror registries +might not copy manifests for unused architectures, which causes the default pull command to fail. +Ref: https://github.com/containerd/containerd/pull/9029#issuecomment-1706963854 +*/}} +CTR_MAJOR=$(ctr version | grep Version | tail -n1 | awk '{print $2}' | cut -d '.' -f 1 | sed 's/[a-zA-Z]//g') +CTR_EXTRA_ARGS="" +if [ "$CTR_MAJOR" -gt 1 ]; then + CTR_EXTRA_ARGS="--skip-metadata" +fi +ctr images pull $CTR_EXTRA_ARGS --hosts-dir "/etc/containerd/certs.d" "{{ .image }}" ctr images mount "{{ .image }}" "$tmp_dir" echo "> Copy {{ .binaryName }} binary to host ({{ .binaryDirectory }}) and make it executable" From 2413790b52df5d7fecd3d019864cfb87a834157f Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Mon, 17 Nov 2025 12:29:53 +0200 Subject: [PATCH 122/176] Clean up unused consts (#13461) --- pkg/gardenlet/apis/config/v1alpha1/types.go | 12 ------------ pkg/scheduler/apis/config/v1alpha1/types.go | 8 -------- 2 files changed, 20 deletions(-) diff --git a/pkg/gardenlet/apis/config/v1alpha1/types.go b/pkg/gardenlet/apis/config/v1alpha1/types.go index 0eecac89e33..99d4f1a7aad 100644 --- a/pkg/gardenlet/apis/config/v1alpha1/types.go +++ b/pkg/gardenlet/apis/config/v1alpha1/types.go @@ -719,18 +719,6 @@ const ( // By default we set this to 0 so that then BackupEntryController will trigger deletion immediately. DefaultBackupEntryDeletionGracePeriodHours = 0 - // DefaultDiscoveryDirName is the name of the default directory used for discovering Kubernetes APIs. - DefaultDiscoveryDirName = "gardenlet-discovery" - - // DefaultDiscoveryCacheDirName is the name of the default directory used for the discovery cache. - DefaultDiscoveryCacheDirName = "cache" - - // DefaultDiscoveryHTTPCacheDirName is the name of the default directory used for the discovery HTTP cache. - DefaultDiscoveryHTTPCacheDirName = "http-cache" - - // DefaultDiscoveryTTL is the default ttl for the cached discovery client. - DefaultDiscoveryTTL = 10 * time.Second - // DefaultControllerConcurrentSyncs is a default value for concurrent syncs for controllers. DefaultControllerConcurrentSyncs = 20 diff --git a/pkg/scheduler/apis/config/v1alpha1/types.go b/pkg/scheduler/apis/config/v1alpha1/types.go index 8ef3e085947..bc6f6993158 100644 --- a/pkg/scheduler/apis/config/v1alpha1/types.go +++ b/pkg/scheduler/apis/config/v1alpha1/types.go @@ -5,8 +5,6 @@ package v1alpha1 import ( - "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" ) @@ -22,12 +20,6 @@ const ( SchedulerDefaultLockObjectNamespace = "garden" // SchedulerDefaultLockObjectName is the default lock name for leader election. SchedulerDefaultLockObjectName = "gardener-scheduler-leader-election" - // SchedulerDefaultConfigurationConfigMapNamespace is the namespace of the scheduler configuration config map - SchedulerDefaultConfigurationConfigMapNamespace = "garden" - // SchedulerDefaultConfigurationConfigMapName is the name of the scheduler configuration config map - SchedulerDefaultConfigurationConfigMapName = "gardener-scheduler-configmap" - // DefaultDiscoveryTTL is the default ttl for the cached discovery client. - DefaultDiscoveryTTL = 10 * time.Second // LogLevelDebug is the debug log level, i.e. the most verbose. LogLevelDebug = "debug" From f2b12a89277582f63791da22d6ff9ae8fc0d492f Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 17 Nov 2025 14:55:46 +0100 Subject: [PATCH 123/176] Update dependency gardener/gardener-metrics-exporter to v0.42.0 (#13455) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 6d39c7731f9..320c1ca6f19 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -311,7 +311,7 @@ images: - name: gardener-metrics-exporter sourceRepository: github.com/gardener/gardener-metrics-exporter repository: europe-docker.pkg.dev/gardener-project/releases/gardener/metrics-exporter - tag: "0.41.0" + tag: "0.42.0" resourceId: name: metrics-exporter - name: node-exporter From 5c1ba1d485012f7759fb0d2f0f30b00646e22caa Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 17 Nov 2025 16:43:47 +0100 Subject: [PATCH 124/176] Update dependency kyverno/kyverno to v1.16.0 (#13428) --- example/provider-extensions/kyverno/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/provider-extensions/kyverno/kustomization.yaml b/example/provider-extensions/kyverno/kustomization.yaml index ad73a020620..fc5276abbce 100644 --- a/example/provider-extensions/kyverno/kustomization.yaml +++ b/example/provider-extensions/kyverno/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- https://github.com/kyverno/kyverno/releases/download/v1.15.2/install.yaml +- https://github.com/kyverno/kyverno/releases/download/v1.16.0/install.yaml - kyverno-poddisruptionbudget.yaml - kyverno-rbac.yaml From c54c59266fb4ed5efe106b3a4c0272d581aced2f Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 17 Nov 2025 16:43:55 +0100 Subject: [PATCH 125/176] Update dependency gardener/gardener-extension-networking-calico to v1.53.0 (#13447) --- .../components/controllerregistration/kustomization.yaml | 2 +- example/provider-local/garden/base/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-setup/extensions/networking-calico/components/controllerregistration/kustomization.yaml b/dev-setup/extensions/networking-calico/components/controllerregistration/kustomization.yaml index b1059050ec9..18451bc8522 100644 --- a/dev-setup/extensions/networking-calico/components/controllerregistration/kustomization.yaml +++ b/dev-setup/extensions/networking-calico/components/controllerregistration/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: -- https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.52.0/example/controller-registration.yaml +- https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.53.0/example/controller-registration.yaml diff --git a/example/provider-local/garden/base/kustomization.yaml b/example/provider-local/garden/base/kustomization.yaml index 318b12267f5..aacf6926218 100644 --- a/example/provider-local/garden/base/kustomization.yaml +++ b/example/provider-local/garden/base/kustomization.yaml @@ -15,4 +15,4 @@ components: resources: - https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.0/example/controller-registration.yaml -- https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.52.0/example/controller-registration.yaml +- https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.53.0/example/controller-registration.yaml From 0d16cf494ebfeaa8c8a56869f9d7b9cf3122cb79 Mon Sep 17 00:00:00 2001 From: Tobias Schlicht <155947440+tobschli@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:31:46 +0100 Subject: [PATCH 126/176] [GEP-28] `gardenadm connect`: Enable `shoot/state` controller in `gardenlet` (#13432) * Adapt `shoot-state` reconciler for SHSC * Add `shoot-state` reconciler to `gardenlet` for SHSC * Adapt `shoot-authorizer` * Update `skaffold` * Implement PR feedback * Implement PR feedback --- cmd/gardenlet/app/app.go | 4 + .../admission/shootrestriction/handler.go | 12 +++ .../shootrestriction/handler_test.go | 60 ++++++++++++++ .../webhook/auth/shoot/authorizer.go | 7 ++ .../webhook/auth/shoot/authorizer_test.go | 79 +++++++++++++++++++ pkg/gardenlet/controller/add.go | 8 ++ pkg/utils/gardener/shootstate/machines.go | 5 ++ pkg/utils/gardener/shootstate/shootstate.go | 30 +++---- 8 files changed, 191 insertions(+), 14 deletions(-) diff --git a/cmd/gardenlet/app/app.go b/cmd/gardenlet/app/app.go index 0ebde01e40a..3c46cbfe6cf 100644 --- a/cmd/gardenlet/app/app.go +++ b/cmd/gardenlet/app/app.go @@ -358,6 +358,10 @@ func (g *garden) Start(ctx context.Context) error { Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: g.selfHostedShootMeta.Name}), Namespaces: map[string]cache.Config{g.selfHostedShootMeta.Namespace: {}}, }, + &gardencorev1beta1.ShootState{}: { + Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: g.selfHostedShootMeta.Name}), + Namespaces: map[string]cache.Config{g.selfHostedShootMeta.Namespace: {}}, + }, &seedmanagementv1alpha1.Gardenlet{}: { Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: gardenlet.ResourcePrefixSelfHostedShoot + g.selfHostedShootMeta.Name}), Namespaces: map[string]cache.Config{g.selfHostedShootMeta.Namespace: {}}, diff --git a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go index 354257c635a..5ef5ca23089 100644 --- a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go +++ b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go @@ -38,6 +38,7 @@ var ( leaseResource = coordinationv1.Resource("leases") projectResource = gardencorev1beta1.Resource("projects") shootResource = gardencorev1beta1.Resource("shoots") + shootStateResource = gardencorev1beta1.Resource("shootstates") ) // Handler restricts requests made by shoot gardenlets. @@ -74,6 +75,9 @@ func (h *Handler) Handle(_ context.Context, request admission.Request) admission case leaseResource: return h.admitCreateWithResourcePrefix(gardenletShootInfo, request) + case shootStateResource: + return h.admitShootState(gardenletShootInfo, request) + default: log.Info( "Unhandled resource request", @@ -113,6 +117,14 @@ func (h *Handler) admitCertificateSigningRequest(gardenletShootInfo types.Namesp return h.admit(gardenletShootInfo, types.NamespacedName{Name: name, Namespace: namespace}) } +func (h *Handler) admitShootState(gardenletShootInfo types.NamespacedName, request admission.Request) admission.Response { + if request.Operation != admissionv1.Create { + return admission.Errored(http.StatusBadRequest, fmt.Errorf("unexpected operation: %q", request.Operation)) + } + + return h.admit(gardenletShootInfo, types.NamespacedName{Name: request.Name, Namespace: request.Namespace}) +} + func (h *Handler) admit(gardenletShootInfo, objectShootInfo types.NamespacedName) admission.Response { // Allow request if the shoot the gardenlet is responsible for matches with the shoot related to the object. if gardenletShootInfo.Name == objectShootInfo.Name && gardenletShootInfo.Namespace == objectShootInfo.Namespace { diff --git a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go index 117fca007db..9940d09d4f9 100644 --- a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go +++ b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go @@ -426,6 +426,66 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d }) }) + Context("when requested for ShootStates", func() { + var name string + + BeforeEach(func() { + name = "foo" + + request.Name = name + request.UserInfo = gardenletUser + request.Resource = metav1.GroupVersionResource{ + Group: gardencorev1beta1.SchemeGroupVersion.Group, + Version: gardencorev1beta1.SchemeGroupVersion.Version, + Resource: "shootstates", + } + }) + + DescribeTable("should not allow the request because no allowed verb", + func(operation admissionv1.Operation) { + request.Operation = operation + + Expect(handler.Handle(ctx, request)).To(Equal(admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: int32(http.StatusBadRequest), + Message: fmt.Sprintf("unexpected operation: %q", operation), + }, + }, + })) + }, + + Entry("update", admissionv1.Update), + Entry("delete", admissionv1.Delete), + ) + + Context("when operation is create", func() { + BeforeEach(func() { + request.Operation = admissionv1.Create + }) + + It("should return an error because the requestor is not responsible for the resource", func() { + Expect(handler.Handle(ctx, request)).To(Equal(admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: int32(http.StatusForbidden), + Message: "object does not belong to shoot " + shootNamespace + "/" + shootName, + }, + }, + })) + }) + + It("should return success because the requestor is responsible for the resource", func() { + request.Name = shootName + request.Namespace = shootNamespace + + Expect(handler.Handle(ctx, request)).To(Equal(responseAllowed)) + }) + }) + }) + Context("gardenadm client", func() { Context("when requested for Projects", func() { var ( diff --git a/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go b/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go index d6348896ae0..f3f33c022c4 100644 --- a/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go +++ b/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go @@ -68,6 +68,7 @@ var ( secretResource = corev1.Resource("secrets") secretBindingResource = gardencorev1beta1.Resource("secretbindings") shootResource = gardencorev1beta1.Resource("shoots") + shootStateResource = gardencorev1beta1.Resource("shootstates") ) func (a *authorizer) Authorize(ctx context.Context, attrs auth.Attributes) (auth.Decision, string, error) { @@ -143,6 +144,12 @@ func (a *authorizer) Authorize(ctx context.Context, attrs auth.Attributes) (auth authwebhook.WithAllowedSubresources("status"), ) + case shootStateResource: + return requestAuthorizer.Check(graph.VertexTypeShootState, attrs, + authwebhook.WithAllowedVerbs("get", "list", "watch", "patch"), + authwebhook.WithAlwaysAllowedVerbs("create"), + ) + default: log.Info( "Unhandled resource request", diff --git a/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go b/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go index baad1267cca..adb03b8708d 100644 --- a/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go +++ b/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go @@ -614,6 +614,85 @@ var _ = Describe("Shoot", func() { }) }) + Context("when requested for ShootStates", func() { + var ( + name, namespace string + attrs *auth.AttributesRecord + ) + + BeforeEach(func() { + name, namespace = "foo", "bar" + attrs = &auth.AttributesRecord{ + User: gardenletUser, + Name: name, + Namespace: namespace, + APIGroup: gardencorev1beta1.SchemeGroupVersion.Group, + Resource: "shootstates", + ResourceRequest: true, + Verb: "get", + } + }) + + DescribeTable("should allow without consulting the graph because verb is create", + func(verb string) { + attrs.Verb = verb + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionAllow)) + Expect(reason).To(BeEmpty()) + }, + + Entry("create", "create"), + ) + + DescribeTable("should return correct result if path exists", + func(verb string) { + attrs.Verb = verb + + graph.EXPECT().HasPathFrom(graphutils.VertexTypeShootState, namespace, name, graphutils.VertexTypeShoot, shootNamespace, shootName).Return(true) + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionAllow)) + Expect(reason).To(BeEmpty()) + + graph.EXPECT().HasPathFrom(graphutils.VertexTypeShootState, namespace, name, graphutils.VertexTypeShoot, shootNamespace, shootName).Return(false) + decision, reason, err = authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("no relationship found")) + }, + + Entry("get", "get"), + Entry("list", "list"), + Entry("watch", "watch"), + Entry("patch", "patch"), + ) + + DescribeTable("should have no opinion because verb is not allowed", + func(verb string) { + attrs.Verb = verb + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("only the following verbs are allowed for this resource type: [create get list patch watch]")) + + }, + Entry("delete", "delete"), + Entry("deletecollection", "deletecollection"), + ) + + It("should have no opinion because no allowed subresource", func() { + attrs.Subresource = "foo" + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("only the following subresources are allowed for this resource type: []")) + }) + }) + Context("when requested for Secrets", func() { var ( name, namespace string diff --git a/pkg/gardenlet/controller/add.go b/pkg/gardenlet/controller/add.go index d26ce197f65..5da00252e2d 100644 --- a/pkg/gardenlet/controller/add.go +++ b/pkg/gardenlet/controller/add.go @@ -31,6 +31,7 @@ import ( "github.com/gardener/gardener/pkg/gardenlet/controller/seed" "github.com/gardener/gardener/pkg/gardenlet/controller/shoot" "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/lease" + "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/state" "github.com/gardener/gardener/pkg/gardenlet/controller/tokenrequestor/workloadidentity" "github.com/gardener/gardener/pkg/gardenlet/controller/vpaevictionrequirements" "github.com/gardener/gardener/pkg/healthz" @@ -94,6 +95,13 @@ func AddToManager( return fmt.Errorf("failed adding shoot-lease reconciler: %w", err) } + // TODO(tobschli): Remove this once all shoot reconcilers are added via `shoot.AddToManager`. + if err := (&state.Reconciler{ + Config: *cfg.Controllers.ShootState, + }).AddToManager(mgr, gardenCluster, seedCluster); err != nil { + return fmt.Errorf("failed adding ShootState controller: %w", err) + } + return nil } diff --git a/pkg/utils/gardener/shootstate/machines.go b/pkg/utils/gardener/shootstate/machines.go index 346fc6cb589..261e21c6d51 100644 --- a/pkg/utils/gardener/shootstate/machines.go +++ b/pkg/utils/gardener/shootstate/machines.go @@ -14,6 +14,7 @@ import ( "slices" machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,6 +39,10 @@ func computeMachineState(ctx context.Context, seedClient client.Client, namespac machineDeployments := &machinev1alpha1.MachineDeploymentList{} if err := seedClient.List(ctx, machineDeployments, client.InNamespace(namespace)); err != nil { + // In case of self-hosted shoot clusters with unmanaged infrastructure, the MCM resources are not present. + if meta.IsNoMatchError(err) { + return nil, nil + } return nil, err } diff --git a/pkg/utils/gardener/shootstate/shootstate.go b/pkg/utils/gardener/shootstate/shootstate.go index c32e9d4287b..3a45f98f37f 100644 --- a/pkg/utils/gardener/shootstate/shootstate.go +++ b/pkg/utils/gardener/shootstate/shootstate.go @@ -131,22 +131,24 @@ func computeGardenerData( return nil, err } - machineStateJSON, err := json.Marshal(machineState) - if err != nil { - return nil, fmt.Errorf("failed marshalling machine state to JSON: %w", err) - } + if machineState != nil { + machineStateJSON, err := json.Marshal(machineState) + if err != nil { + return nil, fmt.Errorf("failed marshalling machine state to JSON: %w", err) + } - machineStateJSONCompressed, err := compressMachineState(machineStateJSON) - if err != nil { - return nil, fmt.Errorf("failed compressing machine state data: %w", err) - } + machineStateJSONCompressed, err := compressMachineState(machineStateJSON) + if err != nil { + return nil, fmt.Errorf("failed compressing machine state data: %w", err) + } - if machineStateJSONCompressed != nil { - secretsToPersist = append(secretsToPersist, gardencorev1beta1.GardenerResourceData{ - Name: v1beta1constants.DataTypeMachineState, - Type: v1beta1constants.DataTypeMachineState, - Data: runtime.RawExtension{Raw: machineStateJSONCompressed}, - }) + if machineStateJSONCompressed != nil { + secretsToPersist = append(secretsToPersist, gardencorev1beta1.GardenerResourceData{ + Name: v1beta1constants.DataTypeMachineState, + Type: v1beta1constants.DataTypeMachineState, + Data: runtime.RawExtension{Raw: machineStateJSONCompressed}, + }) + } } return secretsToPersist, nil From 02ede79c359c750c4c01213b7fc0f5b0ac93013e Mon Sep 17 00:00:00 2001 From: Tobias Schlicht <155947440+tobschli@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:31:53 +0100 Subject: [PATCH 127/176] Remove outdated info from version upgrade guide (#13470) --- docs/development/new-kubernetes-version.md | 27 ---------------------- 1 file changed, 27 deletions(-) diff --git a/docs/development/new-kubernetes-version.md b/docs/development/new-kubernetes-version.md index 57759ce3fa3..6c2c59ac4bf 100644 --- a/docs/development/new-kubernetes-version.md +++ b/docs/development/new-kubernetes-version.md @@ -172,33 +172,6 @@ After the PR in `gardener/gardener` for the support of the new version has been Provider extensions are using upstream `cloud-controller-manager` images. Make sure to adopt the new `cloud-controller-manager` release for the new Kubernetes minor version ([example PR](https://github.com/gardener/gardener-extension-provider-aws/pull/1055)). -Some of the cloud providers are not using upstream `cloud-controller-manager` images for some of the supported Kubernetes versions. -Instead, we build and maintain the images ourselves: - -- [cloud-provider-gcp](https://github.com/gardener/cloud-provider-gcp) - -Use the instructions below in case you need to maintain a release branch for such `cloud-controller-manager` image: - -
- -Expand the instructions! - -Until we switch to upstream images, you need to update the Kubernetes dependencies and release a new image. -The required steps are as follows: - -- Checkout the `legacy-cloud-provider` branch of the respective repository -- Bump the versions in the `Dockerfile` ([example commit](https://github.com/gardener/cloud-provider-gcp/commit/b7eb3f56b252aaf29adc78406672574b1bc17495)). -- Update the `VERSION` to `vX.Y.Z-dev` where `Z` is the latest available Kubernetes patch version for the `vX.Y` minor version. -- Update the `k8s.io/*` dependencies in the `go.mod` file to `vX.Y.Z` and run `go mod tidy` ([example commit](https://github.com/gardener/cloud-provider-gcp/commit/d41cc9f035bcc4893b40d90a4f617c4d436c5d62)). -- Checkout a new `release-vX.Y` branch and release it ([example](https://github.com/gardener/cloud-provider-gcp/commits/release-v1.23)) - -> As you are already on it, it is great if you also bump the `k8s.io/*` dependencies for the last three minor releases as well. -In this case, you need to check out the `release-vX.{Y-{1,2,3}}` branches and only perform the last three steps ([example branch](https://github.com/gardener/cloud-provider-gcp/commits/release-v1.20), [example commit](https://github.com/gardener/cloud-provider-gcp/commit/372aa43fbacdeb76b3da9f6fad6cfd924d916227)). - -Now you need to update the new releases in the `imagevector/images.yaml` of the respective provider extension so that they are used (see this [example commit](https://github.com/gardener/gardener-extension-provider-aws/pull/942/commits/7e5c0d95ff95d65459d13ae7f79a030049322c71) for reference). - -
- #### Maintaining Additional Images Provider extensions might also deploy additional images other than `cloud-controller-manager` that are specific for a given Kubernetes minor version. From de2b1f326e0daf970bba8a0667e068987818b6f9 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Mon, 17 Nov 2025 20:14:59 +0100 Subject: [PATCH 128/176] Update module github.com/docker/cli to v29.0.2+incompatible (#13473) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7fb8b91f953..4973096d607 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/containerd/errdefs v1.0.0 github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 - github.com/docker/cli v29.0.1+incompatible + github.com/docker/cli v29.0.2+incompatible github.com/fluent/fluent-operator/v3 v3.5.0 github.com/gardener/cert-management v0.19.0 github.com/gardener/dependency-watchdog v1.6.0 diff --git a/go.sum b/go.sum index 6125c0e5c32..6edde6dbdab 100644 --- a/go.sum +++ b/go.sum @@ -188,8 +188,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/cli v29.0.1+incompatible h1:EnvMEAR9Ro5xQEKbMitlabj5vCDY0vwcDyY/Lsow7FQ= -github.com/docker/cli v29.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.0.2+incompatible h1:iLuKy2GWOSLXGp8feLYBJQVDv7m/8xoofz6lPq41x6A= +github.com/docker/cli v29.0.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= From 0dded52a46d421abccad3a5c088bece3334b40e3 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Mon, 17 Nov 2025 23:02:58 +0200 Subject: [PATCH 129/176] chore(vpa): Instrument few more VPA resources with update mode `Recreate` (#13411) * Instrument `etcd-{events,main}` VPAs * Instrument `etcd-druid` VPA * Instrument `cluster-autoscaler` VPA --- .../autoscaling/clusterautoscaler/cluster_autoscaler.go | 2 +- .../autoscaling/clusterautoscaler/cluster_autoscaler_test.go | 2 +- pkg/component/etcd/etcd/bootstrap.go | 2 +- pkg/component/etcd/etcd/bootstrap_test.go | 2 +- pkg/component/etcd/etcd/etcd.go | 2 +- pkg/component/etcd/etcd/etcd_test.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler.go b/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler.go index 0c32d0859bb..b430c529a78 100644 --- a/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler.go +++ b/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler.go @@ -268,7 +268,7 @@ func (c *clusterAutoscaler) Deploy(ctx context.Context) error { Name: v1beta1constants.DeploymentNameClusterAutoscaler, } vpa.Spec.UpdatePolicy = &vpaautoscalingv1.PodUpdatePolicy{ - UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto), + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeRecreate), } vpa.Spec.ResourcePolicy = &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{{ diff --git a/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler_test.go b/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler_test.go index 77632dab40b..9fdfee7926d 100644 --- a/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler_test.go +++ b/pkg/component/autoscaling/clusterautoscaler/cluster_autoscaler_test.go @@ -182,7 +182,7 @@ var _ = Describe("ClusterAutoscaler", func() { Name: deploymentName, }, UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ - UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto), + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeRecreate), }, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{{ diff --git a/pkg/component/etcd/etcd/bootstrap.go b/pkg/component/etcd/etcd/bootstrap.go index 7ad57bd1844..9809b3bd4d4 100644 --- a/pkg/component/etcd/etcd/bootstrap.go +++ b/pkg/component/etcd/etcd/bootstrap.go @@ -246,7 +246,7 @@ func (b *bootstrapper) Deploy(ctx context.Context) error { }, } - vpaUpdateMode = vpaautoscalingv1.UpdateModeAuto + vpaUpdateMode = vpaautoscalingv1.UpdateModeRecreate vpa = &vpaautoscalingv1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: druidVPAName, diff --git a/pkg/component/etcd/etcd/bootstrap_test.go b/pkg/component/etcd/etcd/bootstrap_test.go index 69aa555cc40..18ea2e4279a 100644 --- a/pkg/component/etcd/etcd/bootstrap_test.go +++ b/pkg/component/etcd/etcd/bootstrap_test.go @@ -252,7 +252,7 @@ var _ = Describe("Etcd", func() { Name: "etcd-druid", }, UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ - UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto), + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeRecreate), }, }, } diff --git a/pkg/component/etcd/etcd/etcd.go b/pkg/component/etcd/etcd/etcd.go index 301fc83fae9..7e9c890eaf0 100644 --- a/pkg/component/etcd/etcd/etcd.go +++ b/pkg/component/etcd/etcd/etcd.go @@ -784,7 +784,7 @@ func (e *etcd) emptyVerticalPodAutoscaler() *vpaautoscalingv1.VerticalPodAutosca } func (e *etcd) reconcileVerticalPodAutoscaler(ctx context.Context, vpa *vpaautoscalingv1.VerticalPodAutoscaler, minAllowedETCD corev1.ResourceList) error { - vpaUpdateMode := vpaautoscalingv1.UpdateModeAuto + vpaUpdateMode := vpaautoscalingv1.UpdateModeRecreate containerPolicyOff := vpaautoscalingv1.ContainerScalingModeOff containerPolicyAuto := vpaautoscalingv1.ContainerScalingModeAuto controlledValues := vpaautoscalingv1.ContainerControlledValuesRequestsOnly diff --git a/pkg/component/etcd/etcd/etcd_test.go b/pkg/component/etcd/etcd/etcd_test.go index 2944a4be7a8..d3678589063 100644 --- a/pkg/component/etcd/etcd/etcd_test.go +++ b/pkg/component/etcd/etcd/etcd_test.go @@ -113,7 +113,7 @@ var _ = Describe("Etcd", func() { backupLeaderElectionEtcdConnectionTimeout = &metav1.Duration{Duration: 10 * time.Second} backupLeaderElectionReelectionPeriod = &metav1.Duration{Duration: 11 * time.Second} - vpaUpdateMode = vpaautoscalingv1.UpdateModeAuto + vpaUpdateMode = vpaautoscalingv1.UpdateModeRecreate containerPolicyOff = vpaautoscalingv1.ContainerScalingModeOff containerPolicyAuto = vpaautoscalingv1.ContainerScalingModeAuto controlledValues = vpaautoscalingv1.ContainerControlledValuesRequestsOnly From cd92224ece305a3b044c96c7f6dd51a9292b6d02 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:25:39 +0200 Subject: [PATCH 130/176] Make the `ShootValidator` admission plugin validating one (part 7) (#13352) * Make the `ShootValidator` admission plugin validating one * Adapt `ShootValidator` unit tests * Update docs * Move validation for the Shoot `.spec.cloudProfileName` and `.spec.cloudProfile` fields to the `ShootMutator` The order of execution is: 1. Mutating admission plugins 2. `strategy.PrepareForCreate` 3. `strategy.Validate` 4. Validating admission plugins Hence, a validating admission plugin such as `ShootValidator` always receives an object with both of the fields populated and the validation always fails. That's why the validation is moved to the `ShootMutator`. --- docs/concepts/apiserver-admission-plugins.md | 5 +- plugin/pkg/shoot/mutator/admission.go | 12 + plugin/pkg/shoot/mutator/admission_test.go | 14 + plugin/pkg/shoot/validator/admission.go | 12 +- plugin/pkg/shoot/validator/admission_test.go | 498 +++++++++---------- 5 files changed, 273 insertions(+), 268 deletions(-) diff --git a/docs/concepts/apiserver-admission-plugins.md b/docs/concepts/apiserver-admission-plugins.md index b5b10336d62..0726f115127 100644 --- a/docs/concepts/apiserver-admission-plugins.md +++ b/docs/concepts/apiserver-admission-plugins.md @@ -239,9 +239,6 @@ It mutates the `Shoot` in the following way: - It defaults the Shoot Kubernetes version (`.spec.kubernetes.version` and `.spec.provider.workers[].kubernetes.version`). - It defaults the Shoot machine image version (`.spec.provider.workers[].machine.image.{name,version}`). -Over time, the `ShootMutator` admission plugin will take over all the mutations that are performed by `ShootValidator`. -For more details, see https://github.com/gardener/gardener/issues/2158. - ## `ShootNodeLocalDNSEnabledByDefault` **Type**: Mutating. **Enabled by default**: No. @@ -295,7 +292,7 @@ will not be affected by this admission plugin. ## `ShootValidator` -**Type**: Mutating. **Enabled by default**: Yes. +**Type**: Validating. **Enabled by default**: Yes. This admission controller reacts on `CREATE`, `UPDATE` and `DELETE` operations for `Shoot`s. It validates certain configurations in the specification against the referred `CloudProfile` (e.g., machine images, machine types, used Kubernetes version, ...). diff --git a/plugin/pkg/shoot/mutator/admission.go b/plugin/pkg/shoot/mutator/admission.go index da5bcd75064..7307d15a967 100644 --- a/plugin/pkg/shoot/mutator/admission.go +++ b/plugin/pkg/shoot/mutator/admission.go @@ -152,6 +152,18 @@ func (m *MutateShoot) Admit(_ context.Context, a admission.Attributes, _ admissi } } + // Conceptually, the below validation belongs to the `ShootValidator` admission plugin. + // However it cannot be put there because: + // - `shootStrategy.PrepareForCreate` syncs the `.spec.cloudProfileName` and `.spec.cloudProfile` fields. + // - `PrepareForCreate` of a strategy implementation is executed before the validating admission plugins. + // Hence, a validating admission plugin such as `ShootValidator` always receives an object with both of the fields populated + // and the validation always fails. That's why the validation is moved to the `ShootMutator`. + if a.GetOperation() == admission.Create { + if len(ptr.Deref(shoot.Spec.CloudProfileName, "")) > 0 && shoot.Spec.CloudProfile != nil { + return fmt.Errorf("new shoot can only specify either cloudProfileName or cloudProfile reference") + } + } + cloudProfileSpec, err := gardenerutils.GetCloudProfileSpec(m.cloudProfileLister, m.namespacedCloudProfileLister, shoot) if err != nil { return apierrors.NewInternalError(fmt.Errorf("could not find referenced cloud profile: %w", err)) diff --git a/plugin/pkg/shoot/mutator/admission_test.go b/plugin/pkg/shoot/mutator/admission_test.go index 3b10fb2db60..6f12fa4376a 100644 --- a/plugin/pkg/shoot/mutator/admission_test.go +++ b/plugin/pkg/shoot/mutator/admission_test.go @@ -232,6 +232,20 @@ var _ = Describe("mutator", func() { err := admissionHandler.Admit(ctx, attrs, nil) Expect(err).To(BeInternalServerError()) }) + + Context("CloudProfile reference and CloudProfileName", func() { + It("should fail when both cloudProfileName and cloudProfile are provided for a new shoot", func() { + shoot.Spec.CloudProfileName = ptr.To("profile") + shoot.Spec.CloudProfile = &core.CloudProfileReference{ + Kind: "CloudProfile", + Name: "profile", + } + attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) + err := admissionHandler.Admit(ctx, attrs, nil) + + Expect(err).To(MatchError(ContainSubstring("new shoot can only specify either cloudProfileName or cloudProfile reference"))) + }) + }) }) Context("created-by annotation", func() { diff --git a/plugin/pkg/shoot/validator/admission.go b/plugin/pkg/shoot/validator/admission.go index f9d4e80cfcd..2d94e7119e8 100644 --- a/plugin/pkg/shoot/validator/admission.go +++ b/plugin/pkg/shoot/validator/admission.go @@ -177,10 +177,10 @@ func (v *ValidateShoot) ValidateInitialization() error { return nil } -var _ admission.MutationInterface = (*ValidateShoot)(nil) +var _ admission.ValidationInterface = (*ValidateShoot)(nil) -// Admit validates the Shoot details against the referenced CloudProfile. -func (v *ValidateShoot) Admit(ctx context.Context, a admission.Attributes, _ admission.ObjectInterfaces) error { +// Validate validates the Shoot details against the referenced CloudProfile and Seed. +func (v *ValidateShoot) Validate(ctx context.Context, a admission.Attributes, _ admission.ObjectInterfaces) error { // Wait until the caches have been synced if v.readyFunc == nil { v.AssignReadyFunc(func() bool { @@ -250,12 +250,6 @@ func (v *ValidateShoot) Admit(ctx context.Context, a admission.Attributes, _ adm } } - if a.GetOperation() == admission.Create { - if len(ptr.Deref(shoot.Spec.CloudProfileName, "")) > 0 && shoot.Spec.CloudProfile != nil { - return fmt.Errorf("new shoot can only specify either cloudProfileName or cloudProfile reference") - } - } - cloudProfileSpec, err := gardenerutils.GetCloudProfileSpec(v.cloudProfileLister, v.namespacedCloudProfileLister, shoot) if err != nil { return apierrors.NewInternalError(fmt.Errorf("could not find referenced cloud profile: %+v", err.Error())) diff --git a/plugin/pkg/shoot/validator/admission_test.go b/plugin/pkg/shoot/validator/admission_test.go index a317e2fa7f2..a3d34cd2077 100644 --- a/plugin/pkg/shoot/validator/admission_test.go +++ b/plugin/pkg/shoot/validator/admission_test.go @@ -37,7 +37,7 @@ import ( ) var _ = Describe("validator", func() { - Describe("#Admit", func() { + Describe("#Validate", func() { var ( ctx context.Context admissionHandler *ValidateShoot @@ -386,7 +386,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("already marked for deletion")) @@ -411,7 +411,7 @@ var _ = Describe("validator", func() { authorizeAttributes.Name = shoot.Name attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeInvalidError()) Expect(err.Error()).To(ContainSubstring("name must not exceed")) @@ -433,12 +433,12 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).NotTo(ContainSubstring("name must not exceed")) attrs = admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, nil) - err = admissionHandler.Admit(ctx, attrs, nil) + err = admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).NotTo(ContainSubstring("name must not exceed")) }) @@ -465,7 +465,7 @@ var _ = Describe("validator", func() { authorizeAttributes.Name = shoot.Name attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -485,7 +485,7 @@ var _ = Describe("validator", func() { authorizeAttributes.Name = shoot.Name attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeInvalidError()) Expect(err.Error()).To(ContainSubstring("name must not exceed")) @@ -517,7 +517,7 @@ var _ = Describe("validator", func() { shoot.Status.Constraints = constraints attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(match) }, Entry("should allow if set to True", []core.Condition{ @@ -563,7 +563,7 @@ var _ = Describe("validator", func() { It("should reject creating a shoot on a seed which is marked for deletion", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot schedule shoot '%s' on seed '%s' that is already marked for deletion", shoot.Name, seed.Name)) @@ -571,7 +571,7 @@ var _ = Describe("validator", func() { It("should allow no-op updates", func() { attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -581,7 +581,7 @@ var _ = Describe("validator", func() { shoot.Finalizers = []string{} attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -591,7 +591,7 @@ var _ = Describe("validator", func() { shoot.Annotations[v1beta1constants.ConfirmationDeletion] = "true" attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -600,7 +600,7 @@ var _ = Describe("validator", func() { shoot.Spec.Region = "other-region" attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot update spec of shoot '%s' on seed '%s' already marked for deletion", shoot.Name, seed.Name)) @@ -611,7 +611,7 @@ var _ = Describe("validator", func() { shoot.Annotations["foo"] = "bar" attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot update annotations of shoot '%s' on seed '%s' already marked for deletion", shoot.Name, seed.Name)) @@ -632,7 +632,7 @@ var _ = Describe("validator", func() { auth.EXPECT().Authorize(ctx, authorizeAttributes).Return(authorizer.DecisionAllow, "", nil) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -641,7 +641,7 @@ var _ = Describe("validator", func() { auth.EXPECT().Authorize(ctx, authorizeAttributes).Return(authorizer.DecisionDeny, "", nil) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("user %q is not allowed to set .spec.seedName for %q", userInfo.Name, "shoots"))) @@ -673,7 +673,7 @@ var _ = Describe("validator", func() { shoot.Spec.SeedName = &newSeed.Name attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("spec.seedName 'seed' cannot be changed to 'new-seed' by patching the shoot, please use the shoots/binding subresource"))) @@ -684,7 +684,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -693,7 +693,7 @@ var _ = Describe("validator", func() { shoot.Spec.SeedName = nil attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("spec.seedName is already set to 'seed' and cannot be changed to 'nil'"))) @@ -703,7 +703,7 @@ var _ = Describe("validator", func() { shoot.Spec.SeedName = nil attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("spec.seedName is already set to 'seed' and cannot be changed to 'nil'"))) @@ -713,7 +713,7 @@ var _ = Describe("validator", func() { oldShoot.Spec.SeedName = nil attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("spec.seedName 'nil' cannot be changed to 'seed' by patching the shoot, please use the shoots/binding subresource"))) @@ -722,7 +722,7 @@ var _ = Describe("validator", func() { It("should reject update of binding when shoot.spec.seedName is not nil and the binding has the same seedName", func() { attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(ContainSubstring("update of binding rejected, shoot is already assigned to the same seed"))) @@ -733,7 +733,7 @@ var _ = Describe("validator", func() { It("should reject because the referenced cloud profile was not found", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeInternalServerError()) }) @@ -752,18 +752,6 @@ var _ = Describe("validator", func() { shoot.Spec.CloudProfileName = nil }) - It("should fail when both cloudProfileName and cloudProfile are provided for a new shoot", func() { - shoot.Spec.CloudProfileName = ptr.To("profile") - shoot.Spec.CloudProfile = &core.CloudProfileReference{ - Kind: "CloudProfile", - Name: "profile", - } - attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) - - Expect(err).To(MatchError(ContainSubstring("either cloudProfileName or cloudProfile reference"))) - }) - It("should pass on update for a unchanged CloudProfile reference with CloudProfileName set accordingly", func() { shoot.Spec.CloudProfileName = ptr.To("profile") shoot.Spec.CloudProfile = &core.CloudProfileReference{ @@ -772,7 +760,7 @@ var _ = Describe("validator", func() { } oldShoot := shoot.DeepCopy() attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -780,7 +768,7 @@ var _ = Describe("validator", func() { It("should pass for a given CloudProfile by CloudProfileName", func() { shoot.Spec.CloudProfileName = ptr.To("profile") attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -791,7 +779,7 @@ var _ = Describe("validator", func() { Name: "profile", } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -802,7 +790,7 @@ var _ = Describe("validator", func() { Name: profileName, } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -818,7 +806,7 @@ var _ = Describe("validator", func() { Name: profileName, } attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -831,7 +819,7 @@ var _ = Describe("validator", func() { Name: profileName, } attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -856,7 +844,7 @@ var _ = Describe("validator", func() { Name: "another-namespaced-profile", } attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("cannot change from \"profile\" to \"another-namespaced-profile\" (root: \"another-root-profile\")"))) }) @@ -878,7 +866,7 @@ var _ = Describe("validator", func() { Name: "another-namespaced-profile", } attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("cannot change from \"profile\" to \"another-namespaced-profile\" (root: \"another-root-profile\")"))) }) @@ -894,7 +882,7 @@ var _ = Describe("validator", func() { Name: "profile", } attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -914,7 +902,7 @@ var _ = Describe("validator", func() { Name: profileName + "-1", } attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -935,7 +923,7 @@ var _ = Describe("validator", func() { Name: profileName + "-unrelated", } attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("cannot change from \"namespaced-profile\" (root: \"profile\") to \"namespaced-profile-unrelated\" (root: \"unrelated-profile\")"))) }) @@ -961,7 +949,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(MatchError(ContainSubstring("newly referenced cloud profile does not contain the machine type \"a-special-machine-type\" currently in use by worker \"testing\""))) }) @@ -991,7 +979,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(MatchError(ContainSubstring("newly referenced cloud profile does not contain the volume type \"a-special-volume-type\" currently in use by worker \"testing\""))) }) @@ -1034,7 +1022,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(MatchError(ContainSubstring("newly referenced cloud profile does not contain the machine image version \"gardenlinux@1592.1.0-dev\" currently in use by worker \"testing\""))) }) @@ -1045,7 +1033,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().CloudProfiles().Informer().GetStore().Add(&cloudProfile)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeInternalServerError()) }) @@ -1055,7 +1043,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeInternalServerError()) }) @@ -1071,7 +1059,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("provider type in shoot must equal provider type of referenced CloudProfile: %q", cloudProfile.Spec.Type)) @@ -1091,7 +1079,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("provider type in shoot must match provider type of referenced SecretBinding: %q", secretBinding.Provider.Type)) @@ -1114,7 +1102,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("provider type in shoot must match provider type of referenced CredentialsBinding: %q", credentialsBinding.Provider.Type)) @@ -1142,7 +1130,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("it is not allowed to change the referenced Secret when migrating from SecretBindingName to CredentialsBindingName")) @@ -1170,7 +1158,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("it is not allowed to change the referenced Secret when migrating from SecretBindingName to CredentialsBindingName")) @@ -1198,7 +1186,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should not error if and secret and credentials binding are nil", func() { @@ -1214,7 +1202,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -1228,7 +1216,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -1279,7 +1267,7 @@ var _ = Describe("validator", func() { auth.EXPECT().Authorize(ctx, authorizeAttributes).Return(authorizer.DecisionDeny, "", nil) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("user %q is not allowed to read the previously referenced Secret %q", userInfo.Name, shoot.Namespace+"/secret1"))) }) @@ -1310,7 +1298,7 @@ var _ = Describe("validator", func() { auth.EXPECT().Authorize(ctx, newAuthorizeAttributes).Return(authorizer.DecisionDeny, "", nil) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("user %q is not allowed to read the newly referenced Secret %q", userInfo.Name, shoot.Namespace+"/new-secret"))) }) @@ -1341,7 +1329,7 @@ var _ = Describe("validator", func() { auth.EXPECT().Authorize(ctx, newAuthorizeAttributes).Return(authorizer.DecisionAllow, "", nil) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) }) @@ -1363,7 +1351,7 @@ var _ = Describe("validator", func() { cloudProfile.Spec.Regions = nil attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(Not(HaveOccurred())) }) @@ -1372,7 +1360,7 @@ var _ = Describe("validator", func() { shoot.Spec.Region = "does-not-exist" attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("Unsupported value: \"does-not-exist\": supported values: \"europe\", \"asia\"")) @@ -1382,7 +1370,7 @@ var _ = Describe("validator", func() { cloudProfile.Spec.Regions[0].Zones = []gardencorev1beta1.AvailabilityZone{{Name: "not-available"}} attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(Not(HaveOccurred())) }) @@ -1391,7 +1379,7 @@ var _ = Describe("validator", func() { shoot.Spec.Region = "asia" attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("Unsupported value: \"europe-a\": supported values: \"asia-a\"")) @@ -1402,7 +1390,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Zones = append(shoot.Spec.Provider.Workers[0].Zones, "zone-1") attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("Unsupported value: \"europe-a\": supported values: \"zone-1\", \"zone-2\"")) @@ -1416,7 +1404,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -1429,7 +1417,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -1444,7 +1432,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -1459,7 +1447,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -1469,7 +1457,7 @@ var _ = Describe("validator", func() { shoot.Spec.AccessRestrictions = []core.AccessRestrictionWithOptions{{AccessRestriction: core.AccessRestriction{Name: "foo"}}} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("spec.accessRestrictions[0]: Unsupported value: \"foo\"")) @@ -1482,7 +1470,7 @@ var _ = Describe("validator", func() { shoot.Spec.AccessRestrictions = []core.AccessRestrictionWithOptions{{AccessRestriction: core.AccessRestriction{Name: "foo"}}} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("forbidden to use a seed which doesn't support the access restrictions of the shoot")) @@ -1498,7 +1486,7 @@ var _ = Describe("validator", func() { shoot.Spec.AccessRestrictions = []core.AccessRestrictionWithOptions{{AccessRestriction: core.AccessRestriction{Name: "foo"}}} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -1507,7 +1495,7 @@ var _ = Describe("validator", func() { shoot.Spec.AccessRestrictions = []core.AccessRestrictionWithOptions{{AccessRestriction: core.AccessRestriction{Name: "foo"}}} attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("spec.accessRestrictions[0]: Unsupported value: \"foo\"")) @@ -1520,7 +1508,7 @@ var _ = Describe("validator", func() { shoot.Spec.AccessRestrictions = []core.AccessRestrictionWithOptions{{AccessRestriction: core.AccessRestriction{Name: "foo"}}} attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("spec.accessRestrictions[0]: Forbidden: access restriction \"foo\" is not supported by the seed")) @@ -1536,7 +1524,7 @@ var _ = Describe("validator", func() { shoot.Spec.AccessRestrictions = []core.AccessRestrictionWithOptions{{AccessRestriction: core.AccessRestriction{Name: "foo"}}} attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -1558,7 +1546,7 @@ var _ = Describe("validator", func() { shoot.Spec.SeedName = &newSeedName attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("forbidden to use a seed which doesn't support the access restrictions of the shoot")) @@ -1574,7 +1562,7 @@ var _ = Describe("validator", func() { shoot.Spec.AccessRestrictions = nil attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow deletion even if shoot access restrictions are (no longer) supported in this region", func() { @@ -1585,7 +1573,7 @@ var _ = Describe("validator", func() { oldShoot = shoot.DeepCopy() attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.UpdateOptions{}, false, nil) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow deletion even if shoot access restrictions are (no longer) supported by the seed", func() { @@ -1596,7 +1584,7 @@ var _ = Describe("validator", func() { oldShoot = shoot.DeepCopy() attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.UpdateOptions{}, false, nil) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) }) @@ -1619,7 +1607,7 @@ var _ = Describe("validator", func() { Context("taints and tolerations", func() { It("create should pass because the Seed specified in shoot manifest does not have any taints", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -1628,7 +1616,7 @@ var _ = Describe("validator", func() { seed.Spec.Taints = []gardencorev1beta1.SeedTaint{{Key: gardencorev1beta1.SeedTaintProtected}} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -1640,7 +1628,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Update(&versionedShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -1649,7 +1637,7 @@ var _ = Describe("validator", func() { seed.Spec.Taints = []gardencorev1beta1.SeedTaint{{Key: gardencorev1beta1.SeedTaintProtected}} attrs := admission.NewAttributesRecord(nil, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -1674,7 +1662,7 @@ var _ = Describe("validator", func() { It("should pass because seed allocatable capacity is not set", func() { attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -1693,7 +1681,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(otherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -1712,7 +1700,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(otherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("already has the maximum number of shoots scheduled on it"))) }) @@ -1731,7 +1719,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(otherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("already has the maximum number of shoots scheduled on it"))) }) @@ -1753,7 +1741,7 @@ var _ = Describe("validator", func() { oldShoot.Spec.SeedName = &seedName attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -1775,7 +1763,7 @@ var _ = Describe("validator", func() { It("should allow scheduling non-HA shoot", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow scheduling HA shoot with failure tolerance type 'node'", func() { @@ -1783,7 +1771,7 @@ var _ = Describe("validator", func() { shoot.Spec.ControlPlane = &core.ControlPlane{HighAvailability: &core.HighAvailability{FailureTolerance: core.FailureTolerance{Type: core.FailureToleranceTypeNode}}} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should reject scheduling HA shoot with failure tolerance type 'zone'", func() { @@ -1791,7 +1779,7 @@ var _ = Describe("validator", func() { shoot.Spec.ControlPlane = &core.ControlPlane{HighAvailability: &core.HighAvailability{FailureTolerance: core.FailureTolerance{Type: core.FailureToleranceTypeZone}}} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(BeForbiddenError()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(BeForbiddenError()) }) }) @@ -1802,7 +1790,7 @@ var _ = Describe("validator", func() { It("should allow scheduling non-HA shoot", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow scheduling HA shoot with failure tolerance type 'node'", func() { @@ -1810,7 +1798,7 @@ var _ = Describe("validator", func() { shoot.Spec.ControlPlane = &core.ControlPlane{HighAvailability: &core.HighAvailability{FailureTolerance: core.FailureTolerance{Type: core.FailureToleranceTypeNode}}} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow scheduling HA shoot with failure tolerance type 'zone'", func() { @@ -1818,7 +1806,7 @@ var _ = Describe("validator", func() { shoot.Spec.ControlPlane = &core.ControlPlane{HighAvailability: &core.HighAvailability{FailureTolerance: core.FailureTolerance{Type: core.FailureToleranceTypeZone}}} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) }) }) @@ -1840,7 +1828,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("label selector conversion failed")) @@ -1861,7 +1849,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -1881,7 +1869,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot schedule shoot '%s' on seed '%s' because the cloud profile seed selector is not matching the labels of the seed", shoot.Name, seed.Name)) @@ -1902,7 +1890,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -1922,7 +1910,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot schedule shoot '%s' on seed '%s' because none of the provider types in the cloud profile seed selector is matching the provider type of the seed", shoot.Name, seed.Name)) @@ -1943,7 +1931,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -1963,7 +1951,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot schedule shoot '%s' on seed '%s' because the cloud profile seed selector is not matching the labels of the seed", shoot.Name, seed.Name)) @@ -1984,7 +1972,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -2004,7 +1992,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot schedule shoot '%s' on seed '%s' because none of the provider types in the cloud profile seed selector is matching the provider type of the seed", shoot.Name, seed.Name)) @@ -2032,7 +2020,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -2052,7 +2040,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2082,7 +2070,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2113,7 +2101,7 @@ var _ = Describe("validator", func() { Expect(kubeInformerFactory.Core().V1().Secrets().Informer().GetStore().Add(&secret)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2145,7 +2133,7 @@ var _ = Describe("validator", func() { Expect(kubeInformerFactory.Core().V1().Secrets().Informer().GetStore().Add(&secret)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -2170,7 +2158,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(errorMatcher) }, @@ -2192,7 +2180,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admissionOperation, operationOptions, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }, @@ -2223,7 +2211,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -2237,7 +2225,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Add(&seed)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2253,7 +2241,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(nil, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -2268,7 +2256,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2284,7 +2272,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -2299,7 +2287,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("services is required, spec.networking.services"))) @@ -2316,7 +2304,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(ContainSubstring("services is required, spec.networking.services"))) @@ -2335,7 +2323,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(ContainSubstring("services is required"))) @@ -2353,7 +2341,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -2369,7 +2357,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2384,7 +2372,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(Not(HaveOccurred())) }) @@ -2400,7 +2388,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2415,7 +2403,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(Not(HaveOccurred())) }) @@ -2431,7 +2419,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2446,7 +2434,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(Not(HaveOccurred())) }) @@ -2462,7 +2450,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2477,7 +2465,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(Not(HaveOccurred())) }) @@ -2493,7 +2481,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2508,7 +2496,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(Not(HaveOccurred())) }) @@ -2523,7 +2511,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2538,7 +2526,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2553,7 +2541,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2572,7 +2560,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(anotherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) }) @@ -2592,7 +2580,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(anotherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("the domain is already used by another shoot or it is a subdomain of an already used domain"))) }) @@ -2612,7 +2600,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(anotherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow to update the shoot with deletion confirmation annotation although the specified domain is a subdomain of a domain already used by another shoot (case one)", func() { @@ -2632,7 +2620,7 @@ var _ = Describe("validator", func() { oldShoot := shoot.DeepCopy() metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "confirmation.gardener.cloud/deletion", "true") attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should reject because the specified domain is a subdomain of a domain already used by another shoot (case two)", func() { @@ -2649,7 +2637,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(anotherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("the domain is already used by another shoot or it is a subdomain of an already used domain"))) }) @@ -2668,7 +2656,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(anotherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow to update the shoot with deletion confirmation annotation although the specified domain is a subdomain of a domain already used by another shoot (case two)", func() { @@ -2687,7 +2675,7 @@ var _ = Describe("validator", func() { oldShoot := shoot.DeepCopy() metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "confirmation.gardener.cloud/deletion", "true") attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow because the specified domain is not a subdomain of a domain already used by another shoot", func() { @@ -2705,7 +2693,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(anotherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -2749,7 +2737,7 @@ var _ = Describe("validator", func() { shoot.Spec.Kubernetes.Version = "1.2.3" attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("Unsupported value: \"1.2.3\""))) }) @@ -2758,7 +2746,7 @@ var _ = Describe("validator", func() { shoot.Spec.Kubernetes.Version = expiredVersion.Version attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("spec.kubernetes.version: Unsupported value: %q", expiredVersion.Version))) }) @@ -2768,7 +2756,7 @@ var _ = Describe("validator", func() { shoot.Spec.Kubernetes.Version = expiredVersion.Version attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -2777,7 +2765,7 @@ var _ = Describe("validator", func() { shoot.Spec.Kubernetes.Version = expiredVersion.Version attrs := admission.NewAttributesRecord(nil, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -2787,7 +2775,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: ptr.To("1.26.6")} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) Expect(shoot.Spec.Kubernetes.Version).To(Equal(highestSupportedVersion.Version)) @@ -2799,7 +2787,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: ptr.To(highestSupportedVersion.Version)} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) Expect(shoot.Spec.Kubernetes.Version).To(Equal(highestSupportedVersion.Version)) @@ -2811,7 +2799,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: &expiredVersion.Version} attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("spec.provider.workers[0].kubernetes.version: Unsupported value: %q", expiredVersion.Version))) }) @@ -2823,7 +2811,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -2856,7 +2844,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) Expect(err).To(BeForbiddenError()) @@ -2868,7 +2856,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: &expiredVersion.Version} attrs := admission.NewAttributesRecord(nil, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -2906,7 +2894,7 @@ var _ = Describe("validator", func() { shoot.Spec.Kubernetes.Version = expiredVersion.Version attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(MatchError(And(ContainSubstring("Unsupported value"), ContainSubstring("1.26.8")))) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(MatchError(And(ContainSubstring("Unsupported value"), ContainSubstring("1.26.8")))) }) It("should allow to create a cluster with an extended Kubernetes version", func() { @@ -2917,7 +2905,7 @@ var _ = Describe("validator", func() { shoot.Spec.Kubernetes.Version = expiredVersion.Version attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) }) @@ -2990,7 +2978,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -3000,7 +2988,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -3014,7 +3002,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved CPU (kubeReserved + systemReserved) cannot be more than the Node's CPU capacity")) @@ -3028,7 +3016,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -3043,7 +3031,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved CPU (kubeReserved + systemReserved) cannot be more than the Node's CPU capacity")) @@ -3058,7 +3046,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved CPU (kubeReserved + systemReserved) cannot be more than the Node's CPU capacity")) @@ -3073,7 +3061,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved CPU (kubeReserved + systemReserved) cannot be more than the Node's CPU capacity")) @@ -3086,7 +3074,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved CPU (kubeReserved + systemReserved) cannot be more than the Node's CPU capacity")) @@ -3101,7 +3089,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved memory (kubeReserved + systemReserved) cannot be more than the Node's memory capacity")) @@ -3115,7 +3103,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -3130,7 +3118,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved memory (kubeReserved + systemReserved) cannot be more than the Node's memory capacity")) @@ -3145,7 +3133,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved memory (kubeReserved + systemReserved) cannot be more than the Node's memory capacity")) @@ -3160,7 +3148,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved memory (kubeReserved + systemReserved) cannot be more than the Node's memory capacity")) @@ -3173,7 +3161,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved memory (kubeReserved + systemReserved) cannot be more than the Node's memory capacity")) @@ -3187,7 +3175,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved CPU (kubeReserved + systemReserved) cannot be more than the Node's CPU capacity")) @@ -3201,7 +3189,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers = append(shoot.Spec.Provider.Workers, worker) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("total reserved memory (kubeReserved + systemReserved) cannot be more than the Node's memory capacity")) @@ -3228,7 +3216,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Machine.Architecture = ptr.To("foo") attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError( @@ -3241,7 +3229,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Machine.Image.Version = "1.2.0" attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError( @@ -3547,7 +3535,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine image version 'some-machine-image:1.1.1' is expired"))) @@ -3605,7 +3593,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine image version '%s' does not support CPU architecture %q, supported machine image versions are: [%s]", fmt.Sprintf("%s:%s", validMachineImageName, nonExpiredVersion), "amd64", fmt.Sprintf("%s:%s", validMachineImageName, latestNonExpiredVersionThatSupportsCapabilities)))) @@ -3665,7 +3653,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine image version '%s' does not support CPU architecture %q, is expired, supported machine image versions are: [%s]", fmt.Sprintf("%s:%s", validMachineImageName, expiredVersion), "amd64", fmt.Sprintf("%s:%s", validMachineImageName, latestNonExpiredVersionThatSupportsCapabilities)))) @@ -3753,7 +3741,7 @@ var _ = Describe("validator", func() { } } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine image version '%s' does not support in-place updates, supported machine image versions are: [%s]", fmt.Sprintf("%s:%s", validMachineImageName, latestNonExpiredVersion), fmt.Sprintf("%s:%s", validMachineImageName, nonExpiredVersion)))) @@ -3844,7 +3832,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine image version '%s' does not support CPU architecture %q, is expired, does not support in-place updates, supported machine image versions are: [%s %s]", fmt.Sprintf("%s:%s", validMachineImageName, expiredVersion), "amd64", fmt.Sprintf("%s:%s", validMachineImageName, nonExpiredVersion), fmt.Sprintf("%s:%s", validMachineImageName, latestNonExpiredVersionThatSupportsCapabilities)))) @@ -3899,7 +3887,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("machine image 'constraint-image-name@1.2.3' does not support kubelet version '1.26.0', supported kubelet versions by this machine image version: '< 1.26'")) @@ -3959,7 +3947,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("machine image 'constraint-image-name@1.2.3' does not support kubelet version '1.25.0', supported kubelet versions by this machine image version: '>= 1.26'")) @@ -3972,7 +3960,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("Unsupported value: \"1.2.baz\": supported values:"))) }) @@ -3984,7 +3972,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(MatchError(ContainSubstring("machine image version 'some-machine-image:%s' is expired", expiredVersion))) }) @@ -4038,7 +4026,7 @@ var _ = Describe("validator", func() { }) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -4092,7 +4080,7 @@ var _ = Describe("validator", func() { }) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("machine image 'cr-image-name@1.2.3' does not support CRI 'unsupported-cri', supported values: [containerd]")) @@ -4144,7 +4132,7 @@ var _ = Describe("validator", func() { }) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("machine image 'cr-image-name@1.2.3' does not support container runtime 'unsupported-cr-1', supported values: [supported-cr-1 supported-cr-2")) @@ -4198,7 +4186,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) }) @@ -4241,7 +4229,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(HaveOccurred()) }) @@ -4280,7 +4268,7 @@ var _ = Describe("validator", func() { newShoot.Spec.Kubernetes.Version = "1.26.0" attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("machine image 'constraint-image-name@1.2.3' does not support kubelet version '1.26.0', supported kubelet versions by this machine image version: '< 1.26'")) @@ -4323,7 +4311,7 @@ var _ = Describe("validator", func() { newShoot.Spec.Provider.Workers[0].Kubernetes.Version = ptr.To("1.25.0") attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("machine image 'constraint-image-name@1.2.3' does not support kubelet version '1.25.0', supported kubelet versions by this machine image version: '>= 1.26'")) @@ -4376,7 +4364,7 @@ var _ = Describe("validator", func() { newShoot.Spec.Provider.Workers = append(newShoot.Spec.Provider.Workers, newWorker) attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("machine image version 'constraint-image-name:1.2.4' is expired")) @@ -4435,7 +4423,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("machine image version 'constraint-image-name:1.2.3' is expired")) @@ -4493,7 +4481,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -4611,7 +4599,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine image version '%s' cannot be in-place updated from the current version, supported machine image versions are: [%s]", fmt.Sprintf("%s:%s", "constraint-image-name", "1.2.5"), fmt.Sprintf("%s:%s", "constraint-image-name", "1.2.4")))) @@ -4738,7 +4726,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine image version '%s' cannot be in-place updated from the current version, supported machine image versions are: [%s]", fmt.Sprintf("%s:%s", "constraint-image-name", "1.2.5"), fmt.Sprintf("%s:%s %s:%s", "constraint-image-name", "1.2.3", "constraint-image-name", "1.2.4")))) @@ -4837,7 +4825,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(newShoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) Expect(newShoot.Spec.Provider.Workers[0].Machine.Image).To(Equal(&core.ShootMachineImage{Name: "constraint-image-name", Version: "1.2.4"})) @@ -4858,7 +4846,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(nil, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -4922,7 +4910,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -4965,7 +4953,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) @@ -5003,7 +4991,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Machine.Type = "machine-type-1" attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -5012,7 +5000,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Machine.Type = "machine-type-old" attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine type %q is unusable", shoot.Spec.Provider.Workers[0].Machine.Type))) @@ -5022,7 +5010,7 @@ var _ = Describe("validator", func() { shoot.Spec.Provider.Workers[0].Machine.Type = "not-present-in-cloudprofile" attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("Unsupported value: %q: supported values: %q, %q", "not-present-in-cloudprofile", "machine-type-1", "machine-type-2"))) @@ -5058,7 +5046,7 @@ var _ = Describe("validator", func() { ) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine type %q is unavailable in at least one zone", unavailableMachine))) @@ -5121,7 +5109,7 @@ var _ = Describe("validator", func() { ) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("machine type %q is unusable, is unavailable in at least one zone, does not support CPU architecture %q, supported types are [%s]", "machine-type-1", architecture, "machine-type-2"))) @@ -5159,7 +5147,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("Unsupported value: %q", notAllowed))) @@ -5211,7 +5199,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("volume type %q is unavailable in at least one zone, supported types are [%s]", unavailableVolume, volumeType2))) @@ -5263,7 +5251,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("volume type %q is unusable, is unavailable in at least one zone, supported types are [%s]", unavailableVolume, volumeType2))) @@ -5301,7 +5289,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(ContainSubstring("volume type %q is unusable, supported types are [%s]", volumeType, volumeType2))) @@ -5313,7 +5301,7 @@ var _ = Describe("validator", func() { oldShoot.Spec.Provider.Workers[0].Volume.VolumeSize = "20Gi" attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -5371,7 +5359,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("spec.provider.workers[0].volume.size")) @@ -5459,7 +5447,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("spec.provider.infrastructureConfig: Invalid value: \"azure.provider.extensions.gardener.cloud/__internal, Kind=InfrastructureConfig\": must not use apiVersion 'internal'")) @@ -5532,7 +5520,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -5553,7 +5541,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -5572,7 +5560,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -5591,7 +5579,7 @@ var _ = Describe("validator", func() { Expect(securityInformerFactory.Security().V1alpha1().CredentialsBindings().Informer().GetStore().Add(&credentialsBinding)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).ToNot(HaveOccurred()) }) @@ -5645,7 +5633,7 @@ var _ = Describe("validator", func() { shoot.Status.SeedName = newSeedName attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(matcher) }, @@ -5665,7 +5653,7 @@ var _ = Describe("validator", func() { oldShoot.Spec.Kubernetes.Version = "1.6.3" attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -5678,7 +5666,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(matcher) }, @@ -5699,7 +5687,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Seeds().Informer().GetStore().Update(oldSeed)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(matcher) }, @@ -5736,7 +5724,7 @@ var _ = Describe("validator", func() { It("should allow update of binding when shoot.spec.seedName is nil", func() { oldShoot.Spec.SeedName = nil attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -5744,7 +5732,7 @@ var _ = Describe("validator", func() { It("should reject update of binding if the non-nil seedName is set to nil", func() { shoot.Spec.SeedName = nil attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("spec.seedName is already set to 'seed' and cannot be changed to 'nil'")) @@ -5754,7 +5742,7 @@ var _ = Describe("validator", func() { shoot.Spec.SeedName = ptr.To(newSeed.Name) attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -5763,7 +5751,7 @@ var _ = Describe("validator", func() { shoot.Spec.SeedName = ptr.To(newSeed.Name + " other") attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("Internal error occurred: could not find referenced seed")) @@ -5774,7 +5762,7 @@ var _ = Describe("validator", func() { shoot.Spec.Hibernation = &core.Hibernation{Enabled: ptr.To(true)} attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("only spec.seedName can be changed using the binding subresource when the shoot is being rescheduled to a new seed")) @@ -5788,7 +5776,7 @@ var _ = Describe("validator", func() { seed.DeletionTimestamp = &now attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot schedule shoot 'shoot' on seed 'seed' that is already marked for deletion")) @@ -5805,7 +5793,7 @@ var _ = Describe("validator", func() { newSeed.DeletionTimestamp = &now attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cannot schedule shoot '%s' on seed '%s' that is already marked for deletion", shoot.Name, newSeedName)) @@ -5815,7 +5803,7 @@ var _ = Describe("validator", func() { newSeed.Spec.Backup = nil attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("backup is not configured for seed %q", newSeedName))) @@ -5825,7 +5813,7 @@ var _ = Describe("validator", func() { seed.Spec.Backup = nil attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("backup is not configured for old seed %q", seedName))) @@ -5836,7 +5824,7 @@ var _ = Describe("validator", func() { newSeed.Spec.Provider.Type = "aws" attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -5851,7 +5839,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -5890,7 +5878,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -5930,7 +5918,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring(`cannot reschedule shoot "shoot" to seed "new-seed" because the shoot uses the default domain "default.example.com" which is not supported by the new seed (supported domains: [other.example.com]`)) @@ -5962,7 +5950,7 @@ var _ = Describe("validator", func() { newSeed.Spec.DNS.Defaults = []gardencorev1beta1.SeedDNSProviderConfig{} attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring(`cannot reschedule shoot "shoot" to seed "new-seed" because the shoot uses the default domain "default.example.com" which is not supported by the new seed (supported domains: []`)) @@ -6011,7 +5999,7 @@ var _ = Describe("validator", func() { } attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -6027,7 +6015,7 @@ var _ = Describe("validator", func() { It("update of binding should succeed because the Seed specified in the binding does not have any taints", func() { attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -6036,7 +6024,7 @@ var _ = Describe("validator", func() { newSeed.Spec.Taints = []gardencorev1beta1.SeedTaint{{Key: gardencorev1beta1.SeedTaintProtected}} attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("forbidden to use a seed whose taints are not tolerated by the shoot")) @@ -6047,7 +6035,7 @@ var _ = Describe("validator", func() { newSeed.Spec.Taints = []gardencorev1beta1.SeedTaint{{Key: gardencorev1beta1.SeedTaintProtected}} attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("forbidden to use a seed whose taints are not tolerated by the shoot")) @@ -6059,7 +6047,7 @@ var _ = Describe("validator", func() { oldShoot.Spec.Tolerations = []core.Toleration{{Key: "foo", Value: ptr.To("bar")}} attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -6079,7 +6067,7 @@ var _ = Describe("validator", func() { It("update of binding should pass because seed allocatable capacity is not set", func() { attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -6098,7 +6086,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(otherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).NotTo(HaveOccurred()) }) @@ -6117,7 +6105,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(otherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(MatchError(ContainSubstring("already has the maximum number of shoots scheduled on it"))) }) @@ -6136,7 +6124,7 @@ var _ = Describe("validator", func() { Expect(coreInformerFactory.Core().V1beta1().Shoots().Informer().GetStore().Add(otherShoot)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, &oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "binding", admission.Update, &metav1.UpdateOptions{}, false, nil) - err := admissionHandler.Admit(context.TODO(), attrs, nil) + err := admissionHandler.Validate(context.TODO(), attrs, nil) Expect(err).To(MatchError(ContainSubstring("already has the maximum number of shoots scheduled on it"))) }) @@ -6170,7 +6158,7 @@ var _ = Describe("validator", func() { Expect(shootStore.Add(&shoot)).NotTo(HaveOccurred()) attrs := admission.NewAttributesRecord(nil, shootBase.DeepCopyObject(), core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, nil) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(matcher) }, @@ -6216,7 +6204,7 @@ var _ = Describe("validator", func() { It("should reject creating a shoot if managed service account issuer is not configured", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("cannot enable managed service account issuer as it is not supported in this Gardener installation")) @@ -6225,7 +6213,7 @@ var _ = Describe("validator", func() { It("should reject updating a shoot if managed service account issuer is not configured but old shoot has been annotated", func() { shoot.Annotations = nil attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeInternalServerError()) Expect(err.Error()).To(ContainSubstring("old shoot object has managed service account issuer enabled, but Gardener configuration is missing")) @@ -6235,7 +6223,7 @@ var _ = Describe("validator", func() { shoot.Annotations = nil Expect(kubeInformerFactory.Core().V1().Secrets().Informer().GetStore().Add(issuerSecret)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, oldShoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.UpdateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("once enabled managed service account issuer cannot be disabled")) @@ -6249,7 +6237,7 @@ var _ = Describe("validator", func() { } Expect(kubeInformerFactory.Core().V1().Secrets().Informer().GetStore().Add(issuerSecret)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err.Error()).To(ContainSubstring("managed service account issuer cannot be enabled when .kubernetes.kubeAPIServer.serviceAccountConfig.issuer is set")) @@ -6259,7 +6247,7 @@ var _ = Describe("validator", func() { Expect(kubeInformerFactory.Core().V1().Secrets().Informer().GetStore().Add(issuerSecret)).To(Succeed()) attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) }) @@ -6281,7 +6269,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow updating shoots with deletionTimestamp independent of limits", func() { @@ -6289,13 +6277,13 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.DeleteOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow deleting shoots independent of limits", func() { attrs := admission.NewAttributesRecord(nil, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) Context("maxNodesTotal", func() { @@ -6315,7 +6303,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should forbid shoots with individual maximum over the limit", func() { @@ -6325,7 +6313,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(And( ContainSubstring("spec.provider.workers[0].maximum"), @@ -6344,7 +6332,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(And( ContainSubstring("spec.provider.workers"), @@ -6360,7 +6348,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, nil, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, userInfo) - err := admissionHandler.Admit(ctx, attrs, nil) + err := admissionHandler.Validate(ctx, attrs, nil) Expect(err).To(BeForbiddenError()) Expect(err).To(MatchError(And( ContainSubstring("spec.provider.workers[0].maximum"), @@ -6378,7 +6366,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(&shoot, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Update, &metav1.DeleteOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) It("should allow deleting shoots over the limit", func() { @@ -6386,7 +6374,7 @@ var _ = Describe("validator", func() { attrs := admission.NewAttributesRecord(nil, &shoot, core.Kind("Shoot").WithVersion("version"), shoot.Namespace, shoot.Name, core.Resource("shoots").WithVersion("version"), "", admission.Delete, &metav1.DeleteOptions{}, false, userInfo) - Expect(admissionHandler.Admit(ctx, attrs, nil)).To(Succeed()) + Expect(admissionHandler.Validate(ctx, attrs, nil)).To(Succeed()) }) }) }) From 32492eb1e4f13b92bf067de2ce707bd519f8d884 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Tue, 18 Nov 2025 10:25:47 +0100 Subject: [PATCH 131/176] Update registry.k8s.io/dns/k8s-dns-node-cache Docker tag to v1.26.7 (#13474) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 320c1ca6f19..7da6ea4bb60 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -480,7 +480,7 @@ images: - name: node-local-dns sourceRepository: github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/nodelocaldns repository: registry.k8s.io/dns/k8s-dns-node-cache - tag: "1.26.5" + tag: "1.26.7" labels: - name: 'gardener.cloud/cve-categorisation' value: From 8018d917882e4399d85bde20eefc174a79b23273 Mon Sep 17 00:00:00 2001 From: Aleksandar Savchev <57963548+AleksandarSavchev@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:40:40 +0200 Subject: [PATCH 132/176] Extend the `gardener.cloud/operation` annotation to allow specifying multiple operations (#12717) * Add helper functions to get Shoot operations * Adapt Shoot validation for parallel operations * Update shoot strategy and admission validator * Adapth shoot reconcile * Adapt Shoot maintenance controller * Add helper funcs for garden operations * Adapt Garden validation for parallel operations * Adapt Garden reconciliation * Update documentation * Fix typo * Fix patchOperations * Fix removeOperation func * Add review suggestions * Add suggested changes * Fix typos * Fix typo * Add max size limits to operation * Clean duplicates * Fix typo * Adress review feedback * Bug fix * Address review comments * Improve skip on maintenance op check when hibernated * Remove no longer needed operations length max * Improve docu * Remove length operation check from Garden Validation * Simplify ValidateHibernation for Shoot resource * Fix early exit shoot validation * Add comments to ValidateHibernation * Modify early exit in ValidateHibernation * Simplify ValidateHibernation * Improve RemoveOperation func * Add suggested changes * Fix early exit and remove unneeded set --- .../shoot-operations/shoot_operations.md | 26 ++ docs/usage/shoot/shoot_maintenance.md | 9 + .../core/v1beta1/constants/types_constants.go | 2 + pkg/apis/core/v1beta1/helper/shoot.go | 19 ++ pkg/apis/core/v1beta1/helper/shoot_test.go | 95 +++++++ pkg/apis/core/validation/shoot.go | 145 ++++++---- pkg/apis/core/validation/shoot_test.go | 180 +++++++++++-- pkg/apis/operator/v1alpha1/helper/garden.go | 7 + .../operator/v1alpha1/helper/garden_test.go | 22 ++ .../operator/v1alpha1/validation/garden.go | 228 ++++++++-------- .../v1alpha1/validation/garden_test.go | 29 ++ pkg/apiserver/registry/core/shoot/strategy.go | 123 ++++++--- .../registry/core/shoot/strategy_test.go | 97 +++++-- .../shoot/maintenance/reconciler.go | 4 +- .../controller/shoot/shoot/reconciler.go | 253 +++++++++--------- pkg/operator/controller/garden/garden/add.go | 9 +- .../controller/garden/garden/add_test.go | 25 ++ .../controller/garden/garden/reconciler.go | 142 ++++++---- pkg/utils/miscellaneous.go | 14 + pkg/utils/miscellaneous_test.go | 10 + plugin/pkg/shoot/mutator/admission.go | 2 +- 21 files changed, 1013 insertions(+), 428 deletions(-) diff --git a/docs/usage/shoot-operations/shoot_operations.md b/docs/usage/shoot-operations/shoot_operations.md index 0acde586186..69df60bfbf9 100644 --- a/docs/usage/shoot-operations/shoot_operations.md +++ b/docs/usage/shoot-operations/shoot_operations.md @@ -66,6 +66,32 @@ It will also be removed even if the restart of one or more services failed. > ℹ️ In the example mentioned above, you could additionally verify when/whether the kubelet restarted by using `kubectl describe node ` and looking for such a `Starting kubelet` event. +## Running Operations In Parallel + +Specific `Shoot` operations can run in parallel by using `;` as separator. + +``` bash +kubectl -n garden- annotate shoot gardener.cloud/operation="rotate-ssh-keypair;rotate-ca-start" +``` + +List of operations that can run in parallel: +- `reconcile` +- `rotate-ca-start` +- `rotate-ca-start-without-workers-rollout` +- `rotate-ca-complete` +- `rotate-observability-credentials` +- `rotate-ssh-keypair` +- `rotate-rollout-workers` +- `rotate-credentials-start` +- `rotate-credentials-start-without-workers-rollout` +- `rotate-credentials-complete` +- `rotate-etcd-encryption-key` +- `rotate-etcd-encryption-key-start` +- `rotate-etcd-encryption-key-complete` +- `rotate-serviceaccount-key-start` +- `rotate-serviceaccount-key-start-without-workers-rollout` +- `rotate-serviceaccount-key-complete` + ## Force Deletion When a Shoot fails to be deleted normally, users can force-delete the Shoot if it meets the following conditions: diff --git a/docs/usage/shoot/shoot_maintenance.md b/docs/usage/shoot/shoot_maintenance.md index d353c5f9d60..e7fbbe86b3e 100644 --- a/docs/usage/shoot/shoot_maintenance.md +++ b/docs/usage/shoot/shoot_maintenance.md @@ -115,6 +115,15 @@ maintenance.gardener.cloud/operation= This will execute the specified `` during the next maintenance reconciliation. Note that Gardener will remove this annotation after it has been performed in the maintenance reconciliation. + +Specific `Shoot` operations can run in parallel with + +``` +maintenance.gardener.cloud/operation=;;..; +``` + +A list of operations that can run in parallel can be found in the [Shoot Operations documentation](../shoot-operations/shoot_operations.md#running-operations-in-parallel). + > ⚠️ This is skipped when the `Shoot`'s `.status.lastOperation.state=Failed`. Make sure to [retry](../shoot-operations/shoot_operations.md#retry-failed-reconciliation) your shoot reconciliation beforehand. ## Special Operations During Maintenance diff --git a/pkg/apis/core/v1beta1/constants/types_constants.go b/pkg/apis/core/v1beta1/constants/types_constants.go index f5af73e1e92..9965a609bc7 100644 --- a/pkg/apis/core/v1beta1/constants/types_constants.go +++ b/pkg/apis/core/v1beta1/constants/types_constants.go @@ -209,6 +209,8 @@ const ( GardenCreatedBy = "gardener.cloud/created-by" // GardenerOperation is a constant for an annotation on a resource that describes a desired operation. GardenerOperation = "gardener.cloud/operation" + // GardenerOperationsSeparator is the separator used to separate parallel operations in the GardenerOperation annotation. + GardenerOperationsSeparator = ";" // GardenerMaintenanceOperation is a constant for an annotation on a Shoot that describes a desired operation which // will be performed during maintenance. GardenerMaintenanceOperation = "maintenance.gardener.cloud/operation" diff --git a/pkg/apis/core/v1beta1/helper/shoot.go b/pkg/apis/core/v1beta1/helper/shoot.go index 7f23df97ede..0f6b7b05848 100644 --- a/pkg/apis/core/v1beta1/helper/shoot.go +++ b/pkg/apis/core/v1beta1/helper/shoot.go @@ -18,6 +18,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/utils" versionutils "github.com/gardener/gardener/pkg/utils/version" ) @@ -757,3 +758,21 @@ func GetShootEncryptedResourcesInStatus(shootStatus gardencorev1beta1.ShootStatu return nil } + +// GetShootGardenerOperations returns the Shoot's gardener operations specified in the operation annotation. +func GetShootGardenerOperations(annotations map[string]string) []string { + return utils.SplitAndTrimString(annotations[v1beta1constants.GardenerOperation], v1beta1constants.GardenerOperationsSeparator) +} + +// GetShootMaintenanceOperations returns the Shoot's maintenance operations specified in the operation annotation. +func GetShootMaintenanceOperations(annotations map[string]string) []string { + return utils.SplitAndTrimString(annotations[v1beta1constants.GardenerMaintenanceOperation], v1beta1constants.GardenerOperationsSeparator) +} + +// RemoveOperation removes listed operations from the operations slice and returns a new slice that does not contain these operations. +// Note that the input operations slice is modified. +func RemoveOperation(operations []string, operationsToRemove ...string) []string { + return slices.DeleteFunc(operations, func(operation string) bool { + return slices.Contains(operationsToRemove, operation) + }) +} diff --git a/pkg/apis/core/v1beta1/helper/shoot_test.go b/pkg/apis/core/v1beta1/helper/shoot_test.go index aa2ed3772df..b19b658b3fc 100644 --- a/pkg/apis/core/v1beta1/helper/shoot_test.go +++ b/pkg/apis/core/v1beta1/helper/shoot_test.go @@ -1638,4 +1638,99 @@ var _ = Describe("Helper", func() { Entry("without resources", gardencorev1beta1.ShootStatus{Credentials: &gardencorev1beta1.ShootCredentials{}}, nil), Entry("with resources", gardencorev1beta1.ShootStatus{Credentials: &gardencorev1beta1.ShootCredentials{EncryptionAtRest: &gardencorev1beta1.EncryptionAtRest{Resources: []string{"configmaps", "shoots.core.gardener.cloud"}}}}, []string{"configmaps", "shoots.core.gardener.cloud"}), ) + + DescribeTable("#GetShootGardenerOperations", + func(annotations map[string]string, expectedResult []string) { + Expect(GetShootGardenerOperations(annotations)).To(Equal(expectedResult)) + }, + Entry("annotations are empty", nil, nil), + Entry("gardener.cloud/operation annotation is not set", map[string]string{ + "maintenance.gardener.cloud/operation": "foo", + }, nil), + Entry("gardener.cloud/operation annotation is empty", map[string]string{ + "gardener.cloud/operation": "", + }, nil), + Entry("gardener.cloud/operation has single operation", map[string]string{ + "gardener.cloud/operation": "reconcile", + }, []string{"reconcile"}), + Entry("gardener.cloud/operation has multiple operations", map[string]string{ + "gardener.cloud/operation": "reconcile;rotate-credentials-start;rotate-ssh-keypair", + }, []string{"reconcile", "rotate-credentials-start", "rotate-ssh-keypair"}), + Entry("gardener.cloud/operation has whitespaces", map[string]string{ + "gardener.cloud/operation": "reconcile ;rotate-credentials-start ; rotate-ssh-keypair; ", + }, []string{"reconcile", "rotate-credentials-start", "rotate-ssh-keypair", ""}), + ) + + DescribeTable("#GetShootMaintenanceOperations", + func(annotations map[string]string, expectedResult []string) { + Expect(GetShootMaintenanceOperations(annotations)).To(Equal(expectedResult)) + }, + Entry("annotations are empty", nil, nil), + Entry("maintenance.gardener.cloud/operation annotation is not set", map[string]string{ + "gardener.cloud/operation": "foo", + }, nil), + Entry("maintenance.gardener.cloud/operation annotation is empty", map[string]string{ + "maintenance.gardener.cloud/operation": "", + }, nil), + Entry("maintenance.gardener.cloud/operation has single operation", map[string]string{ + "maintenance.gardener.cloud/operation": "reconcile", + }, []string{"reconcile"}), + Entry("maintenance.gardener.cloud/operation has multiple operations", map[string]string{ + "maintenance.gardener.cloud/operation": "reconcile;rotate-credentials-start;rotate-ssh-keypair", + }, []string{"reconcile", "rotate-credentials-start", "rotate-ssh-keypair"}), + Entry("maintenance.gardener.cloud/operation has whitespaces", map[string]string{ + "maintenance.gardener.cloud/operation": "reconcile ;rotate-credentials-start ; rotate-ssh-keypair; ", + }, []string{"reconcile", "rotate-credentials-start", "rotate-ssh-keypair", ""}), + ) + + DescribeTable("#RemoveOperation", + func(operations, operationsToRemove, newOperations, expectedResult []string) { + result := RemoveOperation(operations, operationsToRemove...) + + Expect(operations).To(Equal(newOperations)) + Expect(result).To(Equal(expectedResult)) + }, + + Entry("remove single operation from list", + []string{"rotate-ssh-keypair", "rotate-ca-start", "rotate-observability-credentials"}, + []string{"rotate-ca-start"}, + []string{"rotate-ssh-keypair", "rotate-observability-credentials", ""}, + []string{"rotate-ssh-keypair", "rotate-observability-credentials"}), + + Entry("remove multiple operations from list", + []string{"rotate-ssh-keypair", "rotate-ca-start", "rotate-observability-credentials", "reconcile"}, + []string{"rotate-ca-start", "reconcile"}, + []string{"rotate-ssh-keypair", "rotate-observability-credentials", "", ""}, + []string{"rotate-ssh-keypair", "rotate-observability-credentials"}), + + Entry("remove operation that doesn't exist", + []string{"rotate-ssh-keypair", "rotate-ca-start"}, + []string{"non-existent-operation"}, + []string{"rotate-ssh-keypair", "rotate-ca-start"}, + []string{"rotate-ssh-keypair", "rotate-ca-start"}), + + Entry("remove from empty list", + []string{}, + []string{"rotate-ssh-keypair"}, + []string{}, + []string{}), + + Entry("remove all operations", + []string{"rotate-ssh-keypair", "rotate-ca-start"}, + []string{"rotate-ssh-keypair", "rotate-ca-start"}, + []string{"", ""}, + []string{}), + + Entry("remove no operations (empty remove list)", + []string{"rotate-ssh-keypair", "rotate-ca-start"}, + []string{}, + []string{"rotate-ssh-keypair", "rotate-ca-start"}, + []string{"rotate-ssh-keypair", "rotate-ca-start"}), + + Entry("remove duplicate operation from list with duplicates", + []string{"rotate-ssh-keypair", "rotate-ca-start", "rotate-ssh-keypair"}, + []string{"rotate-ssh-keypair"}, + []string{"rotate-ca-start", "", ""}, + []string{"rotate-ca-start"}), + ) }) diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index 3f7572092b0..dd39587471d 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -36,6 +36,7 @@ import ( "github.com/gardener/gardener/pkg/apis/core/helper" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" securityv1alpha1 "github.com/gardener/gardener/pkg/apis/security/v1alpha1" "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/utils" @@ -98,6 +99,27 @@ var ( v1beta1constants.OperationRotateETCDEncryptionKey, v1beta1constants.OperationRotateETCDEncryptionKeyStart, ) + availableShootOperationsToRunInParallel = availableShootMaintenanceOperations + incompatibleShootOperations = map[string][]string{ + v1beta1constants.OperationRotateCredentialsStart: { + v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, + v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, + v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout, + }, + v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout: { + v1beta1constants.OperationRotateCAStart, + v1beta1constants.OperationRotateServiceAccountKeyStart, + }, + v1beta1constants.OperationRotateCAStart: { + v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, + }, + v1beta1constants.OperationRotateServiceAccountKeyStart: { + v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, + }, + v1beta1constants.OperationRotateETCDEncryptionKey: { + v1beta1constants.OperationRotateETCDEncryptionKeyStart, + }, + } availableShootPurposes = sets.New( string(core.ShootPurposeEvaluation), string(core.ShootPurposeTesting), @@ -158,7 +180,7 @@ func ValidateShoot(shoot *core.Shoot) field.ErrorList { allErrs = append(allErrs, apivalidation.ValidateObjectMeta(&shoot.ObjectMeta, true, apivalidation.NameIsDNSLabel, field.NewPath("metadata"))...) allErrs = append(allErrs, validateNameConsecutiveHyphens(shoot.Name, field.NewPath("metadata", "name"))...) - allErrs = append(allErrs, validateShootOperation(shoot.Annotations[v1beta1constants.GardenerOperation], shoot.Annotations[v1beta1constants.GardenerMaintenanceOperation], shoot, field.NewPath("metadata", "annotations"))...) + allErrs = append(allErrs, validateShootOperation(v1beta1helper.GetShootGardenerOperations(shoot.Annotations), v1beta1helper.GetShootMaintenanceOperations(shoot.Annotations), shoot, field.NewPath("metadata", "annotations"))...) allErrs = append(allErrs, ValidateShootSpec(shoot.ObjectMeta, &shoot.Spec, field.NewPath("spec"), false)...) allErrs = append(allErrs, ValidateShootHAConfig(shoot)...) @@ -2630,8 +2652,10 @@ func ValidateHibernation(annotations map[string]string, hibernation *core.Hibern return allErrs } - if maintenanceOp := annotations[v1beta1constants.GardenerMaintenanceOperation]; forbiddenShootOperationsWhenHibernated.Has(maintenanceOp) && ptr.Deref(hibernation.Enabled, false) { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("enabled"), fmt.Sprintf("shoot cannot be hibernated when %s=%s annotation is set", v1beta1constants.GardenerMaintenanceOperation, maintenanceOp))) + for _, op := range v1beta1helper.GetShootMaintenanceOperations(annotations) { + if forbiddenShootOperationsWhenHibernated.Has(op) && ptr.Deref(hibernation.Enabled, false) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("enabled"), fmt.Sprintf("shoot cannot be hibernated when %s annotation contains %s operation", v1beta1constants.GardenerMaintenanceOperation, op))) + } } allErrs = append(allErrs, ValidateHibernationSchedules(hibernation.Schedules, fldPath.Child("schedules"))...) @@ -2916,92 +2940,123 @@ func ValidateCoreDNSRewritingCommonSuffixes(commonSuffixes []string, fldPath *fi return allErrs } -func validateShootOperation(operation, maintenanceOperation string, shoot *core.Shoot, fldPath *field.Path) field.ErrorList { +func validateShootOperation(operations, maintenanceOperations []string, shoot *core.Shoot, fldPath *field.Path) field.ErrorList { var ( + fldPathOp = fldPath.Key(v1beta1constants.GardenerOperation) + fldPathMaintOp = fldPath.Key(v1beta1constants.GardenerMaintenanceOperation) allErrs = field.ErrorList{} encryptedResources = sets.New[schema.GroupResource]() k8sLess134, _ = versionutils.CheckVersionMeetsConstraint(shoot.Spec.Kubernetes.Version, "< 1.34") ) - if operation == "" && maintenanceOperation == "" { + if len(operations) == 0 && len(maintenanceOperations) == 0 { return allErrs } + for _, op := range operations { + if !availableShootOperations.Has(op) && !strings.HasPrefix(op, v1beta1constants.OperationRotateRolloutWorkers) && !strings.HasPrefix(op, v1beta1constants.OperationRolloutWorkers) { + return append(allErrs, field.NotSupported(fldPathOp, op, sets.List(availableShootOperations))) + } else if len(operations) > 1 && !availableShootOperationsToRunInParallel.Has(op) && !strings.HasPrefix(op, v1beta1constants.OperationRotateRolloutWorkers) && !strings.HasPrefix(op, v1beta1constants.OperationRolloutWorkers) { + return append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted to be run in parallel with other operations", op))) + } + } + + for _, op := range maintenanceOperations { + if !availableShootOperations.Has(op) && !strings.HasPrefix(op, v1beta1constants.OperationRotateRolloutWorkers) && !strings.HasPrefix(op, v1beta1constants.OperationRolloutWorkers) { + return append(allErrs, field.NotSupported(fldPathMaintOp, op, sets.List(availableShootOperations))) + } else if len(maintenanceOperations) > 1 && !availableShootOperationsToRunInParallel.Has(op) && !strings.HasPrefix(op, v1beta1constants.OperationRotateRolloutWorkers) && !strings.HasPrefix(op, v1beta1constants.OperationRolloutWorkers) { + return append(allErrs, field.Forbidden(fldPathMaintOp, fmt.Sprintf("operation '%s' is not permitted to be run in parallel with other operations", op))) + } + } + if shoot.Status.Credentials != nil && shoot.Status.Credentials.EncryptionAtRest != nil { for _, er := range shoot.Status.Credentials.EncryptionAtRest.Resources { encryptedResources.Insert(schema.ParseGroupResource(er)) } } - fldPathOp := fldPath.Key(v1beta1constants.GardenerOperation) - fldPathMaintOp := fldPath.Key(v1beta1constants.GardenerMaintenanceOperation) + var ( + operationsSet = sets.New(operations...) + maintenanceOperationsSet = sets.New(maintenanceOperations...) + ) - if operation == maintenanceOperation { - allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("annotations %s and %s must not be equal", fldPathOp, fldPathMaintOp))) + if operationsSet.HasAny(maintenanceOperations...) { + allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("annotations %s and %s must not have any equal operations", fldPathOp, fldPathMaintOp))) } - if operation != "" { - if forbiddenETCDEncryptionKeyShootOperationsWithK8s134.Has(operation) && !k8sLess134 { - allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("for Kubernetes versions >= 1.34, operation '%s' is no longer supported, please use 'rotate-etcd-encryption-key' instead, which performs a complete etcd encryption key rotation", operation))) + for _, op := range operations { + if forbiddenETCDEncryptionKeyShootOperationsWithK8s134.Has(op) && !k8sLess134 { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("for Kubernetes versions >= 1.34, operation '%s' is no longer supported, please use 'rotate-etcd-encryption-key' instead, which performs a complete etcd encryption key rotation", op))) } - if !availableShootOperations.Has(operation) && !strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers) && !strings.HasPrefix(operation, v1beta1constants.OperationRolloutWorkers) { - allErrs = append(allErrs, field.NotSupported(fldPathOp, operation, sets.List(availableShootOperations))) + if forbiddenOps, ok := incompatibleShootOperations[op]; ok && operationsSet.HasAny(forbiddenOps...) { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted to be run together with %s operations", op, strings.Join(forbiddenOps, ", ")))) } if helper.IsShootInHibernation(shoot) && - (forbiddenShootOperationsWhenHibernated.Has(operation) || strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers) || strings.HasPrefix(operation, v1beta1constants.OperationRolloutWorkers)) { - allErrs = append(allErrs, field.Forbidden(fldPathOp, "operation is not permitted when shoot is hibernated or is waking up")) + (forbiddenShootOperationsWhenHibernated.Has(op) || strings.HasPrefix(op, v1beta1constants.OperationRotateRolloutWorkers) || strings.HasPrefix(op, v1beta1constants.OperationRolloutWorkers)) { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted when shoot is hibernated or is waking up", op))) } if !encryptedResources.Equal(sets.New(getResourcesForEncryption(shoot.Spec.Kubernetes.KubeAPIServer)...)) && - forbiddenShootOperationsWhenEncryptionChangeIsRollingOut.Has(operation) { - allErrs = append(allErrs, field.Forbidden(fldPathOp, "operation is not permitted because a previous encryption configuration change is currently being rolled out")) + forbiddenShootOperationsWhenEncryptionChangeIsRollingOut.Has(op) { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted because a previous encryption configuration change is currently being rolled out", op))) } } - if maintenanceOperation != "" { - if forbiddenETCDEncryptionKeyShootOperationsWithK8s134.Has(maintenanceOperation) && !k8sLess134 { - allErrs = append(allErrs, field.Forbidden(fldPathMaintOp, fmt.Sprintf("for Kubernetes versions >= 1.34, operation '%s' is no longer supported, please use 'rotate-etcd-encryption-key' instead, which performs a complete etcd encryption key rotation", maintenanceOperation))) + for _, op := range maintenanceOperations { + if forbiddenETCDEncryptionKeyShootOperationsWithK8s134.Has(op) && !k8sLess134 { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("for Kubernetes versions >= 1.34, operation '%s' is no longer supported, please use 'rotate-etcd-encryption-key' instead, which performs a complete etcd encryption key rotation", op))) } - if !availableShootMaintenanceOperations.Has(maintenanceOperation) && !strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRotateRolloutWorkers) && !strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRolloutWorkers) { - allErrs = append(allErrs, field.NotSupported(fldPathMaintOp, maintenanceOperation, sets.List(availableShootMaintenanceOperations))) + if forbiddenOps, ok := incompatibleShootOperations[op]; ok && maintenanceOperationsSet.HasAny(forbiddenOps...) { + allErrs = append(allErrs, field.Forbidden(fldPathMaintOp, fmt.Sprintf("operation '%s' is not permitted to be run together with %s maintenance operations", op, strings.Join(forbiddenOps, ", ")))) } if helper.IsShootInHibernation(shoot) && - (forbiddenShootOperationsWhenHibernated.Has(maintenanceOperation) || strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRotateRolloutWorkers) || strings.HasPrefix(maintenanceOperation, v1beta1constants.OperationRolloutWorkers)) { - allErrs = append(allErrs, field.Forbidden(fldPathMaintOp, "operation is not permitted when shoot is hibernated or is waking up")) + (forbiddenShootOperationsWhenHibernated.Has(op) || strings.HasPrefix(op, v1beta1constants.OperationRotateRolloutWorkers) || strings.HasPrefix(op, v1beta1constants.OperationRolloutWorkers)) { + allErrs = append(allErrs, field.Forbidden(fldPathMaintOp, fmt.Sprintf("operation '%s' is not permitted when shoot is hibernated or is waking up", op))) } - if !encryptedResources.Equal(sets.New(getResourcesForEncryption(shoot.Spec.Kubernetes.KubeAPIServer)...)) && forbiddenShootOperationsWhenEncryptionChangeIsRollingOut.Has(maintenanceOperation) { - allErrs = append(allErrs, field.Forbidden(fldPathMaintOp, "operation is not permitted because a previous encryption configuration change is currently being rolled out")) + if !encryptedResources.Equal(sets.New(getResourcesForEncryption(shoot.Spec.Kubernetes.KubeAPIServer)...)) && forbiddenShootOperationsWhenEncryptionChangeIsRollingOut.Has(op) { + allErrs = append(allErrs, field.Forbidden(fldPathMaintOp, fmt.Sprintf("operation '%s' is not permitted because a previous encryption configuration change is currently being rolled out", op))) } } - switch maintenanceOperation { - case v1beta1constants.OperationRotateCredentialsStart, v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout: - if sets.New(v1beta1constants.OperationRotateCAStart, v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, v1beta1constants.OperationRotateServiceAccountKeyStart, v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, v1beta1constants.OperationRotateETCDEncryptionKey, v1beta1constants.OperationRotateETCDEncryptionKeyStart).Has(operation) { - allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted when maintenance operation is '%s'", operation, maintenanceOperation))) - } - case v1beta1constants.OperationRotateCredentialsComplete: - if sets.New(v1beta1constants.OperationRotateCAComplete, v1beta1constants.OperationRotateServiceAccountKeyComplete, v1beta1constants.OperationRotateETCDEncryptionKeyComplete).Has(operation) { - allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted when maintenance operation is '%s'", operation, maintenanceOperation))) + // Validate operation conflicts with maintenance operations + for _, op := range maintenanceOperations { + switch op { + case v1beta1constants.OperationRotateCredentialsStart, v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout: + if intersection := operationsSet.Intersection(sets.New(v1beta1constants.OperationRotateCAStart, v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, v1beta1constants.OperationRotateServiceAccountKeyStart, v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, v1beta1constants.OperationRotateETCDEncryptionKey, v1beta1constants.OperationRotateETCDEncryptionKeyStart)); intersection.Len() > 0 { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted when maintenance operation is '%s'", strings.Join(sets.List(intersection), ", "), op))) + } + case v1beta1constants.OperationRotateCredentialsComplete: + if intersection := operationsSet.Intersection(sets.New(v1beta1constants.OperationRotateCAComplete, v1beta1constants.OperationRotateServiceAccountKeyComplete, v1beta1constants.OperationRotateETCDEncryptionKeyComplete)); intersection.Len() > 0 { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted when maintenance operation is '%s'", strings.Join(sets.List(intersection), ", "), op))) + } } } - switch operation { - case v1beta1constants.OperationRotateCredentialsStart, v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout: - if sets.New(v1beta1constants.OperationRotateCAStart, v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, v1beta1constants.OperationRotateServiceAccountKeyStart, v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, v1beta1constants.OperationRotateETCDEncryptionKey, v1beta1constants.OperationRotateETCDEncryptionKeyStart).Has(maintenanceOperation) { - allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted when maintenance operation is '%s'", operation, maintenanceOperation))) - } - case v1beta1constants.OperationRotateCredentialsComplete: - if sets.New(v1beta1constants.OperationRotateCAComplete, v1beta1constants.OperationRotateServiceAccountKeyComplete, v1beta1constants.OperationRotateETCDEncryptionKeyComplete).Has(maintenanceOperation) { - allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted when maintenance operation is '%s'", operation, maintenanceOperation))) + for _, op := range operations { + switch op { + case v1beta1constants.OperationRotateCredentialsStart, v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout: + if intersection := maintenanceOperationsSet.Intersection(sets.New(v1beta1constants.OperationRotateCAStart, v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, v1beta1constants.OperationRotateServiceAccountKeyStart, v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, v1beta1constants.OperationRotateETCDEncryptionKey, v1beta1constants.OperationRotateETCDEncryptionKeyStart)); intersection.Len() > 0 { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted when maintenance operation is '%s'", op, strings.Join(sets.List(intersection), ", ")))) + } + case v1beta1constants.OperationRotateCredentialsComplete: + if intersection := maintenanceOperationsSet.Intersection(sets.New(v1beta1constants.OperationRotateCAComplete, v1beta1constants.OperationRotateServiceAccountKeyComplete, v1beta1constants.OperationRotateETCDEncryptionKeyComplete)); intersection.Len() > 0 { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted when maintenance operation is '%s'", op, strings.Join(sets.List(intersection), ", ")))) + } } } - allErrs = append(allErrs, validateShootOperationContext(operation, shoot, fldPathOp)...) + // Validate operation contexts for each operation + for _, op := range operations { + allErrs = append(allErrs, validateShootOperationContext(op, shoot, fldPathOp)...) + } + if shoot.DeletionTimestamp == nil { // Only validate maintenance operation context when shoot has no deletion timestamp. If it has such a timestamp, // any validation is pointless since there are no maintenance operations for shoots in deletion, so we basically // don't care. Without this, we could wrongly prevent metadata changes in case the annotation is still present // but the shoot is in deletion. - allErrs = append(allErrs, validateShootOperationContext(maintenanceOperation, shoot, fldPathMaintOp)...) + for _, op := range maintenanceOperations { + allErrs = append(allErrs, validateShootOperationContext(op, shoot, fldPathMaintOp)...) + } } return allErrs diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index f4dd32a7eaf..47c0c7e2fd2 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -29,6 +29,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" . "github.com/gardener/gardener/pkg/apis/core/validation" "github.com/gardener/gardener/pkg/features" + "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/test" . "github.com/gardener/gardener/pkg/utils/test/matchers" ) @@ -6245,6 +6246,15 @@ var _ = Describe("Shoot Validation Tests", func() { })))) }) + It("should return an error only on first invalid operation annotation", func() { + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "gardener.cloud/operation", "foo-bar;baz") + Expect(ValidateShoot(shoot)).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeNotSupported), + "Field": Equal("metadata.annotations[gardener.cloud/operation]"), + "BadValue": Equal("foo-bar"), + })))) + }) + It("should return an error if the maintenance operation annotation is invalid", func() { metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "maintenance.gardener.cloud/operation", "foo-bar") Expect(ValidateShoot(shoot)).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ @@ -6253,6 +6263,15 @@ var _ = Describe("Shoot Validation Tests", func() { })))) }) + It("should return an error only on first invalid maintenance operation annotation", func() { + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "maintenance.gardener.cloud/operation", "foo-bar;baz") + Expect(ValidateShoot(shoot)).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeNotSupported), + "Field": Equal("metadata.annotations[maintenance.gardener.cloud/operation]"), + "BadValue": Equal("foo-bar"), + })))) + }) + It("should return an error if maintenance annotation is not allowed in this context", func() { metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "maintenance.gardener.cloud/operation", "rotate-etcd-encryption-key-complete") shoot.Status = core.ShootStatus{ @@ -6314,7 +6333,7 @@ var _ = Describe("Shoot Validation Tests", func() { if sets.New(v1beta1constants.OperationRotateCredentialsComplete, v1beta1constants.OperationRotateCAComplete, v1beta1constants.OperationRotateServiceAccountKeyComplete, - v1beta1constants.OperationRotateETCDEncryptionKeyComplete).Has(maintenanceOpAnnotation) { + v1beta1constants.OperationRotateETCDEncryptionKeyComplete).HasAny(utils.SplitAndTrimString(maintenanceOpAnnotation, v1beta1constants.GardenerOperationsSeparator)...) { shoot.Status.Credentials = &core.ShootCredentials{ Rotation: &core.ShootCredentialsRotation{ CertificateAuthorities: &core.CARotation{ @@ -6346,15 +6365,18 @@ var _ = Describe("Shoot Validation Tests", func() { Entry("rotate-serviceaccount-key-start-without-workers-rollout", "rotate-credentials-start", "rotate-serviceaccount-key-start-without-workers-rollout", "operation 'rotate-serviceaccount-key-start-without-workers-rollout' is not permitted when maintenance operation is 'rotate-credentials-start'"), Entry("rotate-etcd-encryption-key-start", "rotate-credentials-start", "rotate-etcd-encryption-key-start", "operation 'rotate-etcd-encryption-key-start' is not permitted when maintenance operation is 'rotate-credentials-start'"), Entry("rotate-etcd-encryption-key", "rotate-credentials-start", "rotate-etcd-encryption-key", "operation 'rotate-etcd-encryption-key' is not permitted when maintenance operation is 'rotate-credentials-start'"), + Entry("rotate-ca-start & rotate-etcd-encryption-key", "rotate-credentials-start", "rotate-ca-start; rotate-etcd-encryption-key", "operation 'rotate-ca-start, rotate-etcd-encryption-key' is not permitted when maintenance operation is 'rotate-credentials-start'"), Entry("rotate-ca-complete", "rotate-credentials-complete", "rotate-ca-complete", "operation 'rotate-ca-complete' is not permitted when maintenance operation is 'rotate-credentials-complete'"), Entry("rotate-serviceaccount-key-complete", "rotate-credentials-complete", "rotate-serviceaccount-key-complete", "operation 'rotate-serviceaccount-key-complete' is not permitted when maintenance operation is 'rotate-credentials-complete'"), Entry("rotate-etcd-encryption-key-complete", "rotate-credentials-complete", "rotate-etcd-encryption-key-complete", "operation 'rotate-etcd-encryption-key-complete' is not permitted when maintenance operation is 'rotate-credentials-complete'"), + Entry("rotate-ca-complete & rotate-serviceaccount-key-complete", "rotate-credentials-complete", "rotate-ca-complete; rotate-serviceaccount-key-complete", "operation 'rotate-ca-complete, rotate-serviceaccount-key-complete' is not permitted when maintenance operation is 'rotate-credentials-complete'"), Entry("rotate-credentials-start", "rotate-ca-start", "rotate-credentials-start", "operation 'rotate-credentials-start' is not permitted when maintenance operation is 'rotate-ca-start'"), Entry("rotate-credentials-start", "rotate-serviceaccount-key-start", "rotate-credentials-start", "operation 'rotate-credentials-start' is not permitted when maintenance operation is 'rotate-serviceaccount-key-start'"), Entry("rotate-credentials-start", "rotate-etcd-encryption-key-start", "rotate-credentials-start", "operation 'rotate-credentials-start' is not permitted when maintenance operation is 'rotate-etcd-encryption-key-start'"), Entry("rotate-credentials-start", "rotate-etcd-encryption-key", "rotate-credentials-start", "operation 'rotate-credentials-start' is not permitted when maintenance operation is 'rotate-etcd-encryption-key'"), + Entry("rotate-credentials-start", "rotate-ca-start; rotate-etcd-encryption-key", "rotate-credentials-start", "operation 'rotate-credentials-start' is not permitted when maintenance operation is 'rotate-ca-start, rotate-etcd-encryption-key'"), Entry("rotate-credentials-start-without-workers-rollout", "rotate-ca-start", "rotate-credentials-start-without-workers-rollout", "operation 'rotate-credentials-start-without-workers-rollout' is not permitted when maintenance operation is 'rotate-ca-start'"), Entry("rotate-credentials-start-without-workers-rollout", "rotate-serviceaccount-key-start", "rotate-credentials-start-without-workers-rollout", "operation 'rotate-credentials-start-without-workers-rollout' is not permitted when maintenance operation is 'rotate-serviceaccount-key-start'"), Entry("rotate-credentials-start-without-workers-rollout", "rotate-etcd-encryption-key-start", "rotate-credentials-start-without-workers-rollout", "operation 'rotate-credentials-start-without-workers-rollout' is not permitted when maintenance operation is 'rotate-etcd-encryption-key-start'"), @@ -6363,17 +6385,18 @@ var _ = Describe("Shoot Validation Tests", func() { Entry("rotate-credentials-complete", "rotate-ca-complete", "rotate-credentials-complete", "operation 'rotate-credentials-complete' is not permitted when maintenance operation is 'rotate-ca-complete'"), Entry("rotate-credentials-complete", "rotate-serviceaccount-key-complete", "rotate-credentials-complete", "operation 'rotate-credentials-complete' is not permitted when maintenance operation is 'rotate-serviceaccount-key-complete'"), Entry("rotate-credentials-complete", "rotate-etcd-encryption-key-complete", "rotate-credentials-complete", "operation 'rotate-credentials-complete' is not permitted when maintenance operation is 'rotate-etcd-encryption-key-complete'"), + Entry("rotate-credentials-complete", "rotate-ca-complete; rotate-serviceaccount-key-complete", "rotate-credentials-complete", "operation 'rotate-credentials-complete' is not permitted when maintenance operation is 'rotate-ca-complete, rotate-serviceaccount-key-complete'"), ) DescribeTable("forbid certain rotation operations when shoot is hibernated", - func(operation string) { + func(operation, forbiddenOp string) { shoot.Spec.Hibernation = &core.Hibernation{Enabled: ptr.To(true)} metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "gardener.cloud/operation", operation) Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("metadata.annotations[gardener.cloud/operation]"), - "Detail": ContainSubstring("operation is not permitted when shoot is hibernated or is waking up"), + "Detail": ContainSubstring(fmt.Sprintf("operation '%s' is not permitted when shoot is hibernated or is waking up", forbiddenOp)), })))) delete(shoot.Annotations, "gardener.cloud/operation") @@ -6381,21 +6404,22 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("metadata.annotations[maintenance.gardener.cloud/operation]"), - "Detail": ContainSubstring("operation is not permitted when shoot is hibernated or is waking up"), + "Detail": ContainSubstring(fmt.Sprintf("operation '%s' is not permitted when shoot is hibernated or is waking up", forbiddenOp)), })))) delete(shoot.Annotations, "maintenance.gardener.cloud/operation") }, - Entry("rotate-credentials-start", "rotate-credentials-start"), - Entry("rotate-credentials-start-without-workers-rollout", "rotate-credentials-start-without-workers-rollout"), - Entry("rotate-credentials-complete", "rotate-credentials-complete"), - Entry("rotate-etcd-encryption-key", "rotate-etcd-encryption-key"), - Entry("rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key-start"), - Entry("rotate-etcd-encryption-key-complete", "rotate-etcd-encryption-key-complete"), - Entry("rotate-serviceaccount-key-start", "rotate-serviceaccount-key-start"), - Entry("rotate-serviceaccount-key-start-without-workers-rollout", "rotate-serviceaccount-key-start-without-workers-rollout"), - Entry("rotate-serviceaccount-key-complete", "rotate-serviceaccount-key-complete"), - Entry("rotate-rollout-workers", "rotate-rollout-workers=worker-name"), + Entry("rotate-credentials-start", "rotate-credentials-start", "rotate-credentials-start"), + Entry("rotate-credentials-start-without-workers-rollout", "rotate-credentials-start-without-workers-rollout", "rotate-credentials-start-without-workers-rollout"), + Entry("rotate-credentials-complete", "rotate-credentials-complete", "rotate-credentials-complete"), + Entry("rotate-etcd-encryption-key", "rotate-etcd-encryption-key", "rotate-etcd-encryption-key"), + Entry("rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key-start"), + Entry("rotate-etcd-encryption-key-complete", "rotate-etcd-encryption-key-complete", "rotate-etcd-encryption-key-complete"), + Entry("rotate-serviceaccount-key-start", "rotate-serviceaccount-key-start", "rotate-serviceaccount-key-start"), + Entry("rotate-serviceaccount-key-start-without-workers-rollout", "rotate-serviceaccount-key-start-without-workers-rollout", "rotate-serviceaccount-key-start-without-workers-rollout"), + Entry("rotate-serviceaccount-key-complete", "rotate-serviceaccount-key-complete", "rotate-serviceaccount-key-complete"), + Entry("rotate-rollout-workers", "rotate-rollout-workers=worker-name", "rotate-rollout-workers=worker-name"), + Entry("multiple-operations", "rotate-ssh-keypair;rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key-start"), ) Context("trigger workers rollout", func() { @@ -6433,7 +6457,7 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("metadata.annotations[gardener.cloud/operation]"), - "Detail": Equal("operation is not permitted when shoot is hibernated or is waking up"), + "Detail": Equal("operation 'rotate-rollout-workers=worker-name' is not permitted when shoot is hibernated or is waking up"), })))) }) @@ -6452,7 +6476,7 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("metadata.annotations[gardener.cloud/operation]"), - "Detail": Equal("operation is not permitted when shoot is hibernated or is waking up"), + "Detail": Equal("operation 'rotate-rollout-workers=worker-name' is not permitted when shoot is hibernated or is waking up"), })))) }) @@ -6492,7 +6516,7 @@ var _ = Describe("Shoot Validation Tests", func() { }) DescribeTable("forbid certain rotation operations when shoot is waking up", - func(operation string) { + func(operation, forbiddenOp string) { shoot.Spec.Hibernation = &core.Hibernation{Enabled: ptr.To(false)} shoot.Status = core.ShootStatus{ IsHibernated: true, @@ -6502,7 +6526,7 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("metadata.annotations[gardener.cloud/operation]"), - "Detail": ContainSubstring("operation is not permitted when shoot is hibernated or is waking up"), + "Detail": ContainSubstring(fmt.Sprintf("operation '%s' is not permitted when shoot is hibernated or is waking up", forbiddenOp)), })))) delete(shoot.Annotations, "gardener.cloud/operation") @@ -6510,20 +6534,21 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("metadata.annotations[maintenance.gardener.cloud/operation]"), - "Detail": ContainSubstring("operation is not permitted when shoot is hibernated or is waking up"), + "Detail": ContainSubstring(fmt.Sprintf("operation '%s' is not permitted when shoot is hibernated or is waking up", forbiddenOp)), })))) delete(shoot.Annotations, "maintenance.gardener.cloud/operation") }, - Entry("rotate-credentials-start", "rotate-credentials-start"), - Entry("rotate-credentials-start-without-workers-rollout", "rotate-credentials-start-without-workers-rollout"), - Entry("rotate-credentials-complete", "rotate-credentials-complete"), - Entry("rotate-etcd-encryption-key", "rotate-etcd-encryption-key"), - Entry("rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key-start"), - Entry("rotate-etcd-encryption-key-complete", "rotate-etcd-encryption-key-complete"), - Entry("rotate-serviceaccount-key-start", "rotate-serviceaccount-key-start"), - Entry("rotate-serviceaccount-key-start-without-workers-rollout", "rotate-serviceaccount-key-start-without-workers-rollout"), - Entry("rotate-serviceaccount-key-complete", "rotate-serviceaccount-key-complete"), + Entry("rotate-credentials-start", "rotate-credentials-start", "rotate-credentials-start"), + Entry("rotate-credentials-start-without-workers-rollout", "rotate-credentials-start-without-workers-rollout", "rotate-credentials-start-without-workers-rollout"), + Entry("rotate-credentials-complete", "rotate-credentials-complete", "rotate-credentials-complete"), + Entry("rotate-etcd-encryption-key", "rotate-etcd-encryption-key", "rotate-etcd-encryption-key"), + Entry("rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key-start"), + Entry("rotate-etcd-encryption-key-complete", "rotate-etcd-encryption-key-complete", "rotate-etcd-encryption-key-complete"), + Entry("rotate-serviceaccount-key-start", "rotate-serviceaccount-key-start", "rotate-serviceaccount-key-start"), + Entry("rotate-serviceaccount-key-start-without-workers-rollout", "rotate-serviceaccount-key-start-without-workers-rollout", "rotate-serviceaccount-key-start-without-workers-rollout"), + Entry("rotate-serviceaccount-key-complete", "rotate-serviceaccount-key-complete", "rotate-serviceaccount-key-complete"), + Entry("multiple-operations", "rotate-ssh-keypair;rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key-start"), ) DescribeTable("not forbid certain rotation maintenance operations when shoot is in deletion", @@ -6554,7 +6579,7 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeForbidden), "Field": Equal("spec.hibernation.enabled"), - "Detail": ContainSubstring("shoot cannot be hibernated when maintenance.gardener.cloud/operation=" + operation + " annotation is set"), + "Detail": ContainSubstring(fmt.Sprintf("shoot cannot be hibernated when maintenance.gardener.cloud/operation annotation contains %s operation", operation)), })))) }, @@ -6685,6 +6710,103 @@ var _ = Describe("Shoot Validation Tests", func() { Expect(ValidateShootUpdate(newShoot, shoot)).To(BeEmpty()) }) + + Context("multiple operations", func() { + BeforeEach(func() { + shoot.Status.LastOperation = &core.LastOperation{ + Type: core.LastOperationTypeCreate, + State: core.LastOperationStateSucceeded, + } + }) + + DescribeTable("should accept valid multiple operations", + func(operation string) { + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "gardener.cloud/operation", operation) + Expect(ValidateShoot(shoot)).To(BeEmpty()) + delete(shoot.Annotations, "gardener.cloud/operation") + + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "maintenance.gardener.cloud/operation", operation) + Expect(ValidateShoot(shoot)).To(BeEmpty()) + delete(shoot.Annotations, "maintenance.gardener.cloud/operation") + }, + + Entry("single gardener operation", "rotate-ssh-keypair"), + Entry("two parallel operations", "rotate-ssh-keypair;rotate-observability-credentials"), + Entry("three parallel operations", "rotate-ssh-keypair;rotate-observability-credentials;rotate-ca-start"), + Entry("operations with spaces", " rotate-ssh-keypair ; rotate-observability-credentials "), + ) + + DescribeTable("should reject operations not allowed to run in parallel", + func(operation, expectedOperation string) { + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "gardener.cloud/operation", operation) + Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("metadata.annotations[gardener.cloud/operation]"), + "Detail": ContainSubstring(fmt.Sprintf("operation '%s' is not permitted to be run in parallel with other operations", expectedOperation)), + })))) + delete(shoot.Annotations, "gardener.cloud/operation") + }, + + Entry("retry with other operation", "retry;rotate-ssh-keypair", "retry"), + Entry("maintain with other operation", "rotate-ssh-keypair; maintain", "maintain"), + ) + + It("should return an error on first not allowed to be run in parallel operation", func() { + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "gardener.cloud/operation", "retry;reconcile;maintain") + Expect(ValidateShoot(shoot)).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("metadata.annotations[gardener.cloud/operation]"), + "Detail": ContainSubstring("operation 'retry' is not permitted to be run in parallel with other operations"), + })))) + }) + + DescribeTable("should reject forbidden operation combinations", + func(operation, notCompatibleOperation, notCompatibleWith string) { + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "gardener.cloud/operation", operation) + Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("metadata.annotations[gardener.cloud/operation]"), + "Detail": ContainSubstring(fmt.Sprintf("operation '%s' is not permitted to be run together with %s operations", notCompatibleOperation, notCompatibleWith)), + })))) + delete(shoot.Annotations, "gardener.cloud/operation") + + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "maintenance.gardener.cloud/operation", operation) + Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("metadata.annotations[maintenance.gardener.cloud/operation]"), + "Detail": ContainSubstring(fmt.Sprintf("operation '%s' is not permitted to be run together with %s maintenance operations", notCompatibleOperation, notCompatibleWith)), + })))) + delete(shoot.Annotations, "maintenance.gardener.cloud/operation") + }, + + Entry("rotate-credentials-start with rotate-credentials-start-without-workers-rollout", "rotate-credentials-start;rotate-credentials-start-without-workers-rollout", + "rotate-credentials-start", "rotate-ca-start-without-workers-rollout, rotate-serviceaccount-key-start-without-workers-rollout, rotate-credentials-start-without-workers-rollout"), + Entry("rotate-credentials-start with rotate-ca-start-without-workers-rollout", "rotate-credentials-start;rotate-ca-start-without-workers-rollout", + "rotate-credentials-start", "rotate-ca-start-without-workers-rollout, rotate-serviceaccount-key-start-without-workers-rollout, rotate-credentials-start-without-workers-rollout"), + Entry("rotate-credentials-start with rotate-serviceaccount-key-start-without-workers-rollout", "rotate-credentials-start;rotate-serviceaccount-key-start-without-workers-rollout", + "rotate-credentials-start", "rotate-ca-start-without-workers-rollout, rotate-serviceaccount-key-start-without-workers-rollout, rotate-credentials-start-without-workers-rollout"), + Entry("rotate-credentials-start-without-workers-rollout with rotate-ca-start", "rotate-credentials-start-without-workers-rollout;rotate-ca-start", + "rotate-credentials-start-without-workers-rollout", "rotate-ca-start, rotate-serviceaccount-key-start"), + Entry("rotate-credentials-start-without-workers-rollout with rotate-serviceaccount-key-start", "rotate-credentials-start-without-workers-rollout;rotate-serviceaccount-key-start", + "rotate-credentials-start-without-workers-rollout", "rotate-ca-start, rotate-serviceaccount-key-start"), + Entry("rotate-ca-start with rotate-ca-start-without-workers-rollout", "rotate-ca-start;rotate-ca-start-without-workers-rollout", + "rotate-ca-start", "rotate-ca-start-without-workers-rollout"), + Entry("rotate-serviceaccount-key-start with rotate-serviceaccount-key-start-without-workers-rollout", "rotate-serviceaccount-key-start;rotate-serviceaccount-key-start-without-workers-rollout", + "rotate-serviceaccount-key-start", "rotate-serviceaccount-key-start-without-workers-rollout"), + Entry("rotate-etcd-encryption-key with rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key;rotate-etcd-encryption-key-start", + "rotate-etcd-encryption-key", "rotate-etcd-encryption-key-start"), + ) + + It("should prevent duplicate operations across annotations", func() { + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "gardener.cloud/operation", "rotate-observability-credentials;rotate-ssh-keypair") + metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, "maintenance.gardener.cloud/operation", "rotate-etcd-encryption-key-start;rotate-ssh-keypair") + Expect(ValidateShoot(shoot)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("metadata.annotations"), + "Detail": ContainSubstring("must not have any equal operations"), + })))) + }) + }) }) Context("scheduler name", func() { diff --git a/pkg/apis/operator/v1alpha1/helper/garden.go b/pkg/apis/operator/v1alpha1/helper/garden.go index 7f02484d118..30602b2b357 100644 --- a/pkg/apis/operator/v1alpha1/helper/garden.go +++ b/pkg/apis/operator/v1alpha1/helper/garden.go @@ -12,7 +12,9 @@ import ( "k8s.io/utils/ptr" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" + "github.com/gardener/gardener/pkg/utils" ) // GetCARotationPhase returns the specified garden CA rotation phase or an empty string @@ -251,3 +253,8 @@ func GetEncryptedResourcesInStatus(gardenStatus operatorv1alpha1.GardenStatus) [ return nil } + +// GetGardenerOperations returns the Garden's gardener operations specified in the operation annotation. +func GetGardenerOperations(annotations map[string]string) []string { + return utils.SplitAndTrimString(annotations[v1beta1constants.GardenerOperation], v1beta1constants.GardenerOperationsSeparator) +} diff --git a/pkg/apis/operator/v1alpha1/helper/garden_test.go b/pkg/apis/operator/v1alpha1/helper/garden_test.go index 72fc0297519..a63533578f9 100644 --- a/pkg/apis/operator/v1alpha1/helper/garden_test.go +++ b/pkg/apis/operator/v1alpha1/helper/garden_test.go @@ -285,4 +285,26 @@ var _ = Describe("helper", func() { Entry("without resources", operatorv1alpha1.GardenStatus{Credentials: &operatorv1alpha1.Credentials{}}, nil), Entry("with resources", operatorv1alpha1.GardenStatus{Credentials: &operatorv1alpha1.Credentials{EncryptionAtRest: &operatorv1alpha1.EncryptionAtRest{Resources: []string{"configmaps", "shoots.core.gardener.cloud"}}}}, []string{"configmaps", "shoots.core.gardener.cloud"}), ) + + DescribeTable("#GetGardenerOperations", + func(annotations map[string]string, expectedResult []string) { + Expect(GetGardenerOperations(annotations)).To(Equal(expectedResult)) + }, + Entry("annotations are empty", nil, nil), + Entry("gardener.cloud/operation annotation is not set", map[string]string{ + "foo": "bar", + }, nil), + Entry("gardener.cloud/operation annotation is empty", map[string]string{ + "gardener.cloud/operation": "", + }, nil), + Entry("gardener.cloud/operation has single operation", map[string]string{ + "gardener.cloud/operation": "reconcile", + }, []string{"reconcile"}), + Entry("gardener.cloud/operation has multiple operations", map[string]string{ + "gardener.cloud/operation": "reconcile;rotate-credentials-start;rotate-ssh-keypair", + }, []string{"reconcile", "rotate-credentials-start", "rotate-ssh-keypair"}), + Entry("gardener.cloud/operation has whitespaces", map[string]string{ + "gardener.cloud/operation": "reconcile ;rotate-credentials-start ; rotate-ssh-keypair; ", + }, []string{"reconcile", "rotate-credentials-start", "rotate-ssh-keypair", ""}), + ) }) diff --git a/pkg/apis/operator/v1alpha1/validation/garden.go b/pkg/apis/operator/v1alpha1/validation/garden.go index c88f3d80cb8..5a563554093 100644 --- a/pkg/apis/operator/v1alpha1/validation/garden.go +++ b/pkg/apis/operator/v1alpha1/validation/garden.go @@ -48,9 +48,14 @@ import ( ) var ( - gardenCoreScheme *runtime.Scheme + gardenCoreScheme *runtime.Scheme + incompatibleOperationAnnotations = map[string]sets.Set[string]{ + v1beta1constants.OperationRotateETCDEncryptionKey: sets.New( + v1beta1constants.OperationRotateETCDEncryptionKeyStart, + ), + } // TODO(AleksandarSavchev): Remove this variable and the associated validation after support for Kubernetes v1.33 is dropped. - forbiddenETCDEncryptionKeyShootOperationsWithK8s134 = sets.New( + forbiddenETCDEncryptionKeyOperationsWithK8s134 = sets.New( v1beta1constants.OperationRotateETCDEncryptionKeyStart, v1beta1constants.OperationRotateETCDEncryptionKeyComplete, ) @@ -67,7 +72,7 @@ func init() { func ValidateGarden(garden *operatorv1alpha1.Garden, extensions []operatorv1alpha1.Extension) field.ErrorList { allErrs := field.ErrorList{} - allErrs = append(allErrs, validateOperation(garden.Annotations[v1beta1constants.GardenerOperation], garden, field.NewPath("metadata", "annotations"))...) + allErrs = append(allErrs, validateOperation(helper.GetGardenerOperations(garden.Annotations), garden, field.NewPath("metadata", "annotations"))...) allErrs = append(allErrs, validateDNS(garden.Spec.DNS, field.NewPath("spec", "dns"))...) allErrs = append(allErrs, validateExtensions(garden.Spec.Extensions, extensions, field.NewPath("spec", "extensions"))...) allErrs = append(allErrs, validateRuntimeCluster(garden.Spec.DNS, garden.Spec.RuntimeCluster, helper.HighAvailabilityEnabled(garden), field.NewPath("spec", "runtimeCluster"))...) @@ -610,33 +615,38 @@ func validateGardenerDashboardConfig(config *operatorv1alpha1.GardenerDashboardC return allErrs } -func validateOperation(operation string, garden *operatorv1alpha1.Garden, fldPath *field.Path) field.ErrorList { +func validateOperation(operations []string, garden *operatorv1alpha1.Garden, fldPath *field.Path) field.ErrorList { var ( allErrs = field.ErrorList{} + fldPathOp = fldPath.Key(v1beta1constants.GardenerOperation) k8sLess134, _ = versionutils.CheckVersionMeetsConstraint(garden.Spec.VirtualCluster.Kubernetes.Version, "< 1.34") ) - if operation == "" { + if len(operations) == 0 { return allErrs } - fldPathOp := fldPath.Key(v1beta1constants.GardenerOperation) + // Create a `set` to deduplicate the operations array. + operations = sets.New(operations...).UnsortedList() - if operation != "" { - if forbiddenETCDEncryptionKeyShootOperationsWithK8s134.Has(operation) && !k8sLess134 { + for _, operation := range operations { + if forbiddenETCDEncryptionKeyOperationsWithK8s134.Has(operation) && !k8sLess134 { allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("for Kubernetes versions >= 1.34, operation '%s' is no longer supported, please use 'rotate-etcd-encryption-key' instead, which performs a complete etcd encryption key rotation", operation))) } if !operatorv1alpha1.AvailableOperationAnnotations.Has(operation) { allErrs = append(allErrs, field.NotSupported(fldPathOp, operation, sets.List(operatorv1alpha1.AvailableOperationAnnotations))) } + if forbiddenOps, ok := incompatibleOperationAnnotations[operation]; ok && forbiddenOps.HasAny(operations...) { + allErrs = append(allErrs, field.Forbidden(fldPathOp, fmt.Sprintf("operation '%s' is not permitted to be run together with %s operations", operation, strings.Join(sets.List(forbiddenOps), ", ")))) + } } - allErrs = append(allErrs, validateOperationContext(operation, garden, fldPathOp)...) + allErrs = append(allErrs, validateOperationContext(operations, garden, fldPathOp)...) return allErrs } -func validateOperationContext(operation string, garden *operatorv1alpha1.Garden, fldPath *field.Path) field.ErrorList { +func validateOperationContext(operations []string, garden *operatorv1alpha1.Garden, fldPath *field.Path) field.ErrorList { var ( allErrs = field.ErrorList{} encryptedResources = sets.New[schema.GroupResource]() @@ -659,110 +669,112 @@ func validateOperationContext(operation string, garden *operatorv1alpha1.Garden, resourcesToEncrypt.Insert(sharedcomponent.GetResourcesForEncryptionFromConfig(encryptionConfig)...) resourcesToEncrypt.Insert(sharedcomponent.GetResourcesForEncryptionFromConfig(gardenerEncryptionConfig)...) - switch operation { - case v1beta1constants.OperationRotateCredentialsStart: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if garden has deletion timestamp")) - } - if phase := helper.GetCARotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if .status.credentials.rotation.certificateAuthorities.phase is not 'Completed'")) - } - if phase := helper.GetServiceAccountKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if .status.credentials.rotation.serviceAccountKey.phase is not 'Completed'")) - } - if phase := helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if .status.credentials.rotation.etcdEncryptionKey.phase is not 'Completed'")) - } - if phase := helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if .status.credentials.rotation.workloadIdentityKey.phase is not 'Completed'")) - } - if !resourcesToEncrypt.Equal(encryptedResources) { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials because a previous encryption configuration change is currently being rolled out")) - } - case v1beta1constants.OperationRotateCredentialsComplete: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if garden has deletion timestamp")) - } - if helper.GetCARotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if .status.credentials.rotation.certificateAuthorities.phase is not 'Prepared'")) - } - if helper.GetServiceAccountKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if .status.credentials.rotation.serviceAccountKey.phase is not 'Prepared'")) - } - if k8sLess134 && helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if .status.credentials.rotation.etcdEncryptionKey.phase is not 'Prepared'")) - } - if helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if .status.credentials.rotation.workloadIdentityKey.phase is not 'Prepared'")) - } + for _, operation := range operations { + switch operation { + case v1beta1constants.OperationRotateCredentialsStart: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if garden has deletion timestamp")) + } + if phase := helper.GetCARotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if .status.credentials.rotation.certificateAuthorities.phase is not 'Completed'")) + } + if phase := helper.GetServiceAccountKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if .status.credentials.rotation.serviceAccountKey.phase is not 'Completed'")) + } + if phase := helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if .status.credentials.rotation.etcdEncryptionKey.phase is not 'Completed'")) + } + if phase := helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials if .status.credentials.rotation.workloadIdentityKey.phase is not 'Completed'")) + } + if !resourcesToEncrypt.Equal(encryptedResources) { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start rotation of all credentials because a previous encryption configuration change is currently being rolled out")) + } + case v1beta1constants.OperationRotateCredentialsComplete: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if garden has deletion timestamp")) + } + if helper.GetCARotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if .status.credentials.rotation.certificateAuthorities.phase is not 'Prepared'")) + } + if helper.GetServiceAccountKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if .status.credentials.rotation.serviceAccountKey.phase is not 'Prepared'")) + } + if k8sLess134 && helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if .status.credentials.rotation.etcdEncryptionKey.phase is not 'Prepared'")) + } + if helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete rotation of all credentials if .status.credentials.rotation.workloadIdentityKey.phase is not 'Prepared'")) + } - case v1beta1constants.OperationRotateCAStart: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start CA rotation if garden has deletion timestamp")) - } - if phase := helper.GetCARotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start CA rotation if .status.credentials.rotation.certificateAuthorities.phase is not 'Completed'")) - } - case v1beta1constants.OperationRotateCAComplete: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete CA rotation if garden has deletion timestamp")) - } - if helper.GetCARotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete CA rotation if .status.credentials.rotation.certificateAuthorities.phase is not 'Prepared'")) - } + case v1beta1constants.OperationRotateCAStart: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start CA rotation if garden has deletion timestamp")) + } + if phase := helper.GetCARotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start CA rotation if .status.credentials.rotation.certificateAuthorities.phase is not 'Completed'")) + } + case v1beta1constants.OperationRotateCAComplete: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete CA rotation if garden has deletion timestamp")) + } + if helper.GetCARotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete CA rotation if .status.credentials.rotation.certificateAuthorities.phase is not 'Prepared'")) + } - case v1beta1constants.OperationRotateServiceAccountKeyStart: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start service account key rotation if garden has deletion timestamp")) - } - if phase := helper.GetServiceAccountKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start service account key rotation if .status.credentials.rotation.serviceAccountKey.phase is not 'Completed'")) - } - case v1beta1constants.OperationRotateServiceAccountKeyComplete: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete service account key rotation if garden has deletion timestamp")) - } - if helper.GetServiceAccountKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete service account key rotation if .status.credentials.rotation.serviceAccountKey.phase is not 'Prepared'")) - } + case v1beta1constants.OperationRotateServiceAccountKeyStart: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start service account key rotation if garden has deletion timestamp")) + } + if phase := helper.GetServiceAccountKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start service account key rotation if .status.credentials.rotation.serviceAccountKey.phase is not 'Completed'")) + } + case v1beta1constants.OperationRotateServiceAccountKeyComplete: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete service account key rotation if garden has deletion timestamp")) + } + if helper.GetServiceAccountKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete service account key rotation if .status.credentials.rotation.serviceAccountKey.phase is not 'Prepared'")) + } - case v1beta1constants.OperationRotateETCDEncryptionKeyStart, v1beta1constants.OperationRotateETCDEncryptionKey: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start ETCD encryption key rotation if garden has deletion timestamp")) - } - if phase := helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start ETCD encryption key rotation if .status.credentials.rotation.etcdEncryptionKey.phase is not 'Completed'")) - } - if !resourcesToEncrypt.Equal(encryptedResources) { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start ETCD encryption key rotation because a previous encryption configuration change is currently being rolled out")) - } - case v1beta1constants.OperationRotateETCDEncryptionKeyComplete: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete ETCD encryption key rotation if garden has deletion timestamp")) - } - if helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete ETCD encryption key rotation if .status.credentials.rotation.etcdEncryptionKey.phase is not 'Prepared'")) - } + case v1beta1constants.OperationRotateETCDEncryptionKeyStart, v1beta1constants.OperationRotateETCDEncryptionKey: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start ETCD encryption key rotation if garden has deletion timestamp")) + } + if phase := helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start ETCD encryption key rotation if .status.credentials.rotation.etcdEncryptionKey.phase is not 'Completed'")) + } + if !resourcesToEncrypt.Equal(encryptedResources) { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start ETCD encryption key rotation because a previous encryption configuration change is currently being rolled out")) + } + case v1beta1constants.OperationRotateETCDEncryptionKeyComplete: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete ETCD encryption key rotation if garden has deletion timestamp")) + } + if helper.GetETCDEncryptionKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete ETCD encryption key rotation if .status.credentials.rotation.etcdEncryptionKey.phase is not 'Prepared'")) + } - case v1beta1constants.OperationRotateObservabilityCredentials: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start Observability credentials rotation if garden has deletion timestamp")) - } + case v1beta1constants.OperationRotateObservabilityCredentials: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start Observability credentials rotation if garden has deletion timestamp")) + } - case operatorv1alpha1.OperationRotateWorkloadIdentityKeyStart: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start workload identity key rotation if garden has deletion timestamp")) - } - if phase := helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start workload identity key rotation if .status.credentials.rotation.workloadIdentityKey.phase is not 'Completed'")) - } + case operatorv1alpha1.OperationRotateWorkloadIdentityKeyStart: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start workload identity key rotation if garden has deletion timestamp")) + } + if phase := helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials); len(phase) > 0 && phase != gardencorev1beta1.RotationCompleted { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot start workload identity key rotation if .status.credentials.rotation.workloadIdentityKey.phase is not 'Completed'")) + } - case operatorv1alpha1.OperationRotateWorkloadIdentityKeyComplete: - if garden.DeletionTimestamp != nil { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete workload identity key rotation if garden has deletion timestamp")) - } - if helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { - allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete workload identity key rotation if .status.credentials.rotation.workloadIdentityKey.phase is not 'Prepared'")) + case operatorv1alpha1.OperationRotateWorkloadIdentityKeyComplete: + if garden.DeletionTimestamp != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete workload identity key rotation if garden has deletion timestamp")) + } + if helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPrepared { + allErrs = append(allErrs, field.Forbidden(fldPath, "cannot complete workload identity key rotation if .status.credentials.rotation.workloadIdentityKey.phase is not 'Prepared'")) + } } } diff --git a/pkg/apis/operator/v1alpha1/validation/garden_test.go b/pkg/apis/operator/v1alpha1/validation/garden_test.go index dc7ba2c7ce1..cef2de0ab5e 100644 --- a/pkg/apis/operator/v1alpha1/validation/garden_test.go +++ b/pkg/apis/operator/v1alpha1/validation/garden_test.go @@ -1184,6 +1184,35 @@ var _ = Describe("Validation Tests", func() { }), ) + DescribeTable("multiple operations", + func(operations string, matcher gomegatypes.GomegaMatcher) { + metav1.SetMetaDataAnnotation(&garden.ObjectMeta, "gardener.cloud/operation", operations) + Expect(ValidateGarden(garden, extensions)).To(matcher) + }, + + Entry("single gardener operation", "rotate-ca-start", BeEmpty()), + Entry("two parallel operations", "rotate-ca-start;rotate-observability-credentials", BeEmpty()), + Entry("three parallel operations", "rotate-ca-start;rotate-observability-credentials;rotate-serviceaccount-key-start", BeEmpty()), + Entry("operations with whitespaces", " rotate-ca-start ; rotate-observability-credentials ", BeEmpty()), + Entry("not supported operation", "reconcile;retry", ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeNotSupported), + "Field": Equal("metadata.annotations[gardener.cloud/operation]"), + "BadValue": Equal("retry"), + })))), + ) + + DescribeTable("should reject operations not allowed to run in parallel", + func(operation, expectedOperation string) { + metav1.SetMetaDataAnnotation(&garden.ObjectMeta, "gardener.cloud/operation", operation) + Expect(ValidateGarden(garden, extensions)).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("metadata.annotations[gardener.cloud/operation]"), + "Detail": ContainSubstring(fmt.Sprintf("operation '%s' is not permitted to be run together with", expectedOperation)), + })))) + }, + + Entry("rotate-etcd-encryption-key and rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key;rotate-etcd-encryption-key-start", "rotate-etcd-encryption-key"), + ) }) Context("extensions", func() { diff --git a/pkg/apiserver/registry/core/shoot/strategy.go b/pkg/apiserver/registry/core/shoot/strategy.go index b1220723b12..5ae374744e2 100644 --- a/pkg/apiserver/registry/core/shoot/strategy.go +++ b/pkg/apiserver/registry/core/shoot/strategy.go @@ -7,6 +7,7 @@ package shoot import ( "context" "fmt" + "slices" "strings" "time" @@ -15,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" @@ -27,8 +29,10 @@ import ( "github.com/gardener/gardener/pkg/apis/core" gardencorehelper "github.com/gardener/gardener/pkg/apis/core/helper" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" "github.com/gardener/gardener/pkg/apis/core/validation" "github.com/gardener/gardener/pkg/features" + "github.com/gardener/gardener/pkg/utils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" ) @@ -71,6 +75,13 @@ func (shootStrategy) PrepareForUpdate(_ context.Context, obj, old runtime.Object newShoot.Status = oldShoot.Status // can only be changed by shoots/status subresource newShoot.Spec.SeedName = oldShoot.Spec.SeedName // can only be changed by shoots/binding subresource + if op, ok := newShoot.Annotations[v1beta1constants.GardenerOperation]; ok { + newShoot.Annotations[v1beta1constants.GardenerOperation] = cleanUpOperation(op) + } + if op, ok := newShoot.Annotations[v1beta1constants.GardenerMaintenanceOperation]; ok { + newShoot.Annotations[v1beta1constants.GardenerMaintenanceOperation] = cleanUpOperation(op) + } + if mustIncreaseGeneration(oldShoot, newShoot) { newShoot.Generation = oldShoot.Generation + 1 } @@ -106,6 +117,8 @@ func mustIncreaseGeneration(oldShoot, newShoot *core.Shoot) bool { var ( mustIncrease bool mustRemoveOperationAnnotation bool + operations = v1beta1helper.GetShootGardenerOperations(newShoot.Annotations) + updatedOperations = slices.Clone(operations) ) switch lastOperation.State { @@ -115,51 +128,80 @@ func mustIncreaseGeneration(oldShoot, newShoot *core.Shoot) bool { } default: - switch newShoot.Annotations[v1beta1constants.GardenerOperation] { - case v1beta1constants.GardenerOperationReconcile: - mustIncrease, mustRemoveOperationAnnotation = true, true + for _, operation := range operations { + switch operation { + case v1beta1constants.GardenerOperationReconcile: + mustIncrease = true + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + + case v1beta1constants.OperationRotateCAStart, + v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, + v1beta1constants.OperationRotateCAComplete, + v1beta1constants.OperationRotateServiceAccountKeyStart, + v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, + v1beta1constants.OperationRotateServiceAccountKeyComplete, + v1beta1constants.OperationRotateETCDEncryptionKey, + v1beta1constants.OperationRotateETCDEncryptionKeyStart, + v1beta1constants.OperationRotateETCDEncryptionKeyComplete, + v1beta1constants.OperationRotateObservabilityCredentials: + // We don't want to remove the annotation so that the gardenlet can pick it up and perform + // the rotation. It has to remove the annotation after it is done. + mustIncrease = true + case v1beta1constants.OperationRotateCredentialsStart: + // We remove operations that are covered by rotate-credentials-start + mustIncrease = true + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, v1beta1constants.OperationRotateCAStart, + v1beta1constants.OperationRotateServiceAccountKeyStart, + v1beta1constants.OperationRotateETCDEncryptionKey, + v1beta1constants.OperationRotateETCDEncryptionKeyStart, + v1beta1constants.OperationRotateObservabilityCredentials, + v1beta1constants.ShootOperationRotateSSHKeypair, + ) + case v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout: + // We remove operations that are covered by rotate-credentials-start-without-workers-rollout + mustIncrease = true + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, + v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, + v1beta1constants.OperationRotateETCDEncryptionKey, + v1beta1constants.OperationRotateETCDEncryptionKeyStart, + v1beta1constants.OperationRotateObservabilityCredentials, + v1beta1constants.ShootOperationRotateSSHKeypair, + ) + case v1beta1constants.OperationRotateCredentialsComplete: + // We remove operations that are covered by rotate-credentials-complete + mustIncrease = true + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, v1beta1constants.OperationRotateCAComplete, + v1beta1constants.OperationRotateServiceAccountKeyComplete, + v1beta1constants.OperationRotateETCDEncryptionKeyComplete, + ) + + case v1beta1constants.ShootOperationRotateSSHKeypair: + if !gardencorehelper.ShootEnablesSSHAccess(newShoot) { + // If SSH is not enabled for the Shoot, don't increase generation, just remove the annotation + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + } else { + mustIncrease = true + } + + case v1beta1constants.ShootOperationForceInPlaceUpdate: + // The annotation will be removed later by gardenlet once the in-place update is finished. + // The generation will be increased if there really is a spec change in the object. + mustIncrease, mustRemoveOperationAnnotation = false, false + } - case v1beta1constants.OperationRotateCredentialsStart, - v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout, - v1beta1constants.OperationRotateCredentialsComplete, - v1beta1constants.OperationRotateCAStart, - v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, - v1beta1constants.OperationRotateCAComplete, - v1beta1constants.OperationRotateServiceAccountKeyStart, - v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, - v1beta1constants.OperationRotateServiceAccountKeyComplete, - v1beta1constants.OperationRotateETCDEncryptionKey, - v1beta1constants.OperationRotateETCDEncryptionKeyStart, - v1beta1constants.OperationRotateETCDEncryptionKeyComplete, - v1beta1constants.OperationRotateObservabilityCredentials: - // We don't want to remove the annotation so that the gardenlet can pick it up and perform - // the rotation. It has to remove the annotation after it is done. - mustIncrease, mustRemoveOperationAnnotation = true, false - - case v1beta1constants.ShootOperationRotateSSHKeypair: - if !gardencorehelper.ShootEnablesSSHAccess(newShoot) { - // If SSH is not enabled for the Shoot, don't increase generation, just remove the annotation - mustIncrease, mustRemoveOperationAnnotation = false, true - } else { + if strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers) || + strings.HasPrefix(operation, v1beta1constants.OperationRolloutWorkers) { + // We don't want to remove the annotation so that the gardenlet can pick it up and perform + // the rotation/rollout. It has to remove the annotation after it is done. mustIncrease, mustRemoveOperationAnnotation = true, false } - - case v1beta1constants.ShootOperationForceInPlaceUpdate: - // The annotation will be removed later by gardenlet once the in-place update is finished. - // The generation will be increased if there really is a spec change in the object. - mustIncrease, mustRemoveOperationAnnotation = false, false - } - - if strings.HasPrefix(newShoot.Annotations[v1beta1constants.GardenerOperation], v1beta1constants.OperationRotateRolloutWorkers) || - strings.HasPrefix(newShoot.Annotations[v1beta1constants.GardenerOperation], v1beta1constants.OperationRolloutWorkers) { - // We don't want to remove the annotation so that the gardenlet can pick it up and perform - // the rotation/rollout. It has to remove the annotation after it is done. - mustIncrease, mustRemoveOperationAnnotation = true, false } } - if mustRemoveOperationAnnotation { + if mustRemoveOperationAnnotation || len(updatedOperations) == 0 { delete(newShoot.Annotations, v1beta1constants.GardenerOperation) + } else if len(operations) != len(updatedOperations) { + newShoot.Annotations[v1beta1constants.GardenerOperation] = strings.Join(updatedOperations, v1beta1constants.GardenerOperationsSeparator) } if mustIncrease { return true @@ -386,3 +428,8 @@ func SyncEncryptedResourcesStatus(shoot *core.Shoot) { shoot.Status.Credentials.EncryptionAtRest.Resources = nil } } + +func cleanUpOperation(operation string) string { + operations := utils.SplitAndTrimString(operation, v1beta1constants.GardenerOperationsSeparator) + return strings.Join(sets.New(operations...).UnsortedList(), v1beta1constants.GardenerOperationsSeparator) +} diff --git a/pkg/apiserver/registry/core/shoot/strategy_test.go b/pkg/apiserver/registry/core/shoot/strategy_test.go index 7c5044100b4..d6df682f8b0 100644 --- a/pkg/apiserver/registry/core/shoot/strategy_test.go +++ b/pkg/apiserver/registry/core/shoot/strategy_test.go @@ -6,6 +6,7 @@ package shoot_test import ( "context" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -19,6 +20,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" . "github.com/gardener/gardener/pkg/apiserver/registry/core/shoot" "github.com/gardener/gardener/pkg/features" + "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/test" ) @@ -503,7 +505,7 @@ var _ = Describe("Strategy", func() { }) DescribeTable("operation annotations", - func(operationAnnotation string, mutateOldShoot func(*core.Shoot), shouldIncreaseGeneration, shouldKeepAnnotation bool) { + func(operationAnnotation string, mutateOldShoot func(*core.Shoot), shouldIncreaseGeneration bool, mutatedAnnotation []string) { oldShoot := &core.Shoot{ Spec: core.ShootSpec{ Provider: core.Provider{ @@ -534,8 +536,9 @@ var _ = Describe("Strategy", func() { } Expect(newShoot.Generation).To(Equal(expectedGeneration)) - if shouldKeepAnnotation { - Expect(newShoot.Annotations).To(HaveKeyWithValue(v1beta1constants.GardenerOperation, operationAnnotation)) + if mutatedAnnotation != nil { + Expect(newShoot.Annotations).To(HaveKey(v1beta1constants.GardenerOperation)) + Expect(utils.SplitAndTrimString(newShoot.Annotations[v1beta1constants.GardenerOperation], v1beta1constants.GardenerOperationsSeparator)).To(ConsistOf(mutatedAnnotation)) } else { Expect(newShoot.Annotations).NotTo(HaveKey(v1beta1constants.GardenerOperation)) } @@ -545,134 +548,182 @@ var _ = Describe("Strategy", func() { v1beta1constants.ShootOperationRetry, func(s *core.Shoot) { s.Status.LastOperation.State = core.LastOperationStateFailed }, true, - false, + nil, ), Entry("retry; last operation is not failed", v1beta1constants.ShootOperationRetry, func(s *core.Shoot) { s.Status.LastOperation.State = core.LastOperationStateSucceeded }, false, - true, + []string{v1beta1constants.ShootOperationRetry}, ), Entry("retry; last operation is not set", v1beta1constants.ShootOperationRetry, func(s *core.Shoot) { s.Status.LastOperation = nil }, false, - true, + []string{v1beta1constants.ShootOperationRetry}, ), Entry("reconcile", v1beta1constants.GardenerOperationReconcile, nil, true, - false, + nil, ), Entry("rotate-credentials-start", v1beta1constants.OperationRotateCredentialsStart, nil, true, - true, + []string{v1beta1constants.OperationRotateCredentialsStart}, ), Entry("rotate-credentials-start-without-workers-rollout", v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout, nil, true, - true, + []string{v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout}, ), Entry("rotate-credentials-complete", v1beta1constants.OperationRotateCredentialsComplete, nil, true, - true, + []string{v1beta1constants.OperationRotateCredentialsComplete}, ), Entry("rotate-ssh-keypair (ssh enabled)", v1beta1constants.ShootOperationRotateSSHKeypair, nil, true, - true, + []string{v1beta1constants.ShootOperationRotateSSHKeypair}, ), Entry("rotate-ssh-keypair (ssh is not enabled)", v1beta1constants.ShootOperationRotateSSHKeypair, func(s *core.Shoot) { s.Spec.Provider.Workers = nil }, false, - false, + nil, ), Entry("rotate-observability-credentials", v1beta1constants.OperationRotateObservabilityCredentials, nil, true, - true, + []string{v1beta1constants.OperationRotateObservabilityCredentials}, ), Entry("rotate-etcd-encryption-key", v1beta1constants.OperationRotateETCDEncryptionKey, nil, true, - true, + []string{v1beta1constants.OperationRotateETCDEncryptionKey}, ), Entry("rotate-etcd-encryption-key-start", v1beta1constants.OperationRotateETCDEncryptionKeyStart, nil, true, - true, + []string{v1beta1constants.OperationRotateETCDEncryptionKeyStart}, ), Entry("rotate-etcd-encryption-key-complete", v1beta1constants.OperationRotateETCDEncryptionKeyComplete, nil, true, - true, + []string{v1beta1constants.OperationRotateETCDEncryptionKeyComplete}, ), Entry("rotate-ca-start", v1beta1constants.OperationRotateCAStart, nil, true, - true, + []string{v1beta1constants.OperationRotateCAStart}, ), Entry("rotate-ca-start-without-workers-rollout", v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, nil, true, - true, + []string{v1beta1constants.OperationRotateCAStartWithoutWorkersRollout}, ), Entry("rotate-ca-complete", v1beta1constants.OperationRotateCAComplete, nil, true, - true, + []string{v1beta1constants.OperationRotateCAComplete}, ), Entry("rotate-serviceaccount-key-start", v1beta1constants.OperationRotateServiceAccountKeyStart, nil, true, - true, + []string{v1beta1constants.OperationRotateServiceAccountKeyStart}, ), Entry("rotate-serviceaccount-key-start-without-workers-rollout", v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, nil, true, - true, + []string{v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout}, ), Entry("rotate-serviceaccount-key-complete", v1beta1constants.OperationRotateServiceAccountKeyComplete, nil, true, - true, + []string{v1beta1constants.OperationRotateServiceAccountKeyComplete}, ), Entry("rotate-rollout-workers", v1beta1constants.OperationRotateRolloutWorkers+"=foo", nil, true, - true, + []string{v1beta1constants.OperationRotateRolloutWorkers + "=foo"}, ), Entry("force-in-place-update", v1beta1constants.ShootOperationForceInPlaceUpdate, nil, false, + []string{v1beta1constants.ShootOperationForceInPlaceUpdate}, + ), + + Entry("reconcile and rotate-etcd-encryption-key", + fmt.Sprintf("%s;%s", v1beta1constants.GardenerOperationReconcile, v1beta1constants.OperationRotateETCDEncryptionKey), + nil, + true, + []string{v1beta1constants.OperationRotateETCDEncryptionKey}, + ), + + Entry("remove operations covered by rotate-credentials-start", + fmt.Sprintf("%s;%s;%s;%s;%s;%s;%s;%s", v1beta1constants.OperationRotateCredentialsStart, v1beta1constants.OperationRotateCAStart, + v1beta1constants.OperationRotateServiceAccountKeyStart, v1beta1constants.OperationRotateETCDEncryptionKey, v1beta1constants.OperationRotateETCDEncryptionKeyStart, + v1beta1constants.OperationRotateObservabilityCredentials, v1beta1constants.ShootOperationRotateSSHKeypair, v1beta1constants.OperationRotateCAStartWithoutWorkersRollout), + nil, true, + []string{v1beta1constants.OperationRotateCredentialsStart, v1beta1constants.OperationRotateCAStartWithoutWorkersRollout}, + ), + + Entry("remove operations covered by rotate-credentials-start-without-workers-rollout", + fmt.Sprintf("%s;%s;%s;%s;%s;%s;%s;%s", v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout, v1beta1constants.OperationRotateCAStartWithoutWorkersRollout, + v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout, v1beta1constants.OperationRotateETCDEncryptionKey, v1beta1constants.OperationRotateETCDEncryptionKeyStart, + v1beta1constants.OperationRotateObservabilityCredentials, v1beta1constants.ShootOperationRotateSSHKeypair, v1beta1constants.OperationRotateCAStart), + nil, + true, + []string{v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout, v1beta1constants.OperationRotateCAStart}, + ), + + Entry("remove operations covered by rotate-credentials-complete", + fmt.Sprintf("%s;%s;%s;%s;%s", v1beta1constants.OperationRotateCredentialsComplete, v1beta1constants.OperationRotateCAComplete, v1beta1constants.OperationRotateServiceAccountKeyComplete, + v1beta1constants.OperationRotateETCDEncryptionKeyComplete, v1beta1constants.ShootOperationRotateSSHKeypair), + nil, + true, + []string{v1beta1constants.OperationRotateCredentialsComplete, v1beta1constants.ShootOperationRotateSSHKeypair}, + ), + + Entry("remove duplicate operations", + fmt.Sprintf("%s;%s;%s;%s;%s", v1beta1constants.OperationRotateCredentialsComplete, v1beta1constants.OperationRotateCredentialsComplete, v1beta1constants.ShootOperationRotateSSHKeypair, + v1beta1constants.OperationRotateCredentialsComplete, v1beta1constants.ShootOperationRotateSSHKeypair), + nil, + true, + []string{v1beta1constants.OperationRotateCredentialsComplete, v1beta1constants.ShootOperationRotateSSHKeypair}, + ), + + Entry("reconcile and rotate-ssh-keypair (ssh is not enabled)", + fmt.Sprintf("%s;%s", v1beta1constants.GardenerOperationReconcile, v1beta1constants.ShootOperationRotateSSHKeypair), + func(s *core.Shoot) { s.Spec.Provider.Workers = nil }, + true, + nil, ), ) }) diff --git a/pkg/controllermanager/controller/shoot/maintenance/reconciler.go b/pkg/controllermanager/controller/shoot/maintenance/reconciler.go index 5bc6f29379e..c88a95a2c79 100644 --- a/pkg/controllermanager/controller/shoot/maintenance/reconciler.go +++ b/pkg/controllermanager/controller/shoot/maintenance/reconciler.go @@ -692,8 +692,8 @@ func mustMaintainNow(shoot *gardencorev1beta1.Shoot, clock clock.Clock) bool { } func hasMaintainNowAnnotation(shoot *gardencorev1beta1.Shoot) bool { - operation, ok := shoot.Annotations[v1beta1constants.GardenerOperation] - return ok && operation == v1beta1constants.ShootOperationMaintain + operations := v1beta1helper.GetShootGardenerOperations(shoot.Annotations) + return slices.Contains(operations, v1beta1constants.ShootOperationMaintain) } func needsRetry(shoot *gardencorev1beta1.Shoot) bool { diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler.go b/pkg/gardenlet/controller/shoot/shoot/reconciler.go index 7c2bfdaf6c1..74fa855ce11 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler.go @@ -622,151 +622,152 @@ func (r *Reconciler) updateShootStatusOperationStart( LastUpdateTime: now, } - var ( - operation = shoot.Annotations[v1beta1constants.GardenerOperation] - mustRemoveOperationAnnotation bool - ) - k8sLess134, err := versionutils.CompareVersions(shoot.Spec.Kubernetes.Version, "<", "1.34") if err != nil { return fmt.Errorf("failed checking if Shoot k8s version is less than 1.34: %w", err) } - switch operation { - case v1beta1constants.OperationRotateCredentialsStart: - mustRemoveOperationAnnotation = true - startRotationCA(shoot, &now) - startRotationServiceAccountKey(shoot, &now) - if v1beta1helper.ShootEnablesSSHAccess(shoot) { - startRotationSSHKeypair(shoot, &now) - } - startRotationObservability(shoot, &now) - startRotationETCDEncryptionKey(shoot, !k8sLess134, &now) - case v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout: - mustRemoveOperationAnnotation = true - startRotationCAWithoutWorkersRollout(shoot, &now) - startRotationServiceAccountKeyWithoutWorkersRollout(shoot, &now) - if v1beta1helper.ShootEnablesSSHAccess(shoot) { - startRotationSSHKeypair(shoot, &now) - } - startRotationObservability(shoot, &now) - startRotationETCDEncryptionKey(shoot, !k8sLess134, &now) - case v1beta1constants.OperationRotateCredentialsComplete: - mustRemoveOperationAnnotation = true - completeRotationCA(shoot, &now) - completeRotationServiceAccountKey(shoot, &now) - if k8sLess134 { - completeRotationETCDEncryptionKey(shoot, &now) - } + var ( + operations = v1beta1helper.GetShootGardenerOperations(shoot.Annotations) + updatedOperations = slices.Clone(operations) + ) - case v1beta1constants.OperationRotateCAStart: - mustRemoveOperationAnnotation = true - startRotationCA(shoot, &now) - case v1beta1constants.OperationRotateCAStartWithoutWorkersRollout: - mustRemoveOperationAnnotation = true - startRotationCAWithoutWorkersRollout(shoot, &now) - case v1beta1constants.OperationRotateCAComplete: - mustRemoveOperationAnnotation = true - completeRotationCA(shoot, &now) - - case v1beta1constants.ShootOperationRotateSSHKeypair: - mustRemoveOperationAnnotation = true - if v1beta1helper.ShootEnablesSSHAccess(shoot) { - startRotationSSHKeypair(shoot, &now) - } + for _, operation := range operations { + switch operation { + case v1beta1constants.OperationRotateCredentialsStart: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationCA(shoot, &now) + startRotationServiceAccountKey(shoot, &now) + if v1beta1helper.ShootEnablesSSHAccess(shoot) { + startRotationSSHKeypair(shoot, &now) + } + startRotationObservability(shoot, &now) + startRotationETCDEncryptionKey(shoot, !k8sLess134, &now) + case v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationCAWithoutWorkersRollout(shoot, &now) + startRotationServiceAccountKeyWithoutWorkersRollout(shoot, &now) + if v1beta1helper.ShootEnablesSSHAccess(shoot) { + startRotationSSHKeypair(shoot, &now) + } + startRotationObservability(shoot, &now) + startRotationETCDEncryptionKey(shoot, !k8sLess134, &now) + case v1beta1constants.OperationRotateCredentialsComplete: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + completeRotationCA(shoot, &now) + completeRotationServiceAccountKey(shoot, &now) + if k8sLess134 { + completeRotationETCDEncryptionKey(shoot, &now) + } + case v1beta1constants.OperationRotateCAStart: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationCA(shoot, &now) + case v1beta1constants.OperationRotateCAStartWithoutWorkersRollout: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationCAWithoutWorkersRollout(shoot, &now) + case v1beta1constants.OperationRotateCAComplete: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + completeRotationCA(shoot, &now) + + case v1beta1constants.ShootOperationRotateSSHKeypair: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + if v1beta1helper.ShootEnablesSSHAccess(shoot) { + startRotationSSHKeypair(shoot, &now) + } - case v1beta1constants.OperationRotateObservabilityCredentials: - mustRemoveOperationAnnotation = true - startRotationObservability(shoot, &now) - - case v1beta1constants.OperationRotateServiceAccountKeyStart: - mustRemoveOperationAnnotation = true - startRotationServiceAccountKey(shoot, &now) - case v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout: - mustRemoveOperationAnnotation = true - startRotationServiceAccountKeyWithoutWorkersRollout(shoot, &now) - case v1beta1constants.OperationRotateServiceAccountKeyComplete: - mustRemoveOperationAnnotation = true - completeRotationServiceAccountKey(shoot, &now) - - case v1beta1constants.OperationRotateETCDEncryptionKey: - mustRemoveOperationAnnotation = true - startRotationETCDEncryptionKey(shoot, true, &now) - case v1beta1constants.OperationRotateETCDEncryptionKeyStart: - mustRemoveOperationAnnotation = true - startRotationETCDEncryptionKey(shoot, false, &now) - case v1beta1constants.OperationRotateETCDEncryptionKeyComplete: - mustRemoveOperationAnnotation = true - completeRotationETCDEncryptionKey(shoot, &now) - } + case v1beta1constants.OperationRotateObservabilityCredentials: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationObservability(shoot, &now) + + case v1beta1constants.OperationRotateServiceAccountKeyStart: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationServiceAccountKey(shoot, &now) + case v1beta1constants.OperationRotateServiceAccountKeyStartWithoutWorkersRollout: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationServiceAccountKeyWithoutWorkersRollout(shoot, &now) + case v1beta1constants.OperationRotateServiceAccountKeyComplete: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + completeRotationServiceAccountKey(shoot, &now) + + case v1beta1constants.OperationRotateETCDEncryptionKey: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationETCDEncryptionKey(shoot, true, &now) + case v1beta1constants.OperationRotateETCDEncryptionKeyStart: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationETCDEncryptionKey(shoot, false, &now) + case v1beta1constants.OperationRotateETCDEncryptionKeyComplete: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + completeRotationETCDEncryptionKey(shoot, &now) + } - switch { - case strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers): - mustRemoveOperationAnnotation = true - poolNames := sets.NewString(strings.Split(strings.TrimPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers+"="), ",")...) + if strings.HasPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers) { + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + poolNames := sets.NewString(strings.Split(strings.TrimPrefix(operation, v1beta1constants.OperationRotateRolloutWorkers+"="), ",")...) - if v1beta1helper.GetShootCARotationPhase(shoot.Status.Credentials) == gardencorev1beta1.RotationWaitingForWorkersRollout { - v1beta1helper.MutateShootCARotation(shoot, func(rotation *gardencorev1beta1.CARotation) { - rotation.PendingWorkersRollouts = slices.DeleteFunc(rotation.PendingWorkersRollouts, func(rollout gardencorev1beta1.PendingWorkersRollout) bool { - return poolNames.Has(rollout.Name) + if v1beta1helper.GetShootCARotationPhase(shoot.Status.Credentials) == gardencorev1beta1.RotationWaitingForWorkersRollout { + v1beta1helper.MutateShootCARotation(shoot, func(rotation *gardencorev1beta1.CARotation) { + rotation.PendingWorkersRollouts = slices.DeleteFunc(rotation.PendingWorkersRollouts, func(rollout gardencorev1beta1.PendingWorkersRollout) bool { + return poolNames.Has(rollout.Name) + }) }) - }) - } + } - if v1beta1helper.GetShootServiceAccountKeyRotationPhase(shoot.Status.Credentials) == gardencorev1beta1.RotationWaitingForWorkersRollout { - v1beta1helper.MutateShootServiceAccountKeyRotation(shoot, func(rotation *gardencorev1beta1.ServiceAccountKeyRotation) { - rotation.PendingWorkersRollouts = slices.DeleteFunc(rotation.PendingWorkersRollouts, func(rollout gardencorev1beta1.PendingWorkersRollout) bool { - return poolNames.Has(rollout.Name) + if v1beta1helper.GetShootServiceAccountKeyRotationPhase(shoot.Status.Credentials) == gardencorev1beta1.RotationWaitingForWorkersRollout { + v1beta1helper.MutateShootServiceAccountKeyRotation(shoot, func(rotation *gardencorev1beta1.ServiceAccountKeyRotation) { + rotation.PendingWorkersRollouts = slices.DeleteFunc(rotation.PendingWorkersRollouts, func(rollout gardencorev1beta1.PendingWorkersRollout) bool { + return poolNames.Has(rollout.Name) + }) }) - }) - } - case strings.HasPrefix(operation, v1beta1constants.OperationRolloutWorkers): - mustRemoveOperationAnnotation = true - poolNames := sets.NewString(strings.Split(strings.TrimPrefix(operation, v1beta1constants.OperationRolloutWorkers+"="), ",")...) - - if poolNames.Has("*") { - poolNames = sets.NewString() - for _, pool := range shoot.Spec.Provider.Workers { - poolNames.Insert(pool.Name) } } - - for poolName := range poolNames { - machineDeploymentList := &machinev1alpha1.MachineDeploymentList{} - if err := r.SeedClientSet.Client().List(ctx, machineDeploymentList, client.InNamespace(shoot.Status.TechnicalID), client.MatchingLabels{v1beta1constants.LabelWorkerPool: poolName}); err != nil { - return fmt.Errorf("failed to list MachineDeployments for pool %s in namespace %s: %w", poolName, shoot.Status.TechnicalID, err) + if strings.HasPrefix(operation, v1beta1constants.OperationRolloutWorkers) { + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + poolNames := sets.NewString(strings.Split(strings.TrimPrefix(operation, v1beta1constants.OperationRolloutWorkers+"="), ",")...) + + if poolNames.Has("*") { + poolNames = sets.NewString() + for _, pool := range shoot.Spec.Provider.Workers { + poolNames.Insert(pool.Name) + } } - if len(machineDeploymentList.Items) == 0 { - return fmt.Errorf("no MachineDeployment found for worker pool %s in namespace %s", poolName, shoot.Status.TechnicalID) - } + for poolName := range poolNames { + machineDeploymentList := &machinev1alpha1.MachineDeploymentList{} + if err := r.SeedClientSet.Client().List(ctx, machineDeploymentList, client.InNamespace(shoot.Status.TechnicalID), client.MatchingLabels{v1beta1constants.LabelWorkerPool: poolName}); err != nil { + return fmt.Errorf("failed to list MachineDeployments for pool %s in namespace %s: %w", poolName, shoot.Status.TechnicalID, err) + } - if len(machineDeploymentList.Items) > 1 { - return fmt.Errorf("multiple MachineDeployments found for worker pool %s in namespace %s", poolName, shoot.Status.TechnicalID) - } + if len(machineDeploymentList.Items) == 0 { + return fmt.Errorf("no MachineDeployment found for worker pool %s in namespace %s", poolName, shoot.Status.TechnicalID) + } + + if len(machineDeploymentList.Items) > 1 { + return fmt.Errorf("multiple MachineDeployments found for worker pool %s in namespace %s", poolName, shoot.Status.TechnicalID) + } - machineDeployment := &machineDeploymentList.Items[0] + machineDeployment := &machineDeploymentList.Items[0] - patch := client.MergeFrom(machineDeployment.DeepCopy()) - metav1.SetMetaDataAnnotation(&machineDeployment.Spec.Template.ObjectMeta, v1beta1constants.OperationRolloutWorkers, now.String()) - if err := r.SeedClientSet.Client().Patch(ctx, machineDeployment, patch); err != nil { - return fmt.Errorf("failed to annotate MachineDeployment %s: %w", client.ObjectKeyFromObject(machineDeployment), err) + patch := client.MergeFrom(machineDeployment.DeepCopy()) + metav1.SetMetaDataAnnotation(&machineDeployment.Spec.Template.ObjectMeta, v1beta1constants.OperationRolloutWorkers, now.String()) + if err := r.SeedClientSet.Client().Patch(ctx, machineDeployment, patch); err != nil { + return fmt.Errorf("failed to annotate MachineDeployment %s: %w", client.ObjectKeyFromObject(machineDeployment), err) + } } - } - v1beta1helper.MutateShootWorkerPoolRollout(shoot, func(rollout *gardencorev1beta1.ManualWorkerPoolRollout) { - workerRolloutInitiationTime := &now + v1beta1helper.MutateShootWorkerPoolRollout(shoot, func(rollout *gardencorev1beta1.ManualWorkerPoolRollout) { + workerRolloutInitiationTime := &now - var pendingWorkersRollouts []gardencorev1beta1.PendingWorkersRollout - for worker := range poolNames { - pendingWorkersRollouts = append(pendingWorkersRollouts, gardencorev1beta1.PendingWorkersRollout{ - Name: worker, - LastInitiationTime: workerRolloutInitiationTime, - }) - } + var pendingWorkersRollouts []gardencorev1beta1.PendingWorkersRollout + for worker := range poolNames { + pendingWorkersRollouts = append(pendingWorkersRollouts, gardencorev1beta1.PendingWorkersRollout{ + Name: worker, + LastInitiationTime: workerRolloutInitiationTime, + }) + } - rollout.PendingWorkersRollouts = append(rollout.PendingWorkersRollouts, pendingWorkersRollouts...) - }) + rollout.PendingWorkersRollouts = append(rollout.PendingWorkersRollouts, pendingWorkersRollouts...) + }) + } } removeNonExistentPoolsFromPendingWorkersRollouts(shoot, v1beta1helper.HibernationIsEnabled(shoot)) @@ -783,9 +784,13 @@ func (r *Reconciler) updateShootStatusOperationStart( return err } - if mustRemoveOperationAnnotation { + if len(operations) != len(updatedOperations) { patch := client.MergeFrom(shoot.DeepCopy()) - delete(shoot.Annotations, v1beta1constants.GardenerOperation) + if len(updatedOperations) == 0 { + delete(shoot.Annotations, v1beta1constants.GardenerOperation) + } else { + shoot.Annotations[v1beta1constants.GardenerOperation] = strings.Join(updatedOperations, v1beta1constants.GardenerOperationsSeparator) + } return r.GardenClient.Patch(ctx, shoot, patch) } diff --git a/pkg/operator/controller/garden/garden/add.go b/pkg/operator/controller/garden/garden/add.go index d6ab9e38248..210da65fa47 100644 --- a/pkg/operator/controller/garden/garden/add.go +++ b/pkg/operator/controller/garden/garden/add.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/Masterminds/semver/v3" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -18,6 +19,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" + "github.com/gardener/gardener/pkg/apis/operator/v1alpha1/helper" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/client/kubernetes/clientmap" ) @@ -80,7 +82,8 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, gardenClientMap clientmap // HasOperationAnnotation returns a predicate which returns true when the object has an operation annotation. func (r *Reconciler) HasOperationAnnotation() predicate.Predicate { hasOperationAnnotation := func(annotations map[string]string) bool { - return operatorv1alpha1.AvailableOperationAnnotations.Has(annotations[v1beta1constants.GardenerOperation]) + operations := helper.GetGardenerOperations(annotations) + return operatorv1alpha1.AvailableOperationAnnotations.HasAny(operations...) } return predicate.Funcs{ @@ -88,7 +91,9 @@ func (r *Reconciler) HasOperationAnnotation() predicate.Predicate { return hasOperationAnnotation(e.Object.GetAnnotations()) }, UpdateFunc: func(e event.UpdateEvent) bool { - return !hasOperationAnnotation(e.ObjectOld.GetAnnotations()) && hasOperationAnnotation(e.ObjectNew.GetAnnotations()) + return (!hasOperationAnnotation(e.ObjectOld.GetAnnotations()) && hasOperationAnnotation(e.ObjectNew.GetAnnotations())) || + (hasOperationAnnotation(e.ObjectOld.GetAnnotations()) && hasOperationAnnotation(e.ObjectNew.GetAnnotations()) && + !sets.New(helper.GetGardenerOperations(e.ObjectOld.GetAnnotations())...).Equal(sets.New(helper.GetGardenerOperations(e.ObjectNew.GetAnnotations())...))) }, DeleteFunc: func(_ event.DeleteEvent) bool { return false }, GenericFunc: func(_ event.GenericEvent) bool { return false }, diff --git a/pkg/operator/controller/garden/garden/add_test.go b/pkg/operator/controller/garden/garden/add_test.go index aeff8f579ad..12a218455d0 100644 --- a/pkg/operator/controller/garden/garden/add_test.go +++ b/pkg/operator/controller/garden/garden/add_test.go @@ -68,6 +68,31 @@ var _ = Describe("Add", func() { Expect(p.Update(event.UpdateEvent{ObjectOld: gardenOld, ObjectNew: garden})).To(BeFalse()) }) + It("should return true when different operation annotation present on old and new object", func() { + gardenOld := garden.DeepCopy() + metav1.SetMetaDataAnnotation(&gardenOld.ObjectMeta, "gardener.cloud/operation", "reconcile") + metav1.SetMetaDataAnnotation(&garden.ObjectMeta, "gardener.cloud/operation", "rotate-credentials-start") + + Expect(p.Update(event.UpdateEvent{ObjectOld: gardenOld, ObjectNew: garden})).To(BeTrue()) + }) + + It("should return true when parallel operations differ in old and new object", func() { + gardenOld := garden.DeepCopy() + metav1.SetMetaDataAnnotation(&gardenOld.ObjectMeta, "gardener.cloud/operation", "rotate-etcd-encryption-key;rotate-ssh-keypair") + metav1.SetMetaDataAnnotation(&garden.ObjectMeta, "gardener.cloud/operation", "rotate-etcd-encryption-key;rotate-ssh-keypair;rotate-ca-start") + + Expect(p.Update(event.UpdateEvent{ObjectOld: gardenOld, ObjectNew: garden})).To(BeTrue()) + }) + + It("should return false when parallel operations differ only by order in old and new object", func() { + gardenOld := garden.DeepCopy() + metav1.SetMetaDataAnnotation(&gardenOld.ObjectMeta, "gardener.cloud/operation", "rotate-etcd-encryption-key;rotate-ssh-keypair") + metav1.SetMetaDataAnnotation(&garden.ObjectMeta, "gardener.cloud/operation", "rotate-ssh-keypair;rotate-etcd-encryption-key") + metav1.SetMetaDataAnnotation(&garden.ObjectMeta, "foo", "bar") + + Expect(p.Update(event.UpdateEvent{ObjectOld: gardenOld, ObjectNew: garden})).To(BeFalse()) + }) + DescribeTable("operation annotation present only on new object", func(operation string, matcher gomegatypes.GomegaMatcher) { gardenOld := garden.DeepCopy() diff --git a/pkg/operator/controller/garden/garden/reconciler.go b/pkg/operator/controller/garden/garden/reconciler.go index 1bcf7bd221d..5a80793b92d 100644 --- a/pkg/operator/controller/garden/garden/reconciler.go +++ b/pkg/operator/controller/garden/garden/reconciler.go @@ -8,6 +8,8 @@ import ( "context" "fmt" "net" + "slices" + "strings" "time" "github.com/Masterminds/semver/v3" @@ -15,6 +17,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "k8s.io/utils/clock" "k8s.io/utils/ptr" @@ -172,9 +175,10 @@ func (r *Reconciler) reportProgress(log logr.Logger, garden *operatorv1alpha1.Ga func (r *Reconciler) updateStatusOperationStart(ctx context.Context, garden *operatorv1alpha1.Garden, operationType gardencorev1beta1.LastOperationType) error { var ( - now = metav1.NewTime(r.Clock.Now().UTC()) - description string - mustRemoveOperationAnnotation bool + now = metav1.NewTime(r.Clock.Now().UTC()) + operations = helper.GetGardenerOperations(garden.Annotations) + filteredOperations = sets.New(operations...).UnsortedList() + description string ) k8sLess134, err := versionutils.CompareVersions(garden.Spec.VirtualCluster.Kubernetes.Version, "<", "1.34") @@ -199,60 +203,82 @@ func (r *Reconciler) updateStatusOperationStart(ctx context.Context, garden *ope garden.Status.Gardener = r.Identity garden.Status.ObservedGeneration = garden.Generation - switch garden.Annotations[v1beta1constants.GardenerOperation] { - case v1beta1constants.GardenerOperationReconcile: - mustRemoveOperationAnnotation = true - - case v1beta1constants.OperationRotateCredentialsStart: - mustRemoveOperationAnnotation = true - startRotationCA(garden, &now) - startRotationServiceAccountKey(garden, &now) - startRotationETCDEncryptionKey(garden, !k8sLess134, &now) - startRotationObservability(garden, &now) - startRotationWorkloadIdentityKey(garden, &now) - case v1beta1constants.OperationRotateCredentialsComplete: - mustRemoveOperationAnnotation = true - completeRotationCA(garden, &now) - completeRotationServiceAccountKey(garden, &now) - if k8sLess134 { - completeRotationETCDEncryptionKey(garden, &now) + for _, operation := range operations { + switch operation { + case v1beta1constants.OperationRotateCredentialsStart: + filteredOperations = v1beta1helper.RemoveOperation(filteredOperations, v1beta1constants.OperationRotateCAStart, + v1beta1constants.OperationRotateServiceAccountKeyStart, + v1beta1constants.OperationRotateETCDEncryptionKey, + v1beta1constants.OperationRotateETCDEncryptionKeyStart, + v1beta1constants.OperationRotateObservabilityCredentials, + v1beta1constants.ShootOperationRotateSSHKeypair, + ) + case v1beta1constants.OperationRotateCredentialsComplete: + filteredOperations = v1beta1helper.RemoveOperation(filteredOperations, v1beta1constants.OperationRotateCAComplete, + v1beta1constants.OperationRotateServiceAccountKeyComplete, + v1beta1constants.OperationRotateETCDEncryptionKeyComplete, + ) } - completeRotationWorkloadIdentityKey(garden, &now) - - case v1beta1constants.OperationRotateCAStart: - mustRemoveOperationAnnotation = true - startRotationCA(garden, &now) - case v1beta1constants.OperationRotateCAComplete: - mustRemoveOperationAnnotation = true - completeRotationCA(garden, &now) - - case v1beta1constants.OperationRotateServiceAccountKeyStart: - mustRemoveOperationAnnotation = true - startRotationServiceAccountKey(garden, &now) - case v1beta1constants.OperationRotateServiceAccountKeyComplete: - mustRemoveOperationAnnotation = true - completeRotationServiceAccountKey(garden, &now) - - case v1beta1constants.OperationRotateETCDEncryptionKey: - mustRemoveOperationAnnotation = true - startRotationETCDEncryptionKey(garden, true, &now) - case v1beta1constants.OperationRotateETCDEncryptionKeyStart: - mustRemoveOperationAnnotation = true - startRotationETCDEncryptionKey(garden, false, &now) - case v1beta1constants.OperationRotateETCDEncryptionKeyComplete: - mustRemoveOperationAnnotation = true - completeRotationETCDEncryptionKey(garden, &now) + } + + updatedOperations := slices.Clone(filteredOperations) + + for _, operation := range filteredOperations { + switch operation { + case v1beta1constants.GardenerOperationReconcile: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + + case v1beta1constants.OperationRotateCredentialsStart: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationCA(garden, &now) + startRotationServiceAccountKey(garden, &now) + startRotationETCDEncryptionKey(garden, !k8sLess134, &now) + startRotationObservability(garden, &now) + startRotationWorkloadIdentityKey(garden, &now) + case v1beta1constants.OperationRotateCredentialsComplete: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + completeRotationCA(garden, &now) + completeRotationServiceAccountKey(garden, &now) + if k8sLess134 { + completeRotationETCDEncryptionKey(garden, &now) + } + completeRotationWorkloadIdentityKey(garden, &now) + + case v1beta1constants.OperationRotateCAStart: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationCA(garden, &now) + case v1beta1constants.OperationRotateCAComplete: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + completeRotationCA(garden, &now) + + case v1beta1constants.OperationRotateServiceAccountKeyStart: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationServiceAccountKey(garden, &now) + case v1beta1constants.OperationRotateServiceAccountKeyComplete: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + completeRotationServiceAccountKey(garden, &now) + + case v1beta1constants.OperationRotateETCDEncryptionKey: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationETCDEncryptionKey(garden, true, &now) + case v1beta1constants.OperationRotateETCDEncryptionKeyStart: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationETCDEncryptionKey(garden, false, &now) + case v1beta1constants.OperationRotateETCDEncryptionKeyComplete: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + completeRotationETCDEncryptionKey(garden, &now) - case v1beta1constants.OperationRotateObservabilityCredentials: - mustRemoveOperationAnnotation = true - startRotationObservability(garden, &now) + case v1beta1constants.OperationRotateObservabilityCredentials: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationObservability(garden, &now) - case operatorv1alpha1.OperationRotateWorkloadIdentityKeyStart: - mustRemoveOperationAnnotation = true - startRotationWorkloadIdentityKey(garden, &now) - case operatorv1alpha1.OperationRotateWorkloadIdentityKeyComplete: - mustRemoveOperationAnnotation = true - completeRotationWorkloadIdentityKey(garden, &now) + case operatorv1alpha1.OperationRotateWorkloadIdentityKeyStart: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + startRotationWorkloadIdentityKey(garden, &now) + case operatorv1alpha1.OperationRotateWorkloadIdentityKeyComplete: + updatedOperations = v1beta1helper.RemoveOperation(updatedOperations, operation) + completeRotationWorkloadIdentityKey(garden, &now) + } } // TODO(AleksandarSavchev): Remove the k8s version check in a future release after support for Kubernetes v1.33 is dropped. @@ -267,9 +293,13 @@ func (r *Reconciler) updateStatusOperationStart(ctx context.Context, garden *ope return err } - if mustRemoveOperationAnnotation { + if len(operations) != len(updatedOperations) { patch := client.MergeFrom(garden.DeepCopy()) - delete(garden.Annotations, v1beta1constants.GardenerOperation) + if len(updatedOperations) == 0 { + delete(garden.Annotations, v1beta1constants.GardenerOperation) + } else { + garden.Annotations[v1beta1constants.GardenerOperation] = strings.Join(updatedOperations, ";") + } return r.RuntimeClientSet.Client().Patch(ctx, garden, patch) } diff --git a/pkg/utils/miscellaneous.go b/pkg/utils/miscellaneous.go index e5879a095bc..f9828b76846 100644 --- a/pkg/utils/miscellaneous.go +++ b/pkg/utils/miscellaneous.go @@ -201,3 +201,17 @@ func ComputeOffsetIP(subnet *net.IPNet, offset int64) (net.IP, error) { return nil, fmt.Errorf("computed IPv4 address %q is broadcast for subnet %q", ip, subnet) } + +// SplitAndTrimString returns a new slice from a string separated by the given separator with all empty entries removed. +func SplitAndTrimString(s, sep string) []string { + if len(s) == 0 { + return nil + } + + result := strings.Split(s, sep) + for i := range result { + result[i] = strings.TrimSpace(result[i]) + } + + return result +} diff --git a/pkg/utils/miscellaneous_test.go b/pkg/utils/miscellaneous_test.go index 67275f536d8..a67ff137540 100644 --- a/pkg/utils/miscellaneous_test.go +++ b/pkg/utils/miscellaneous_test.go @@ -341,4 +341,14 @@ baz`, spaces)).To(Equal(`foo })) }) }) + + DescribeTable("#SplitAndTrimString", + func(str, sep string, expectedResult []string) { + Expect(SplitAndTrimString(str, sep)).To(Equal(expectedResult)) + }, + Entry("string is empty", "", ";", nil), + Entry("string should be separated", "foo,bar", ",", []string{"foo", "bar"}), + Entry("string should not be separated", "foo,bar", ";", []string{"foo,bar"}), + Entry("string should be trimmed", " foo ,bar ", ",", []string{"foo", "bar"}), + ) }) diff --git a/plugin/pkg/shoot/mutator/admission.go b/plugin/pkg/shoot/mutator/admission.go index 7307d15a967..223796490a9 100644 --- a/plugin/pkg/shoot/mutator/admission.go +++ b/plugin/pkg/shoot/mutator/admission.go @@ -251,7 +251,7 @@ func (c *mutationContext) addMetadataAnnotations(a admission.Attributes) { v1beta1constants.ShootOperationRotateSSHKeypair, v1beta1constants.OperationRotateCredentialsStart, v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout, - ).Has(c.shoot.Annotations[v1beta1constants.GardenerOperation]) { + ).HasAny(v1beta1helper.GetShootGardenerOperations(c.shoot.Annotations)...) { addInfrastructureDeploymentTask(c.shoot) } From ff4a6c4ea1b721a2428fc3a9c56c6ea0aa4df3ac Mon Sep 17 00:00:00 2001 From: Dimitar Mirchev Date: Tue, 18 Nov 2025 16:40:47 +0200 Subject: [PATCH 133/176] Refactor extension health checker types (#13329) * Refactor extension health checker types * Address review feedback --- .../pkg/controller/healthcheck/actuator.go | 2 - .../healthcheck/general/daemonset.go | 91 +++++++++---------- .../healthcheck/general/deployment.go | 86 +++++++++--------- .../healthcheck/general/managed_resource.go | 14 ++- .../healthcheck/general/statefulsets.go | 90 +++++++++--------- .../healthcheck/healthcheck_actuator.go | 3 +- .../controller/healthcheck/worker/nodes.go | 13 ++- 7 files changed, 141 insertions(+), 158 deletions(-) diff --git a/extensions/pkg/controller/healthcheck/actuator.go b/extensions/pkg/controller/healthcheck/actuator.go index abf9886c611..d8ccc637df6 100644 --- a/extensions/pkg/controller/healthcheck/actuator.go +++ b/extensions/pkg/controller/healthcheck/actuator.go @@ -115,8 +115,6 @@ type HealthCheck interface { Check(context.Context, types.NamespacedName) (*SingleCheckResult, error) // SetLoggerSuffix injects the logger SetLoggerSuffix(string, string) - // DeepCopy clones the healthCheck - DeepCopy() HealthCheck } // SingleCheckResult is the result for a health check diff --git a/extensions/pkg/controller/healthcheck/general/daemonset.go b/extensions/pkg/controller/healthcheck/general/daemonset.go index 6b8af12bacd..eee42d812ca 100644 --- a/extensions/pkg/controller/healthcheck/general/daemonset.go +++ b/extensions/pkg/controller/healthcheck/general/daemonset.go @@ -20,84 +20,81 @@ import ( "github.com/gardener/gardener/pkg/utils/kubernetes/health" ) -// DaemonSetHealthChecker contains all the information for the DaemonSet HealthCheck -type DaemonSetHealthChecker struct { - logger logr.Logger - seedClient client.Client - shootClient client.Client - name string - checkType DaemonSetCheckType +// daemonSetHealthChecker contains all the information for the DaemonSet HealthCheck +type daemonSetHealthChecker struct { + logger logr.Logger + client client.Client + name string } -// DaemonSetCheckType in which cluster the check will be executed -type DaemonSetCheckType string +// SeedDaemonSetHealthChecker is a healthCheck for DaemonSets in the Seed cluster +type SeedDaemonSetHealthChecker struct { + daemonSetHealthChecker +} + +// ShootDaemonSetHealthChecker is a healthCheck for DaemonSets in the Shoot cluster +type ShootDaemonSetHealthChecker struct { + daemonSetHealthChecker +} -const ( - daemonSetCheckTypeSeed DaemonSetCheckType = "Seed" - daemonSetCheckTypeShoot DaemonSetCheckType = "Shoot" +var ( + _ healthcheck.HealthCheck = (*SeedDaemonSetHealthChecker)(nil) + _ healthcheck.SeedClient = (*SeedDaemonSetHealthChecker)(nil) + _ healthcheck.HealthCheck = (*ShootDaemonSetHealthChecker)(nil) + _ healthcheck.ShootClient = (*ShootDaemonSetHealthChecker)(nil) ) -// NewSeedDaemonSetHealthChecker is a healthCheck function to check DaemonSets -func NewSeedDaemonSetHealthChecker(name string) healthcheck.HealthCheck { - return &DaemonSetHealthChecker{ - name: name, - checkType: daemonSetCheckTypeSeed, +// NewSeedDaemonSetHealthChecker is a healthCheck function to check DaemonSets in the Seed cluster +func NewSeedDaemonSetHealthChecker(name string) *SeedDaemonSetHealthChecker { + return &SeedDaemonSetHealthChecker{ + daemonSetHealthChecker: daemonSetHealthChecker{ + name: name, + }, } } -// NewShootDaemonSetHealthChecker is a healthCheck function to check DaemonSets -func NewShootDaemonSetHealthChecker(name string) healthcheck.HealthCheck { - return &DaemonSetHealthChecker{ - name: name, - checkType: daemonSetCheckTypeShoot, +// NewShootDaemonSetHealthChecker is a healthCheck function to check DaemonSets in the Shoot cluster +func NewShootDaemonSetHealthChecker(name string) *ShootDaemonSetHealthChecker { + return &ShootDaemonSetHealthChecker{ + daemonSetHealthChecker: daemonSetHealthChecker{ + name: name, + }, } } // InjectSeedClient injects the seed client -func (healthChecker *DaemonSetHealthChecker) InjectSeedClient(seedClient client.Client) { - healthChecker.seedClient = seedClient +func (h *SeedDaemonSetHealthChecker) InjectSeedClient(seedClient client.Client) { + h.client = seedClient } // InjectShootClient injects the shoot client -func (healthChecker *DaemonSetHealthChecker) InjectShootClient(shootClient client.Client) { - healthChecker.shootClient = shootClient +func (h *ShootDaemonSetHealthChecker) InjectShootClient(shootClient client.Client) { + h.client = shootClient } // SetLoggerSuffix injects the logger -func (healthChecker *DaemonSetHealthChecker) SetLoggerSuffix(provider, extension string) { - healthChecker.logger = log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-deployment", provider, extension)) -} - -// DeepCopy clones the healthCheck struct by making a copy and returning the pointer to that new copy -// Actually, it does not perform a *deep* copy. -func (healthChecker *DaemonSetHealthChecker) DeepCopy() healthcheck.HealthCheck { - shallowCopy := *healthChecker - return &shallowCopy +func (h *daemonSetHealthChecker) SetLoggerSuffix(provider, extension string) { + h.logger = log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-daemonset", provider, extension)) } // Check executes the health check -func (healthChecker *DaemonSetHealthChecker) Check(ctx context.Context, request types.NamespacedName) (*healthcheck.SingleCheckResult, error) { +func (h *daemonSetHealthChecker) Check(ctx context.Context, request types.NamespacedName) (*healthcheck.SingleCheckResult, error) { daemonSet := &appsv1.DaemonSet{} - var err error - if healthChecker.checkType == daemonSetCheckTypeSeed { - err = healthChecker.seedClient.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: healthChecker.name}, daemonSet) - } else { - err = healthChecker.shootClient.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: healthChecker.name}, daemonSet) - } - if err != nil { + + if err := h.client.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: h.name}, daemonSet); err != nil { if apierrors.IsNotFound(err) { return &healthcheck.SingleCheckResult{ Status: gardencorev1beta1.ConditionFalse, - Detail: fmt.Sprintf("DaemonSet %q in namespace %q not found", healthChecker.name, request.Namespace), + Detail: fmt.Sprintf("DaemonSet %q in namespace %q not found", h.name, request.Namespace), }, nil } - err := fmt.Errorf("failed to retrieve DaemonSet %q in namespace %q: %w", healthChecker.name, request.Namespace, err) - healthChecker.logger.Error(err, "Health check failed") + err := fmt.Errorf("failed to retrieve DaemonSet %q in namespace %q: %w", h.name, request.Namespace, err) + h.logger.Error(err, "Health check failed") return nil, err } if isHealthy, err := DaemonSetIsHealthy(daemonSet); !isHealthy { - healthChecker.logger.Error(err, "Health check failed") + h.logger.Error(err, "Health check failed") return &healthcheck.SingleCheckResult{ Status: gardencorev1beta1.ConditionFalse, Detail: err.Error(), diff --git a/extensions/pkg/controller/healthcheck/general/deployment.go b/extensions/pkg/controller/healthcheck/general/deployment.go index 71db5ddb73c..0e9fc40f716 100644 --- a/extensions/pkg/controller/healthcheck/general/deployment.go +++ b/extensions/pkg/controller/healthcheck/general/deployment.go @@ -20,86 +20,82 @@ import ( "github.com/gardener/gardener/pkg/utils/kubernetes/health" ) -// DeploymentHealthChecker contains all the information for the Deployment HealthCheck -type DeploymentHealthChecker struct { - logger logr.Logger - seedClient client.Client - shootClient client.Client - name string - checkType DeploymentCheckType +// deploymentHealthChecker contains all the information for the Deployment HealthCheck +type deploymentHealthChecker struct { + logger logr.Logger + client client.Client + name string } -// DeploymentCheckType in which cluster the check will be executed -type DeploymentCheckType string +// SeedDeploymentHealthChecker is a healthCheck for Deployments in the Seed cluster +type SeedDeploymentHealthChecker struct { + deploymentHealthChecker +} + +// ShootDeploymentHealthChecker is a healthCheck for Deployments in the Shoot cluster +type ShootDeploymentHealthChecker struct { + deploymentHealthChecker +} -const ( - deploymentCheckTypeSeed DeploymentCheckType = "Seed" - deploymentCheckTypeShoot DeploymentCheckType = "Shoot" +var ( + _ healthcheck.HealthCheck = (*SeedDeploymentHealthChecker)(nil) + _ healthcheck.SeedClient = (*SeedDeploymentHealthChecker)(nil) + _ healthcheck.HealthCheck = (*ShootDeploymentHealthChecker)(nil) + _ healthcheck.ShootClient = (*ShootDeploymentHealthChecker)(nil) ) // NewSeedDeploymentHealthChecker is a healthCheck function to check Deployments in the Seed cluster -func NewSeedDeploymentHealthChecker(deploymentName string) healthcheck.HealthCheck { - return &DeploymentHealthChecker{ - name: deploymentName, - checkType: deploymentCheckTypeSeed, +func NewSeedDeploymentHealthChecker(deploymentName string) *SeedDeploymentHealthChecker { + return &SeedDeploymentHealthChecker{ + deploymentHealthChecker: deploymentHealthChecker{ + name: deploymentName, + }, } } // NewShootDeploymentHealthChecker is a healthCheck function to check Deployments in the Shoot cluster -func NewShootDeploymentHealthChecker(deploymentName string) healthcheck.HealthCheck { - return &DeploymentHealthChecker{ - name: deploymentName, - checkType: deploymentCheckTypeShoot, +func NewShootDeploymentHealthChecker(deploymentName string) *ShootDeploymentHealthChecker { + return &ShootDeploymentHealthChecker{ + deploymentHealthChecker: deploymentHealthChecker{ + name: deploymentName, + }, } } // InjectSeedClient injects the seed client -func (healthChecker *DeploymentHealthChecker) InjectSeedClient(seedClient client.Client) { - healthChecker.seedClient = seedClient +func (h *SeedDeploymentHealthChecker) InjectSeedClient(seedClient client.Client) { + h.client = seedClient } // InjectShootClient injects the shoot client -func (healthChecker *DeploymentHealthChecker) InjectShootClient(shootClient client.Client) { - healthChecker.shootClient = shootClient +func (h *ShootDeploymentHealthChecker) InjectShootClient(shootClient client.Client) { + h.client = shootClient } // SetLoggerSuffix injects the logger -func (healthChecker *DeploymentHealthChecker) SetLoggerSuffix(provider, extension string) { - healthChecker.logger = log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-deployment", provider, extension)) -} - -// DeepCopy clones the healthCheck struct by making a copy and returning the pointer to that new copy -// Actually, it does not perform a *deep* copy. -func (healthChecker *DeploymentHealthChecker) DeepCopy() healthcheck.HealthCheck { - shallowCopy := *healthChecker - return &shallowCopy +func (h *deploymentHealthChecker) SetLoggerSuffix(provider, extension string) { + h.logger = log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-deployment", provider, extension)) } // Check executes the health check -func (healthChecker *DeploymentHealthChecker) Check(ctx context.Context, request types.NamespacedName) (*healthcheck.SingleCheckResult, error) { +func (h *deploymentHealthChecker) Check(ctx context.Context, request types.NamespacedName) (*healthcheck.SingleCheckResult, error) { deployment := &appsv1.Deployment{} - var err error - if healthChecker.checkType == deploymentCheckTypeSeed { - err = healthChecker.seedClient.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: healthChecker.name}, deployment) - } else { - err = healthChecker.shootClient.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: healthChecker.name}, deployment) - } - if err != nil { + if err := h.client.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: h.name}, deployment); err != nil { if apierrors.IsNotFound(err) { return &healthcheck.SingleCheckResult{ Status: gardencorev1beta1.ConditionFalse, - Detail: fmt.Sprintf("deployment %q in namespace %q not found", healthChecker.name, request.Namespace), + Detail: fmt.Sprintf("deployment %q in namespace %q not found", h.name, request.Namespace), }, nil } - err := fmt.Errorf("failed to retrieve deployment %q in namespace %q: %w", healthChecker.name, request.Namespace, err) - healthChecker.logger.Error(err, "Health check failed") + err := fmt.Errorf("failed to retrieve deployment %q in namespace %q: %w", h.name, request.Namespace, err) + h.logger.Error(err, "Health check failed") return nil, err } if isHealthy, err := deploymentIsHealthy(deployment); !isHealthy { - healthChecker.logger.Error(err, "Health check failed") + h.logger.Error(err, "Health check failed") return &healthcheck.SingleCheckResult{ Status: gardencorev1beta1.ConditionFalse, Detail: err.Error(), diff --git a/extensions/pkg/controller/healthcheck/general/managed_resource.go b/extensions/pkg/controller/healthcheck/general/managed_resource.go index b5c2fdcb7f0..893899edd18 100644 --- a/extensions/pkg/controller/healthcheck/general/managed_resource.go +++ b/extensions/pkg/controller/healthcheck/general/managed_resource.go @@ -28,8 +28,13 @@ type ManagedResourceHealthChecker struct { managedResourceName string } +var ( + _ healthcheck.HealthCheck = (*ManagedResourceHealthChecker)(nil) + _ healthcheck.SeedClient = (*ManagedResourceHealthChecker)(nil) +) + // CheckManagedResource is a healthCheck function to check ManagedResources -func CheckManagedResource(managedResourceName string) healthcheck.HealthCheck { +func CheckManagedResource(managedResourceName string) *ManagedResourceHealthChecker { return &ManagedResourceHealthChecker{ managedResourceName: managedResourceName, } @@ -45,13 +50,6 @@ func (healthChecker *ManagedResourceHealthChecker) SetLoggerSuffix(provider, ext healthChecker.logger = log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-managed-resource", provider, extension)) } -// DeepCopy clones the healthCheck struct by making a copy and returning the pointer to that new copy -// Actually, it does not perform a *deep* copy. -func (healthChecker *ManagedResourceHealthChecker) DeepCopy() healthcheck.HealthCheck { - shallowCopy := *healthChecker - return &shallowCopy -} - // configurationProblemRegex is used to check if a not healthy managed resource has a configuration problem. var configurationProblemRegex = regexp.MustCompile(`(?i)(error during apply of object .* is invalid:)`) diff --git a/extensions/pkg/controller/healthcheck/general/statefulsets.go b/extensions/pkg/controller/healthcheck/general/statefulsets.go index d14ba93032e..30532276f52 100644 --- a/extensions/pkg/controller/healthcheck/general/statefulsets.go +++ b/extensions/pkg/controller/healthcheck/general/statefulsets.go @@ -20,84 +20,80 @@ import ( "github.com/gardener/gardener/pkg/utils/kubernetes/health" ) -// StatefulSetHealthChecker contains all the information for the StatefulSet HealthCheck -type StatefulSetHealthChecker struct { - logger logr.Logger - seedClient client.Client - shootClient client.Client - name string - checkType StatefulSetCheckType +// statefulSetHealthChecker contains all the information for the StatefulSet HealthCheck +type statefulSetHealthChecker struct { + logger logr.Logger + client client.Client + name string } -// StatefulSetCheckType in which cluster the check will be executed -type StatefulSetCheckType string +// SeedStatefulSetHealthChecker is a healthCheck for StatefulSets in the Seed cluster +type SeedStatefulSetHealthChecker struct { + statefulSetHealthChecker +} + +// ShootStatefulSetHealthChecker is a healthCheck for StatefulSets in the Shoot cluster +type ShootStatefulSetHealthChecker struct { + statefulSetHealthChecker +} -const ( - statefulSetCheckTypeSeed StatefulSetCheckType = "Seed" - statefulSetCheckTypeShoot StatefulSetCheckType = "Shoot" +var ( + _ healthcheck.HealthCheck = (*SeedStatefulSetHealthChecker)(nil) + _ healthcheck.SeedClient = (*SeedStatefulSetHealthChecker)(nil) + _ healthcheck.HealthCheck = (*ShootStatefulSetHealthChecker)(nil) + _ healthcheck.ShootClient = (*ShootStatefulSetHealthChecker)(nil) ) -// NewSeedStatefulSetChecker is a healthCheck function to check StatefulSets -func NewSeedStatefulSetChecker(name string) healthcheck.HealthCheck { - return &StatefulSetHealthChecker{ - name: name, - checkType: statefulSetCheckTypeSeed, +// NewSeedStatefulSetChecker is a healthCheck function to check StatefulSets in the Seed cluster +func NewSeedStatefulSetChecker(name string) *SeedStatefulSetHealthChecker { + return &SeedStatefulSetHealthChecker{ + statefulSetHealthChecker: statefulSetHealthChecker{ + name: name, + }, } } -// NewShootStatefulSetChecker is a healthCheck function to check StatefulSets -func NewShootStatefulSetChecker(name string) healthcheck.HealthCheck { - return &StatefulSetHealthChecker{ - name: name, - checkType: statefulSetCheckTypeShoot, +// NewShootStatefulSetChecker is a healthCheck function to check StatefulSets in the Shoot cluster +func NewShootStatefulSetChecker(name string) *ShootStatefulSetHealthChecker { + return &ShootStatefulSetHealthChecker{ + statefulSetHealthChecker: statefulSetHealthChecker{ + name: name, + }, } } // InjectSeedClient injects the seed client -func (healthChecker *StatefulSetHealthChecker) InjectSeedClient(seedClient client.Client) { - healthChecker.seedClient = seedClient +func (h *SeedStatefulSetHealthChecker) InjectSeedClient(seedClient client.Client) { + h.client = seedClient } // InjectShootClient injects the shoot client -func (healthChecker *StatefulSetHealthChecker) InjectShootClient(shootClient client.Client) { - healthChecker.shootClient = shootClient +func (h *ShootStatefulSetHealthChecker) InjectShootClient(shootClient client.Client) { + h.client = shootClient } // SetLoggerSuffix injects the logger -func (healthChecker *StatefulSetHealthChecker) SetLoggerSuffix(provider, extension string) { - healthChecker.logger = log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-deployment", provider, extension)) -} - -// DeepCopy clones the healthCheck struct by making a copy and returning the pointer to that new copy -// Actually, it does not perform a *deep* copy. -func (healthChecker *StatefulSetHealthChecker) DeepCopy() healthcheck.HealthCheck { - shallowCopy := *healthChecker - return &shallowCopy +func (h *statefulSetHealthChecker) SetLoggerSuffix(provider, extension string) { + h.logger = log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-statefulset", provider, extension)) } // Check executes the health check -func (healthChecker *StatefulSetHealthChecker) Check(ctx context.Context, request types.NamespacedName) (*healthcheck.SingleCheckResult, error) { +func (h *statefulSetHealthChecker) Check(ctx context.Context, request types.NamespacedName) (*healthcheck.SingleCheckResult, error) { statefulSet := &appsv1.StatefulSet{} - var err error - if healthChecker.checkType == statefulSetCheckTypeSeed { - err = healthChecker.seedClient.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: healthChecker.name}, statefulSet) - } else { - err = healthChecker.shootClient.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: healthChecker.name}, statefulSet) - } - if err != nil { + if err := h.client.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: h.name}, statefulSet); err != nil { if apierrors.IsNotFound(err) { return &healthcheck.SingleCheckResult{ Status: gardencorev1beta1.ConditionFalse, - Detail: fmt.Sprintf("StatefulSet %q in namespace %q not found", healthChecker.name, request.Namespace), + Detail: fmt.Sprintf("StatefulSet %q in namespace %q not found", h.name, request.Namespace), }, nil } - err := fmt.Errorf("failed to retrieve StatefulSet %q in namespace %q: %w", healthChecker.name, request.Namespace, err) - healthChecker.logger.Error(err, "Health check failed") + err := fmt.Errorf("failed to retrieve StatefulSet %q in namespace %q: %w", h.name, request.Namespace, err) + h.logger.Error(err, "Health check failed") return nil, err } if isHealthy, err := statefulSetIsHealthy(statefulSet); !isHealthy { - healthChecker.logger.Error(err, "Health check failed") + h.logger.Error(err, "Health check failed") return &healthcheck.SingleCheckResult{ Status: gardencorev1beta1.ConditionFalse, Detail: err.Error(), diff --git a/extensions/pkg/controller/healthcheck/healthcheck_actuator.go b/extensions/pkg/controller/healthcheck/healthcheck_actuator.go index 4a366fc3e56..0fadd7ee6a9 100644 --- a/extensions/pkg/controller/healthcheck/healthcheck_actuator.go +++ b/extensions/pkg/controller/healthcheck/healthcheck_actuator.go @@ -83,8 +83,7 @@ func (a *Actuator) ExecuteHealthCheckFunctions(ctx context.Context, log logr.Log ) for _, hc := range a.healthChecks { - // clone to avoid problems during parallel execution - check := hc.HealthCheck.DeepCopy() + check := hc.HealthCheck SeedClientInto(a.seedClient, check) if _, ok := check.(ShootClient); ok { if shootClient == nil { diff --git a/extensions/pkg/controller/healthcheck/worker/nodes.go b/extensions/pkg/controller/healthcheck/worker/nodes.go index 34f13547f54..43e026119f7 100644 --- a/extensions/pkg/controller/healthcheck/worker/nodes.go +++ b/extensions/pkg/controller/healthcheck/worker/nodes.go @@ -42,6 +42,12 @@ type DefaultHealthChecker struct { scaleDownProgressingThreshold *time.Duration } +var ( + _ healthcheck.HealthCheck = (*DefaultHealthChecker)(nil) + _ healthcheck.SeedClient = (*DefaultHealthChecker)(nil) + _ healthcheck.ShootClient = (*DefaultHealthChecker)(nil) +) + // NewNodesChecker is a health check function which performs certain checks about the nodes registered in the cluster. // It implements the healthcheck.HealthCheck interface. func NewNodesChecker() *DefaultHealthChecker { @@ -81,13 +87,6 @@ func (h *DefaultHealthChecker) SetLoggerSuffix(provider, extension string) { h.logger = log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-nodes", provider, extension)) } -// DeepCopy clones the healthCheck struct by making a copy and returning the pointer to that new copy. -// Actually, it does not perform a *deep* copy. -func (h *DefaultHealthChecker) DeepCopy() healthcheck.HealthCheck { - shallowCopy := *h - return &shallowCopy -} - // Check executes the health check. func (h *DefaultHealthChecker) Check(ctx context.Context, request types.NamespacedName) (*healthcheck.SingleCheckResult, error) { machineDeploymentList := &machinev1alpha1.MachineDeploymentList{} From 340c15e9dc2386e2475c631121b516a34e0a14e8 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Tue, 18 Nov 2025 18:26:40 +0100 Subject: [PATCH 134/176] Update module github.com/prometheus/common to v0.67.3 (#13476) --- go.mod | 3 ++- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 4973096d607..372a07cd438 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.86.2 github.com/prometheus/blackbox_exporter v0.27.0 github.com/prometheus/client_golang v1.23.2 - github.com/prometheus/common v0.67.2 + github.com/prometheus/common v0.67.3 github.com/robfig/cron v1.2.0 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.1 @@ -150,6 +150,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-yaml v1.18.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect diff --git a/go.sum b/go.sum index 6edde6dbdab..fda9c48bf8b 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -639,8 +639,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8= -github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko= +github.com/prometheus/common v0.67.3 h1:shd26MlnwTw5jksTDhC7rTQIteBxy+ZZDr3t7F2xN2Q= +github.com/prometheus/common v0.67.3/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ= github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= From e0e21a450f1306c11dfb74e6921f55aa3c3e5eac Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Tue, 18 Nov 2025 18:26:46 +0100 Subject: [PATCH 135/176] Update dependency gardener/gardener-extension-networking-cilium to v1.45.1 (#13480) --- .../components/controllerregistration/kustomization.yaml | 2 +- example/provider-local/garden/base/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-setup/extensions/networking-cilium/components/controllerregistration/kustomization.yaml b/dev-setup/extensions/networking-cilium/components/controllerregistration/kustomization.yaml index 0a830492df1..9bab7266046 100644 --- a/dev-setup/extensions/networking-cilium/components/controllerregistration/kustomization.yaml +++ b/dev-setup/extensions/networking-cilium/components/controllerregistration/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: -- https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.0/example/controller-registration.yaml +- https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.1/example/controller-registration.yaml diff --git a/example/provider-local/garden/base/kustomization.yaml b/example/provider-local/garden/base/kustomization.yaml index aacf6926218..96c5462a8d8 100644 --- a/example/provider-local/garden/base/kustomization.yaml +++ b/example/provider-local/garden/base/kustomization.yaml @@ -14,5 +14,5 @@ components: - ../../../../dev-setup/gardenconfig/components/credentials/workloadidentity-project-local resources: -- https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.0/example/controller-registration.yaml +- https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.1/example/controller-registration.yaml - https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.53.0/example/controller-registration.yaml From eb9e6792929c7662640c7a6a98e2eab62697ed58 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Tue, 18 Nov 2025 20:23:40 +0100 Subject: [PATCH 136/176] Update dependency gardener/gardener-extension-networking-calico to v1.53.1 (#13484) --- .../components/controllerregistration/kustomization.yaml | 2 +- example/provider-local/garden/base/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-setup/extensions/networking-calico/components/controllerregistration/kustomization.yaml b/dev-setup/extensions/networking-calico/components/controllerregistration/kustomization.yaml index 18451bc8522..b45a69975b5 100644 --- a/dev-setup/extensions/networking-calico/components/controllerregistration/kustomization.yaml +++ b/dev-setup/extensions/networking-calico/components/controllerregistration/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: -- https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.53.0/example/controller-registration.yaml +- https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.53.1/example/controller-registration.yaml diff --git a/example/provider-local/garden/base/kustomization.yaml b/example/provider-local/garden/base/kustomization.yaml index 96c5462a8d8..61ffa923192 100644 --- a/example/provider-local/garden/base/kustomization.yaml +++ b/example/provider-local/garden/base/kustomization.yaml @@ -15,4 +15,4 @@ components: resources: - https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.1/example/controller-registration.yaml -- https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.53.0/example/controller-registration.yaml +- https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.53.1/example/controller-registration.yaml From 12568f0aba9e478f6625e7ff854fddec815bf066 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 19 Nov 2025 12:31:59 +0100 Subject: [PATCH 137/176] Update quay.io/cortexproject/cortex Docker tag to v1.20.0 (#13390) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 7da6ea4bb60..dec40d4d89b 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -277,7 +277,7 @@ images: teamname: 'gardener/monitoring-maintainers' - name: cortex repository: quay.io/cortexproject/cortex - tag: v1.19.1 + tag: v1.20.0 labels: - name: gardener.cloud/cve-categorisation value: From d7bb834ac93f5eb18cebf2e7ae9e86653b5e826f Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Wed, 19 Nov 2025 13:32:05 +0200 Subject: [PATCH 138/176] Update monitoring stack docs about developing dashboards to suggest UTC as default timezone (#13486) --- docs/development/monitoring-stack.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/monitoring-stack.md b/docs/development/monitoring-stack.md index 9d6d8c5a636..77e61635e5a 100644 --- a/docs/development/monitoring-stack.md +++ b/docs/development/monitoring-stack.md @@ -199,7 +199,7 @@ The following parameters should be added to all dashboards to ensure a homogeneo Dashboards have to: * contain a title which refers to the component name(s) -* contain a timezone statement which should be the browser time +* contain a timezone statement which should be UTC * contain tags which express where the component is running (`seed` or `shoot`) and to which category the component belong (see dashboard structure) * contain a version statement with a value of 1 * be immutable From 2acbc7ed74686f5063be3501e4f1893e4231815c Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 19 Nov 2025 12:32:13 +0100 Subject: [PATCH 139/176] Update dependency projectcalico/calico to v3.31.2 (#13488) --- dev-setup/kind/calico/base/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-setup/kind/calico/base/kustomization.yaml b/dev-setup/kind/calico/base/kustomization.yaml index 8f36aadf8da..3c7cdee1112 100644 --- a/dev-setup/kind/calico/base/kustomization.yaml +++ b/dev-setup/kind/calico/base/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- https://raw.githubusercontent.com/projectcalico/calico/v3.31.1/manifests/calico.yaml +- https://raw.githubusercontent.com/projectcalico/calico/v3.31.2/manifests/calico.yaml images: - name: docker.io/calico/cni From 35776748d10d7607a06126df973d558723c8540a Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Wed, 19 Nov 2025 16:57:00 +0100 Subject: [PATCH 140/176] Prefer `sshd` unit and fallback to `ssh` unit (#13456) --- docs/proposals/27-cloudprofile-bastion-section.md | 2 +- pkg/component/extensions/bastion/bastion.go | 2 +- pkg/component/extensions/bastion/bastion_test.go | 2 +- pkg/gardenlet/controller/bastion/reconciler.go | 2 +- test/integration/gardenlet/bastion/bastion_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/proposals/27-cloudprofile-bastion-section.md b/docs/proposals/27-cloudprofile-bastion-section.md index 7b125369609..808e377c9f8 100644 --- a/docs/proposals/27-cloudprofile-bastion-section.md +++ b/docs/proposals/27-cloudprofile-bastion-section.md @@ -61,7 +61,7 @@ Newer GardenLinux versions have ssh disabled by default. In order for the bastion to work with newer GardenLinux versions we need to update the UserData content that is generated by the [gardenlet bastion controller](https://github.com/gardener/gardener/blob/38143e7b9f4913d6ae11072064bb13a39b2ef17f/pkg/gardenlet/controller/bastion/reconciler.go#L269). Since all OSes in the CloudProfile are based on `systemd`, adding the following line should be enough: ```bash -systemctl start ssh +systemctl start sshd || systemctl start ssh ``` In case the ssh service is already running, this command will have no effect. diff --git a/pkg/component/extensions/bastion/bastion.go b/pkg/component/extensions/bastion/bastion.go index e86523f3708..135730a98a8 100644 --- a/pkg/component/extensions/bastion/bastion.go +++ b/pkg/component/extensions/bastion/bastion.go @@ -204,7 +204,7 @@ id %[1]s || useradd %[1]s -mU mkdir -p /home/%[1]s/.ssh echo "%[2]s" > /home/%[1]s/.ssh/authorized_keys chown %[1]s:%[1]s /home/%[1]s/.ssh/authorized_keys -systemctl start ssh +systemctl start sshd || systemctl start ssh `, bastionUser, sshPublicKey)) } diff --git a/pkg/component/extensions/bastion/bastion_test.go b/pkg/component/extensions/bastion/bastion_test.go index 618451c6fd3..5282efd6ec6 100644 --- a/pkg/component/extensions/bastion/bastion_test.go +++ b/pkg/component/extensions/bastion/bastion_test.go @@ -112,7 +112,7 @@ id gardener || useradd gardener -mU mkdir -p /home/gardener/.ssh echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJXpm5HiBLMIX/M7TrES/pEaYNSdc4cBlVyTilWFj8h3yPbOOtWUFmnWSUBEr8y3UE86ZEpC2DvndQJ7BiOlF1OHLOyJwrPgWWDibvannttaz/CL6PY3lFzR4X3xHL/8VjoZuzlUlWLPtZJ8ShdpIURgiS/4IooBD0nSSSJjO2LLP6n+5IPuwg4BWSyPgzn8P7gZW2olX7hpJ1Si2i556EnV/CZz9lOxzMxcCctxXoE/03QZfltQFb6z8dIwud0TL4ZLJ7Up2AtmKXMCh2a161B0tgI5dmyK990J4XyWwuMtX+i4Az4XDAzlBtTWL6JhGpWTwCnLOz1Yy+4CnyarlR" > /home/gardener/.ssh/authorized_keys chown gardener:gardener /home/gardener/.ssh/authorized_keys -systemctl start ssh +systemctl start sshd || systemctl start ssh `)) }) diff --git a/pkg/gardenlet/controller/bastion/reconciler.go b/pkg/gardenlet/controller/bastion/reconciler.go index f57bb932684..eb84f95285d 100644 --- a/pkg/gardenlet/controller/bastion/reconciler.go +++ b/pkg/gardenlet/controller/bastion/reconciler.go @@ -273,7 +273,7 @@ mkdir -p /home/gardener/.ssh echo "%s" > /home/gardener/.ssh/authorized_keys chown gardener:gardener /home/gardener/.ssh/authorized_keys echo "gardener ALL=(ALL) NOPASSWD:ALL" >/etc/sudoers.d/99-gardener-user -systemctl start ssh +systemctl start sshd || systemctl start ssh `, bastion.Spec.SSHPublicKey) return []byte(userData) diff --git a/test/integration/gardenlet/bastion/bastion_test.go b/test/integration/gardenlet/bastion/bastion_test.go index 5f240be30e8..65927d5fffc 100644 --- a/test/integration/gardenlet/bastion/bastion_test.go +++ b/test/integration/gardenlet/bastion/bastion_test.go @@ -368,7 +368,7 @@ mkdir -p /home/gardener/.ssh echo "%s" > /home/gardener/.ssh/authorized_keys chown gardener:gardener /home/gardener/.ssh/authorized_keys echo "gardener ALL=(ALL) NOPASSWD:ALL" >/etc/sudoers.d/99-gardener-user -systemctl start ssh +systemctl start sshd || systemctl start ssh `, bastion.Spec.SSHPublicKey) return []byte(userData) From 74baab997f1fee7ebb290c1ff838b508150b2bb0 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Wed, 19 Nov 2025 19:00:00 +0100 Subject: [PATCH 141/176] Update dependency gardener/gardener-extension-networking-cilium to v1.45.2 (#13494) --- .../components/controllerregistration/kustomization.yaml | 2 +- example/provider-local/garden/base/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-setup/extensions/networking-cilium/components/controllerregistration/kustomization.yaml b/dev-setup/extensions/networking-cilium/components/controllerregistration/kustomization.yaml index 9bab7266046..12172bed512 100644 --- a/dev-setup/extensions/networking-cilium/components/controllerregistration/kustomization.yaml +++ b/dev-setup/extensions/networking-cilium/components/controllerregistration/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: -- https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.1/example/controller-registration.yaml +- https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.2/example/controller-registration.yaml diff --git a/example/provider-local/garden/base/kustomization.yaml b/example/provider-local/garden/base/kustomization.yaml index 61ffa923192..e2619d8739c 100644 --- a/example/provider-local/garden/base/kustomization.yaml +++ b/example/provider-local/garden/base/kustomization.yaml @@ -14,5 +14,5 @@ components: - ../../../../dev-setup/gardenconfig/components/credentials/workloadidentity-project-local resources: -- https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.1/example/controller-registration.yaml +- https://raw.githubusercontent.com/gardener/gardener-extension-networking-cilium/v1.45.2/example/controller-registration.yaml - https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/v1.53.1/example/controller-registration.yaml From e13f74d96257253aee9fff3e9f93420a24cc6025 Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Thu, 20 Nov 2025 10:35:59 +0100 Subject: [PATCH 142/176] [GEP-28] `gardenadm connect`: Enable `vpaevictionrequirements` controller in `gardenlet` (#13458) * Helper function to detect if the seed is a self-hosted shoot * Check if seed is self-hosted shoot and terminate/don't enable controller - This check is mainly relevant for the `gardenlet`s responsible for seeds running in the `garden` namespace. - If they detect their seed is a self-hosted shoot, then they don't enable the controller. - If they detect this only later (maybe because the `gardenlet` deployment in the `kube-system` namespace does not exist yet because the operator has not yet connected the cluster to Gardener, or maybe because it got temporarily deleted), the entire gardenlet is terminated and restarted (similar approach like for the "seed is garden" case). | Self-Hosted Shoot? | Is Seed? | Is Garden? | Who runs this controller? | | ------------------ | -------- | ---------- | ---------------------------- | | 0 | 0 | 0 | nobody | | 0 | 0 | 1 | `gardener-operator` | | 0 | 1 | 0 | `gardenlet` in `garden` | | 0 | 1 | 1 | `gardener-operator` | | 1 | 0 | 0 | `gardenlet` in `kube-system` | | 1 | 0 | 1 | `gardener-operator` | | 1 | 1 | 0 | `gardenlet` in `kube-system` | | 1 | 1 | 1 | `gardener-operator` | * Enable vpa-eviction-requirements controller for self-hosted shoot `gardenlet`s --- pkg/controller/vpaevictionrequirements/add.go | 16 +++--- .../vpaevictionrequirements/reconciler.go | 2 +- pkg/gardenlet/controller/add.go | 4 ++ .../controller/vpaevictionrequirements/add.go | 53 +++++++++++++++---- pkg/utils/gardener/gardenlet/gardenlet.go | 14 +++++ .../gardener/gardenlet/gardenlet_test.go | 26 +++++++++ 6 files changed, 95 insertions(+), 20 deletions(-) diff --git a/pkg/controller/vpaevictionrequirements/add.go b/pkg/controller/vpaevictionrequirements/add.go index 5f91ab407c5..3ce20e86939 100644 --- a/pkg/controller/vpaevictionrequirements/add.go +++ b/pkg/controller/vpaevictionrequirements/add.go @@ -51,13 +51,13 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, seedCluster cluster.Clust MaxConcurrentReconciles: ptr.Deref(r.ConcurrentSyncs, 0), ReconciliationTimeout: controllerutils.DefaultReconciliationTimeout, }). - WatchesRawSource( - source.Kind[client.Object](seedCluster.GetCache(), - &vpaautoscalingv1.VerticalPodAutoscaler{}, - &handler.EnqueueRequestForObject{}, - vpaEvictionRequirementsManagedByControllerPredicate, - predicateutils.ForEventTypes(predicateutils.Create, predicateutils.Update), - predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{})), - ). + WatchesRawSource(source.Kind[client.Object]( + seedCluster.GetCache(), + &vpaautoscalingv1.VerticalPodAutoscaler{}, + &handler.EnqueueRequestForObject{}, + vpaEvictionRequirementsManagedByControllerPredicate, + predicateutils.ForEventTypes(predicateutils.Create, predicateutils.Update), + predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{}), + )). Complete(r) } diff --git a/pkg/controller/vpaevictionrequirements/reconciler.go b/pkg/controller/vpaevictionrequirements/reconciler.go index 83504586201..6779e5f66ae 100644 --- a/pkg/controller/vpaevictionrequirements/reconciler.go +++ b/pkg/controller/vpaevictionrequirements/reconciler.go @@ -31,8 +31,8 @@ var upscaleOnlyRequirement = []*vpaautoscalingv1.EvictionRequirement{{ // Reconciler implements the reconciliation logic for adding/removing EvictionRequirements to VPA objects. type Reconciler struct { - ConcurrentSyncs *int SeedClient client.Client + ConcurrentSyncs *int Clock clock.Clock } diff --git a/pkg/gardenlet/controller/add.go b/pkg/gardenlet/controller/add.go index 5da00252e2d..c1e5601a2f4 100644 --- a/pkg/gardenlet/controller/add.go +++ b/pkg/gardenlet/controller/add.go @@ -102,6 +102,10 @@ func AddToManager( return fmt.Errorf("failed adding ShootState controller: %w", err) } + if err := vpaevictionrequirements.AddToManager(ctx, mgr, gardenletCancel, *cfg.Controllers.VPAEvictionRequirements, seedCluster); err != nil { + return fmt.Errorf("failed adding VPAEvictionRequirements controller: %w", err) + } + return nil } diff --git a/pkg/gardenlet/controller/vpaevictionrequirements/add.go b/pkg/gardenlet/controller/vpaevictionrequirements/add.go index 7aa49f1d8e8..4cf5bb58c27 100644 --- a/pkg/gardenlet/controller/vpaevictionrequirements/add.go +++ b/pkg/gardenlet/controller/vpaevictionrequirements/add.go @@ -35,7 +35,20 @@ func AddToManager( return fmt.Errorf("failed checking whether the seed is the garden cluster: %w", err) } if seedIsGarden { - return nil // When the seed is the garden cluster at the same time, the gardener-operator runs this controller. + // When the seed is the garden cluster, the gardener-operator runs this controller. + return nil + } + + if !gardenletutils.IsResponsibleForSelfHostedShoot() { + seedIsSelfHostedShoot, err := gardenletutils.SeedIsSelfHostedShoot(ctx, seedCluster.GetAPIReader()) + if err != nil { + return fmt.Errorf("failed checking whether the seed is a self-hosted shoot cluster: %w", err) + } + if seedIsSelfHostedShoot { + // When the seed is a self-hosted shoot cluster, the gardenlet responsible for the self-hosted shoot in the + // kube-system namespace runs this controller. + return nil + } } if err := (&vpaevictionrequirements.Reconciler{ @@ -44,25 +57,43 @@ func AddToManager( return err } - // At this point, the seed is not the garden cluster at the same time. However, this could change during the runtime - // of gardenlet. If so, gardener-operator will take over responsibility of the VPA eviction requirements and will run - // this controller. Since there is no way to stop a controller after it started, we cancel the manager context in - // case the seed is registered as garden during runtime. This way, gardenlet will restart and not add the controller - // again. + // At this point, the seed is not the garden cluster. However, this could change during the runtime of gardenlet. + // If so, gardener-operator will take over responsibility of the VPA eviction requirements and will run this + // controller. + // Similarly, the seed is not a self-hosted shoot cluster (at least not detectable right now, maybe because it was + // not yet connected to Gardener, i.e., a gardenlet deployment in the kube-system namespace does not exist, but this + // could change during the runtime of gardenlet). If so, the gardenlet in the kube-system namespace will take over + // responsibility of the VPA eviction requirements and will run this controller. + // Since there is no way to stop a controller after it started, we cancel the manager context in such cases. This + // way, gardenlet will restart and not add the controller again. return mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { wait.Until(func() { seedIsGarden, err = gardenletutils.SeedIsGarden(ctx, seedCluster.GetClient()) if err != nil { - mgr.GetLogger().Error(err, "Failed checking whether the seed cluster is the garden cluster at the same time") + mgr.GetLogger().Error(err, "Failed checking whether the seed cluster is the garden cluster") return } - if !seedIsGarden { + if seedIsGarden { + mgr.GetLogger().Info("Terminating gardenlet since seed cluster has been registered as garden cluster. " + + "This effectively stops the VPAEvictionRequirements controller (gardener-operator takes over now).") + gardenletCancel() return } - mgr.GetLogger().Info("Terminating gardenlet since seed cluster has been registered as garden cluster. " + - "This effectively stops the VPAEvictionRequirements controller (gardener-operator takes over now).") - gardenletCancel() + if !gardenletutils.IsResponsibleForSelfHostedShoot() { + seedIsSelfHostedShoot, err := gardenletutils.SeedIsSelfHostedShoot(ctx, seedCluster.GetAPIReader()) + if err != nil { + mgr.GetLogger().Error(err, "Failed checking whether the seed cluster is a self-hosted shoot cluster") + return + } + if seedIsSelfHostedShoot { + mgr.GetLogger().Info("Terminating gardenlet since seed cluster has been detect to be a self-hosted " + + "shoot cluster. This effectively stops the VPAEvictionRequirements controller (gardenlet in the " + + "kube-system namespace takes over now).") + gardenletCancel() + return + } + } }, SeedIsGardenCheckInterval, ctx.Done()) return nil })) diff --git a/pkg/utils/gardener/gardenlet/gardenlet.go b/pkg/utils/gardener/gardenlet/gardenlet.go index ee8b5724534..c703701b497 100644 --- a/pkg/utils/gardener/gardenlet/gardenlet.go +++ b/pkg/utils/gardener/gardenlet/gardenlet.go @@ -11,7 +11,9 @@ import ( "strings" "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -19,6 +21,7 @@ import ( bootstraptokenapi "k8s.io/cluster-bootstrap/token/api" "sigs.k8s.io/controller-runtime/pkg/client" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" "github.com/gardener/gardener/pkg/apis/seedmanagement/encoding" gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1" @@ -39,6 +42,17 @@ func SeedIsGarden(ctx context.Context, seedClient client.Reader) (bool, error) { return seedIsGarden, nil } +// SeedIsSelfHostedShoot returns 'true' if the cluster is a self-hosted shoot cluster. +func SeedIsSelfHostedShoot(ctx context.Context, seedClient client.Reader) (bool, error) { + if err := seedClient.Get(ctx, client.ObjectKey{Name: v1beta1constants.DeploymentNameGardenlet, Namespace: metav1.NamespaceSystem}, &appsv1.Deployment{}); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + // SetDefaultGardenClusterAddress sets the default garden cluster address in the given gardenlet configuration if it is not already set. func SetDefaultGardenClusterAddress(log logr.Logger, gardenletConfigRaw runtime.RawExtension, gardenClusterAddress string) (runtime.RawExtension, error) { gardenletConfig, err := encoding.DecodeGardenletConfiguration(&gardenletConfigRaw, false) diff --git a/pkg/utils/gardener/gardenlet/gardenlet_test.go b/pkg/utils/gardener/gardenlet/gardenlet_test.go index f1bcafbc671..eecfb2c0c1b 100644 --- a/pkg/utils/gardener/gardenlet/gardenlet_test.go +++ b/pkg/utils/gardener/gardenlet/gardenlet_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/mock/gomock" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -68,6 +69,31 @@ var _ = Describe("Gardenlet", func() { }) }) + Describe("#SeedIsSelfHostedShoot", func() { + var ( + ctx = context.Background() + fakeClient client.Client + ) + + BeforeEach(func() { + fakeClient = fake.NewClientBuilder().Build() + }) + + It("should return that the seed is a self-hosted shoot", func() { + Expect(fakeClient.Create(ctx, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardenlet", + Namespace: "kube-system", + }, + })).To(Succeed()) + Expect(SeedIsSelfHostedShoot(ctx, fakeClient)).To(BeTrue()) + }) + + It("should return that the seed is not a self-hosted shoot because no gardenlet deployment found", func() { + Expect(SeedIsSelfHostedShoot(ctx, fakeClient)).To(BeFalse()) + }) + }) + Describe("#SetDefaultGardenClusterAddress", func() { var ( log logr.Logger From 4c717d9a1afdb6a24c94a5b3602111cc8a8b6ed9 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Thu, 20 Nov 2025 10:36:07 +0100 Subject: [PATCH 143/176] e2e: avoid relying on public IP detection (#13482) --- test/e2e/gardenadm/managedinfra/gardenadm.go | 21 ++++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/test/e2e/gardenadm/managedinfra/gardenadm.go b/test/e2e/gardenadm/managedinfra/gardenadm.go index f9d5ff95b63..944cc1722a0 100644 --- a/test/e2e/gardenadm/managedinfra/gardenadm.go +++ b/test/e2e/gardenadm/managedinfra/gardenadm.go @@ -72,13 +72,13 @@ var _ = Describe("gardenadm managed infrastructure scenario tests", Label("garde It("should start the bootstrap flow", func() { // Start the gardenadm process but don't wait for it to complete so that we can asynchronously perform assertions // on individual steps in the test specs below. - session = Run("bootstrap", "-d", "../../../dev-setup/gardenadm/resources/generated/managed-infra", "--kubeconfig-output", kubeconfigOutputFile) + session = Run("bootstrap", "-d", "../../../dev-setup/gardenadm/resources/generated/managed-infra", "--kubeconfig-output", kubeconfigOutputFile, + // Override bastion ingress CIDR to a fixed value to avoid relying on public IP detection in e2e tests. + // Restricting bastion ingress traffic does not work in provider-local anyway. + "--bastion-ingress-cidr", "1.2.3.4/32", + ) }) - It("should auto-detect the system's public IPs", func(ctx SpecContext) { - Eventually(ctx, session.Err).Should(gbytes.Say("Using auto-detected public IP addresses as bastion ingress CIDRs")) - }, SpecTimeout(time.Minute)) - It("should find the cloud provider secret", func(ctx SpecContext) { cloudProviderSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "cloudprovider", Namespace: technicalID}} Eventually(ctx, Object(cloudProviderSecret)).Should(HaveField("ObjectMeta.Labels", HaveKeyWithValue("gardener.cloud/purpose", "cloudprovider"))) @@ -153,13 +153,12 @@ var _ = Describe("gardenadm managed infrastructure scenario tests", Label("garde } }, SpecTimeout(time.Minute)) - It("should deploy a bastion with the detected public IPs", func(ctx SpecContext) { + It("should deploy a bastion with the configured ingress CIDRs", func(ctx SpecContext) { bastion := &extensionsv1alpha1.Bastion{ObjectMeta: metav1.ObjectMeta{Name: "gardenadm-bootstrap", Namespace: technicalID}} Eventually(ctx, Object(bastion)).Should(And( - HaveField("Spec.Ingress", Not(Or( - ContainElement(HaveField("IPBlock.CIDR", "0.0.0.0/0")), - ContainElement(HaveField("IPBlock.CIDR", "::/0")), - ))), + HaveField("Spec.Ingress", ConsistOf( + HaveField("IPBlock.CIDR", "1.2.3.4/32"), + )), BeHealthy(health.CheckExtensionObject), ), "should be healthy and not have default (open) ingress CIDRs") }, SpecTimeout(time.Minute)) @@ -276,7 +275,7 @@ var _ = Describe("gardenadm managed infrastructure scenario tests", Label("garde }, SpecTimeout(time.Minute)) It("should run successfully a second time (should be idempotent)", func(ctx SpecContext) { - RunAndWait(ctx, "bootstrap", "-d", "../../../dev-setup/gardenadm/resources/generated/managed-infra") + RunAndWait(ctx, "bootstrap", "-d", "../../../dev-setup/gardenadm/resources/generated/managed-infra", "--bastion-ingress-cidr", "1.2.3.4/32") }, SpecTimeout(2*time.Minute)) }) }) From 2ec92a6bb10f8ff9bb9be0a30540f3c70c669c79 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 20 Nov 2025 12:16:59 +0100 Subject: [PATCH 144/176] Update module golang.org/x/crypto to v0.45.0 [SECURITY] (#13497) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 372a07cd438..517eecb32b0 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( go.uber.org/zap v1.27.0 go.yaml.in/yaml/v2 v2.4.3 go.yaml.in/yaml/v4 v4.0.0-rc.2 - golang.org/x/crypto v0.44.0 + golang.org/x/crypto v0.45.0 golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 golang.org/x/text v0.31.0 golang.org/x/time v0.14.0 diff --git a/go.sum b/go.sum index fda9c48bf8b..8e9d486ac83 100644 --- a/go.sum +++ b/go.sum @@ -853,8 +853,8 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= -golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= From dcab36c509e5eeedc081fa38b835f13837b37996 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:22:00 +0200 Subject: [PATCH 145/176] docs: Update the supported Kubernetes versions for Shoots (#13503) --- README.md | 2 +- docs/usage/shoot-operations/supported_k8s_versions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 64d06e2ff6c..8939ece7b35 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Please find more information regarding the concepts and a detailed description o ## K8s Conformance Test Coverage certified kubernetes logo Gardener takes part in the [Certified Kubernetes Conformance Program](https://www.cncf.io/certification/software-conformance/) to attest its compatibility with the K8s conformance testsuite. -Currently, Gardener is certified for K8s versions up to v1.33, see [the conformance spreadsheet](https://docs.google.com/spreadsheets/d/1uF9BoDzzisHSQemXHIKegMhuythuq_GL3N1mlUUK2h0/edit#gid=0&range=110:111). +Currently, Gardener is certified for K8s versions up to v1.33, see [the conformance spreadsheet](https://docs.google.com/spreadsheets/d/1uF9BoDzzisHSQemXHIKegMhuythuq_GL3N1mlUUK2h0/edit#gid=0&range=109:110). Continuous conformance test results of the latest stable Gardener release are uploaded regularly to the CNCF test grid: diff --git a/docs/usage/shoot-operations/supported_k8s_versions.md b/docs/usage/shoot-operations/supported_k8s_versions.md index 544122fc870..5025f31ac20 100644 --- a/docs/usage/shoot-operations/supported_k8s_versions.md +++ b/docs/usage/shoot-operations/supported_k8s_versions.md @@ -16,7 +16,7 @@ The minimum version of a seed cluster that can be connected to Gardener is **`1. ## Shoot Clusters -Gardener itself is capable of spinning up clusters with Kubernetes versions **`1.29`** up to **`1.33`**. +Gardener itself is capable of spinning up clusters with Kubernetes versions **`1.29`** up to **`1.34`**. However, the concrete versions that can be used for shoot clusters depend on the installed provider extension. Consequently, please consult the documentation of your provider extension to see which Kubernetes versions are supported for shoot clusters. From 00d34dd3e2207c45606f2fafcb155b448bed9260 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Thu, 20 Nov 2025 16:31:16 +0100 Subject: [PATCH 146/176] Update dependency credativ/plutono to v7.5.44 (#13504) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index dec40d4d89b..0ec79ef9ea2 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -335,7 +335,7 @@ images: - name: plutono sourceRepository: github.com/credativ/plutono repository: europe-docker.pkg.dev/gardener-project/releases/3rd/credativ/plutono - tag: "v7.5.43" + tag: "v7.5.44" labels: - name: gardener.cloud/cve-categorisation value: From aca8d1b353aeca1deae2a119f8bd3d501bee85a4 Mon Sep 17 00:00:00 2001 From: Vladimir Nachev Date: Thu, 20 Nov 2025 19:59:15 +0200 Subject: [PATCH 147/176] Enhance dev-setup to maintain set of Secret and WorkloadIdentity CredentialsBindings (#13491) * Enhance dev-setup to maintain set of Secret and WorkloadIdentity CredentialsBindings * upgrade tests: fix update of immutable field * Add credentialsbinding for secret with kubeconfig credentials * Address review feedback Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> --------- Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> --- .../credentialsbinding.yaml | 12 +++++++ .../kustomization.yaml | 3 +- .../credentialsbinding.yaml | 12 +++++++ .../secret-project-garden/kustomization.yaml | 1 + .../credentialsbinding.yaml | 12 +++++++ .../secret-project-local/kustomization.yaml | 1 + .../credentialsbinding.yaml | 12 +++++++ .../kustomization.yaml | 6 ++++ .../workloadidentity.yaml | 10 ++++++ .../credentialsbinding.yaml | 2 +- .../garden/base/kustomization.yaml | 1 + hack/ci-e2e-kind-upgrade.sh | 32 +++++++++++++++++++ 12 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/credentialsbinding.yaml create mode 100644 dev-setup/gardenconfig/components/credentials/secret-project-garden/credentialsbinding.yaml create mode 100644 dev-setup/gardenconfig/components/credentials/secret-project-local/credentialsbinding.yaml create mode 100644 dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/credentialsbinding.yaml create mode 100644 dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/kustomization.yaml create mode 100644 dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/workloadidentity.yaml diff --git a/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/credentialsbinding.yaml b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/credentialsbinding.yaml new file mode 100644 index 00000000000..e13bd905c74 --- /dev/null +++ b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/credentialsbinding.yaml @@ -0,0 +1,12 @@ +apiVersion: security.gardener.cloud/v1alpha1 +kind: CredentialsBinding +metadata: + name: local + namespace: garden +credentialsRef: + apiVersion: v1 + kind: Secret + name: local + namespace: garden +provider: + type: local diff --git a/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/kustomization.yaml b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/kustomization.yaml index 269acef9d28..ea63984b611 100644 --- a/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/kustomization.yaml +++ b/dev-setup/gardenconfig/components/credentials/secret-project-garden-with-kind-kubeconfig/kustomization.yaml @@ -3,10 +3,11 @@ kind: Component resources: - secretbinding.yaml +- credentialsbinding.yaml secretGenerator: # Instead of creating an empty "cloudprovider" secret, we create a secret containing the kubeconfig -# for the local kind cluster. If the kubeconfig key in the cloudprovider key is present, provider-local +# for the local kind cluster. If the kubeconfig key in the cloudprovider secret is present, provider-local # uses it to manage infrastructure resources instead of using its in-cluster ServiceAccount. # See docs/extensions/provider-local.md#credentials for details. # This is only needed for self-hosted shoots in the local setup because provider-local needs to manage diff --git a/dev-setup/gardenconfig/components/credentials/secret-project-garden/credentialsbinding.yaml b/dev-setup/gardenconfig/components/credentials/secret-project-garden/credentialsbinding.yaml new file mode 100644 index 00000000000..e13bd905c74 --- /dev/null +++ b/dev-setup/gardenconfig/components/credentials/secret-project-garden/credentialsbinding.yaml @@ -0,0 +1,12 @@ +apiVersion: security.gardener.cloud/v1alpha1 +kind: CredentialsBinding +metadata: + name: local + namespace: garden +credentialsRef: + apiVersion: v1 + kind: Secret + name: local + namespace: garden +provider: + type: local diff --git a/dev-setup/gardenconfig/components/credentials/secret-project-garden/kustomization.yaml b/dev-setup/gardenconfig/components/credentials/secret-project-garden/kustomization.yaml index bf95e6b3133..194cbe3a175 100644 --- a/dev-setup/gardenconfig/components/credentials/secret-project-garden/kustomization.yaml +++ b/dev-setup/gardenconfig/components/credentials/secret-project-garden/kustomization.yaml @@ -4,3 +4,4 @@ kind: Component resources: - secret.yaml - secretbinding.yaml +- credentialsbinding.yaml diff --git a/dev-setup/gardenconfig/components/credentials/secret-project-local/credentialsbinding.yaml b/dev-setup/gardenconfig/components/credentials/secret-project-local/credentialsbinding.yaml new file mode 100644 index 00000000000..d7a7bae4685 --- /dev/null +++ b/dev-setup/gardenconfig/components/credentials/secret-project-local/credentialsbinding.yaml @@ -0,0 +1,12 @@ +apiVersion: security.gardener.cloud/v1alpha1 +kind: CredentialsBinding +metadata: + name: local + namespace: garden-local +credentialsRef: + apiVersion: v1 + kind: Secret + name: local + namespace: garden-local +provider: + type: local diff --git a/dev-setup/gardenconfig/components/credentials/secret-project-local/kustomization.yaml b/dev-setup/gardenconfig/components/credentials/secret-project-local/kustomization.yaml index bf95e6b3133..194cbe3a175 100644 --- a/dev-setup/gardenconfig/components/credentials/secret-project-local/kustomization.yaml +++ b/dev-setup/gardenconfig/components/credentials/secret-project-local/kustomization.yaml @@ -4,3 +4,4 @@ kind: Component resources: - secret.yaml - secretbinding.yaml +- credentialsbinding.yaml diff --git a/dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/credentialsbinding.yaml b/dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/credentialsbinding.yaml new file mode 100644 index 00000000000..44a30d23c44 --- /dev/null +++ b/dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/credentialsbinding.yaml @@ -0,0 +1,12 @@ +apiVersion: security.gardener.cloud/v1alpha1 +kind: CredentialsBinding +metadata: + name: local-wi + namespace: garden +credentialsRef: + apiVersion: security.gardener.cloud/v1alpha1 + kind: WorkloadIdentity + name: local + namespace: garden +provider: + type: local diff --git a/dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/kustomization.yaml b/dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/kustomization.yaml new file mode 100644 index 00000000000..3a0a1d17837 --- /dev/null +++ b/dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: +- workloadidentity.yaml +- credentialsbinding.yaml diff --git a/dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/workloadidentity.yaml b/dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/workloadidentity.yaml new file mode 100644 index 00000000000..3b720f876d5 --- /dev/null +++ b/dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden/workloadidentity.yaml @@ -0,0 +1,10 @@ +apiVersion: security.gardener.cloud/v1alpha1 +kind: WorkloadIdentity +metadata: + name: local + namespace: garden +spec: + audiences: + - provider-local + targetSystem: + type: local diff --git a/dev-setup/gardenconfig/components/credentials/workloadidentity-project-local/credentialsbinding.yaml b/dev-setup/gardenconfig/components/credentials/workloadidentity-project-local/credentialsbinding.yaml index e308a6f6c62..821aa91c4ed 100644 --- a/dev-setup/gardenconfig/components/credentials/workloadidentity-project-local/credentialsbinding.yaml +++ b/dev-setup/gardenconfig/components/credentials/workloadidentity-project-local/credentialsbinding.yaml @@ -1,7 +1,7 @@ apiVersion: security.gardener.cloud/v1alpha1 kind: CredentialsBinding metadata: - name: local + name: local-wi namespace: garden-local credentialsRef: apiVersion: security.gardener.cloud/v1alpha1 diff --git a/example/provider-local/garden/base/kustomization.yaml b/example/provider-local/garden/base/kustomization.yaml index e2619d8739c..75f44b56e82 100644 --- a/example/provider-local/garden/base/kustomization.yaml +++ b/example/provider-local/garden/base/kustomization.yaml @@ -11,6 +11,7 @@ components: - ../../../../dev-setup/gardenconfig/components/credentials/etcd-backup - ../../../../dev-setup/gardenconfig/components/credentials/secret-project-garden - ../../../../dev-setup/gardenconfig/components/credentials/secret-project-local +- ../../../../dev-setup/gardenconfig/components/credentials/workloadidentity-project-garden - ../../../../dev-setup/gardenconfig/components/credentials/workloadidentity-project-local resources: diff --git a/hack/ci-e2e-kind-upgrade.sh b/hack/ci-e2e-kind-upgrade.sh index db6b17d6682..daed594bbed 100755 --- a/hack/ci-e2e-kind-upgrade.sh +++ b/hack/ci-e2e-kind-upgrade.sh @@ -95,6 +95,7 @@ function install_previous_release() { copy_kubeconfig_from_kubeconfig_env_var gardener_up migrate_secretbinding_secret_ref_namespace + migrate_credentialsbinding_name popd >/dev/null } @@ -127,6 +128,37 @@ secretRef: EOF } +# TODO(vpnachev): Remove this after v1.133 has been released. +# See https://prow.gardener.cloud/view/gs/gardener-prow/pr-logs/pull/gardener_gardener/13491/pull-gardener-e2e-kind-upgrade/1991099554581188608#1:build-log.txt%3A1418 +function migrate_credentialsbinding_name() { + if [[ $(kubectl -n garden-local get --ignore-not-found credentialsbinding local -o jsonpath='{.credentialsRef.kind}') == "Secret" ]]; then + return + fi + + kubectl -n garden-local delete credentialsbinding local --ignore-not-found + cat < Date: Thu, 20 Nov 2025 19:59:22 +0200 Subject: [PATCH 148/176] chore(autoscaling/vpa/updater): Deploy vpa-updater RBAC resources for in-place updates unconditionally (#13499) * chore(autoscaling/vpa/updater): attach in-place cluster role by default This commit removes the current checks that require the InPlaceOrRecreate feature gate of the VPA to be enabled in order to attach the necessary pods/resize subresource permissions. * chore(autoscaling/vpa/test): validate `in-place` cluster role is attached to `vpa-updater` * chore(pkg/apis): adapt `Shoot` validation to handle default InPlaceOrRecreate VPA feature gate * chore(autoscaling/vpa): add `vpa-updater` RBAC definitions for `in-place` to the common configs This commit adds the in-place vpa-updater RBAC definitions to the common list of RBAC components as we don't need to verify if the InPlaceOrRecreate feature gate is enabled. Also: - Configure the tests to check when any feature gate is specified in the yaml Shoot spec, the VPA components will get the additional flag `--feature-gates=` with its [feature gate] value. * feat(gardenlet/botanist/test): add test to handle missing `VPA` feature gates when enabling `VPAInPlaceUpdates` * chore(docs/vpa): update VPA `usage` and `operations` docs with default `InPlaceOrRecreate` context * chore(autoscaling/vpa/test): drop redundant test for cluster role presence Also: - remove references to the InPlaceOrRecreate feature gate from the VPA tests. * chore(docs/usage): adapt `compatibility` section with default `InPlaceOrRecreate` VPA feature gate * chore(validation/shoot): remove redundant `version` field in validation utils * chore(docs/usage): fix typo in `Shoot` configuration guide * Update docs/usage/autoscaling/in-place-resource-updates.md Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> --------- Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> --- .../enabling-in-place-resource-updates.md | 4 +- .../autoscaling/in-place-resource-updates.md | 22 ++------- pkg/apis/core/validation/shoot.go | 9 +--- pkg/apis/core/validation/shoot_test.go | 49 +++++++++---------- pkg/component/autoscaling/vpa/updater.go | 14 ++---- pkg/component/autoscaling/vpa/vpa_test.go | 40 ++++----------- .../botanist/resource_manager_test.go | 21 ++++++++ 7 files changed, 66 insertions(+), 93 deletions(-) diff --git a/docs/operations/enabling-in-place-resource-updates.md b/docs/operations/enabling-in-place-resource-updates.md index 8680600b668..5f1faf7e156 100644 --- a/docs/operations/enabling-in-place-resource-updates.md +++ b/docs/operations/enabling-in-place-resource-updates.md @@ -43,13 +43,13 @@ Refer to the `gardenlet` component configuration [manifest](../../../example/20- > When deployed in a `Shoot` cluster, the _mutating_ webhook targets `vertical pod autoscaler` resources __inside__ the `kube-system` and `kubernetes-dashboard` namespaces. -To make use of the _mutating_ resource manager webhook, the `Shoot`'s [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler) deployment must have the `InPlaceOrRecreate` feature gate enabled. Follow the [in-place resource updates](./in-place-resource-updates.md#shoot) guide for more details about the Vertical Pod Autoscaler components setup. +To make use of the _mutating_ resource manager webhook, the `Shoot`'s [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler) deployment must __not have__ the `InPlaceOrRecreate` feature gate __disabled__. Follow the [in-place resource updates](./in-place-resource-updates.md#shoot) guide for more details about the Vertical Pod Autoscaler components setup. #### Seed > When deployed in a `Seed` cluster, the _mutating_ webhook targets `vertical pod autoscaler` resources __outside__ the `kube-system` and `kubernetes-dashboard` namespaces. -To make use of the _mutating_ resource manager webhook, the `Seed`'s [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler) deployment must have the `InPlaceOrRecreate` feature gate enabled. Follow the [in-place resource updates](./in-place-resource-updates.md#seed) guide for more details about the Vertical Pod Autoscaler components setup. +To make use of the _mutating_ resource manager webhook, the `Seed`'s [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler) deployment must __not have__ the `InPlaceOrRecreate` feature gate __disabled__. Follow the [in-place resource updates](./in-place-resource-updates.md#seed) guide for more details about the Vertical Pod Autoscaler components setup. ### Gardener Operator diff --git a/docs/usage/autoscaling/in-place-resource-updates.md b/docs/usage/autoscaling/in-place-resource-updates.md index b8d7bba4c6d..7bfd2026a1d 100644 --- a/docs/usage/autoscaling/in-place-resource-updates.md +++ b/docs/usage/autoscaling/in-place-resource-updates.md @@ -13,7 +13,7 @@ This is a short guide covering the enablement of `in-place` resource updates in ## Compatibility `In-place` resource updates got introduced as an alpha [feature](https://kubernetes.io/blog/2023/05/12/in-place-pod-resize-alpha/) in Kubernetes 1.27. In Kubernetes 1.33, it got promoted to beta and enabled by default. -On the `Vertical Pod Autoscaler` side, with Release [1.4.0](https://github.com/kubernetes/autoscaler/releases/tag/vertical-pod-autoscaler-1.4.0), `in-place` resources updates are available as an _alpha_ [feature](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/docs/features.md#in-place-updates-inplaceorrecreate) for `vpa-admission-controller` and `vpa-updater`. +On the `Vertical Pod Autoscaler` side, with Release [1.5.0](https://github.com/kubernetes/autoscaler/releases/tag/vertical-pod-autoscaler-1.5.0), `in-place` resources updates are available as a _beta_ feature (enabled by default) for `vpa-admission-controller` and `vpa-updater`. For more details, see the [In-Place Updates documentation](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/docs/features.md#in-place-updates-inplaceorrecreate). ### Kubernetes @@ -21,7 +21,7 @@ With Kubernetes 1.33, the `InPlacePodVerticalScaling` feature gate, which enable ### Vertical Pod Autoscaling -With [1.4.0](https://github.com/kubernetes/autoscaler/releases/tag/vertical-pod-autoscaler-1.4.0), the `InPlaceOrRecreate` feature gate, which enables `in-place` resource updates for `vpa-admission-controller` and `vpa-updater`, got introduced as an _alpha_ feature. To explicitly enable the feature for `Shoot`, `Seed` or `Garden` clusters, the `Vertical Pod Autoscaling` configurations, for the respective cluster types need to include an additional `InPlaceOrRecreate: true` entry in the `featureGates` map. +With [1.5.0](https://github.com/kubernetes/autoscaler/releases/tag/vertical-pod-autoscaler-1.5.0), the `InPlaceOrRecreate` feature gate, which enables `in-place` resource updates for `vpa-admission-controller` and `vpa-updater`, got promoted to a __beta__ feature, making it _enabled_ by default. Refer to the [usage guide](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/docs/features.md#usage) for details about instrumenting a `VerticalPodAutoscaler` resource with `in-place` updates. ## Configuration @@ -30,21 +30,9 @@ As described in the [Compatibility](#compatibility) section, _alpha_ versions of ### Shoot -Since `Vertical Pod Autoscaler` has its `InPlaceOrRecreate` feature gate still in _alpha_, and Kubernetes promoted `InPlacePodVerticalScaling` to _beta_ in 1.33, it is allowed to configure `in-place` resource updates only for `Shoot`(s) with Kubernetes version 1.33+. - -- Enable the `InPlaceOrRecreate` feature gate for `Vertical Por Autoscaler` in the Shoot spec: - - ```yaml - spec: - kubernetes: - verticalPodAutoscaler: - featureGates: - InPlaceOrRecreate: true - ``` - -- Make sure that the `InPlacePodVerticalScaling` Kubernetes feature gate is not explicitly disabled in the Shoot spec for: - - kube-apiserver: `.spec.kubernetes.kubeAPIServer.featureGates` - - kubelet: `.spec.kubernetes.kubelet.featureGates` or `.spec.provider.workers[].kubernetes.kubelet.featureGates` +Since `Vertical Pod Autoscaler` has its `InPlaceOrRecreate` feature gate in _beta_, making it enabled by default, make sure that it is __not__ explicitly disabled. In addition, verify that the `InPlacePodVerticalScaling` Kubernetes feature gate is not explicitly disabled in the Shoot spec for: +- kube-apiserver: `.spec.kubernetes.kubeAPIServer.featureGates` +- kubelet: `.spec.kubernetes.kubelet.featureGates` or `.spec.provider.workers[].kubernetes.kubelet.featureGates` ### Seed diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index dd39587471d..cab8b7e9b57 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -1000,7 +1000,7 @@ func validateKubernetes(kubernetes core.Kubernetes, networking *core.Networking, } if verticalPodAutoscaler := kubernetes.VerticalPodAutoscaler; verticalPodAutoscaler != nil { - allErrs = append(allErrs, ValidateVerticalPodAutoscaler(*verticalPodAutoscaler, kubernetes.Version, fldPath.Child("verticalPodAutoscaler"))...) + allErrs = append(allErrs, ValidateVerticalPodAutoscaler(*verticalPodAutoscaler, fldPath.Child("verticalPodAutoscaler"))...) } } @@ -1430,7 +1430,7 @@ func ValidateCloudProfileReference(cloudProfileReference *core.CloudProfileRefer } // ValidateVerticalPodAutoscaler validates the given VerticalPodAutoscaler fields. -func ValidateVerticalPodAutoscaler(autoScaler core.VerticalPodAutoscaler, version string, fldPath *field.Path) field.ErrorList { +func ValidateVerticalPodAutoscaler(autoScaler core.VerticalPodAutoscaler, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if autoScaler.EvictAfterOOMThreshold != nil { @@ -1482,11 +1482,6 @@ func ValidateVerticalPodAutoscaler(autoScaler core.VerticalPodAutoscaler, versio allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(*count, fldPath.Child("memoryAggregationIntervalCount"))...) } allErrs = append(allErrs, featuresvalidation.ValidateVpaFeatureGates(autoScaler.FeatureGates, fldPath.Child("featureGates"))...) - if isEnabled, ok := autoScaler.FeatureGates["InPlaceOrRecreate"]; ok && isEnabled { - if k8sGreaterEqual133, _ := versionutils.CheckVersionMeetsConstraint(version, ">= 1.33"); !k8sGreaterEqual133 { - allErrs = append(allErrs, field.Invalid(fldPath.Child("featureGates"), "InPlaceOrRecreate", "for Kubernetes versions < 1.33, feature gate is not supported")) - } - } allErrs = append(allErrs, ValidateVerticalPodAutoscalerMaxAllowed(autoScaler.MaxAllowed, fldPath)...) return allErrs diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index 47c0c7e2fd2..9b1a9e4c7b0 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -3826,91 +3826,87 @@ var _ = Describe("Shoot Validation Tests", func() { ) DescribeTable("verticalPod autoscaler values", - func(verticalPodAutoscaler core.VerticalPodAutoscaler, version string, matcher gomegatypes.GomegaMatcher) { - Expect(ValidateVerticalPodAutoscaler(verticalPodAutoscaler, version, nil)).To(matcher) + func(verticalPodAutoscaler core.VerticalPodAutoscaler, matcher gomegatypes.GomegaMatcher) { + Expect(ValidateVerticalPodAutoscaler(verticalPodAutoscaler, nil)).To(matcher) }, Entry("with empty field", - core.VerticalPodAutoscaler{}, "", + core.VerticalPodAutoscaler{}, BeEmpty(), ), Entry("with invalid evictAfterOOMThreshold field", - core.VerticalPodAutoscaler{EvictAfterOOMThreshold: &negativeDuration}, "", + core.VerticalPodAutoscaler{EvictAfterOOMThreshold: &negativeDuration}, ConsistOf(field.Invalid(field.NewPath("evictAfterOOMThreshold"), negativeDuration.Duration.String(), "must be non-negative")), ), Entry("with invalid evictionRateBurst field", - core.VerticalPodAutoscaler{EvictionRateBurst: ptr.To[int32](-1)}, "", + core.VerticalPodAutoscaler{EvictionRateBurst: ptr.To[int32](-1)}, ConsistOf(field.Invalid(field.NewPath("evictionRateBurst"), int32(-1), "must be non-negative")), ), Entry("with invalid evictionTolerance field", - core.VerticalPodAutoscaler{EvictionTolerance: ptr.To(-0.1)}, "", + core.VerticalPodAutoscaler{EvictionTolerance: ptr.To(-0.1)}, ConsistOf(field.Invalid(field.NewPath("evictionTolerance"), -0.1, "fraction value must be in the range [0, 1)")), ), Entry("with invalid recommendationMarginFraction field", - core.VerticalPodAutoscaler{RecommendationMarginFraction: ptr.To(1.0)}, "", + core.VerticalPodAutoscaler{RecommendationMarginFraction: ptr.To(1.0)}, ConsistOf(field.Invalid(field.NewPath("recommendationMarginFraction"), 1.0, "fraction value must be in the range [0, 1)")), ), Entry("with invalid updaterInterval field", - core.VerticalPodAutoscaler{UpdaterInterval: &negativeDuration}, "", + core.VerticalPodAutoscaler{UpdaterInterval: &negativeDuration}, ConsistOf(field.Invalid(field.NewPath("updaterInterval"), negativeDuration.Duration.String(), "must be non-negative")), ), Entry("with invalid recommenderInterval field", - core.VerticalPodAutoscaler{RecommenderInterval: &negativeDuration}, "", + core.VerticalPodAutoscaler{RecommenderInterval: &negativeDuration}, ConsistOf(field.Invalid(field.NewPath("recommenderInterval"), negativeDuration.Duration.String(), "must be non-negative")), ), Entry("with invalid targetCPUPercentile field", - core.VerticalPodAutoscaler{TargetCPUPercentile: &percentileLessThanZero}, "", + core.VerticalPodAutoscaler{TargetCPUPercentile: &percentileLessThanZero}, ConsistOf(field.Invalid(field.NewPath("targetCPUPercentile"), percentileLessThanZero, "percentile value must be in the range [0, 1]")), ), Entry("with invalid recommendationLowerBoundCPUPercentile field", - core.VerticalPodAutoscaler{RecommendationLowerBoundCPUPercentile: &percentileLessThanZero}, "", + core.VerticalPodAutoscaler{RecommendationLowerBoundCPUPercentile: &percentileLessThanZero}, ConsistOf(field.Invalid(field.NewPath("recommendationLowerBoundCPUPercentile"), percentileLessThanZero, "percentile value must be in the range [0, 1]")), ), Entry("with invalid recommendationUpperBoundCPUPercentile field", - core.VerticalPodAutoscaler{RecommendationUpperBoundCPUPercentile: &percentileGreaterThanOne}, "", + core.VerticalPodAutoscaler{RecommendationUpperBoundCPUPercentile: &percentileGreaterThanOne}, ConsistOf(field.Invalid(field.NewPath("recommendationUpperBoundCPUPercentile"), percentileGreaterThanOne, "percentile value must be in the range [0, 1]")), ), Entry("with invalid targetMemoryPercentile field", - core.VerticalPodAutoscaler{TargetMemoryPercentile: &percentileGreaterThanOne}, "", + core.VerticalPodAutoscaler{TargetMemoryPercentile: &percentileGreaterThanOne}, ConsistOf(field.Invalid(field.NewPath("targetMemoryPercentile"), percentileGreaterThanOne, "percentile value must be in the range [0, 1]")), ), Entry("with invalid recommendationLowerBoundMemoryPercentile field", - core.VerticalPodAutoscaler{RecommendationLowerBoundMemoryPercentile: &percentileLessThanZero}, "", + core.VerticalPodAutoscaler{RecommendationLowerBoundMemoryPercentile: &percentileLessThanZero}, ConsistOf(field.Invalid(field.NewPath("recommendationLowerBoundMemoryPercentile"), percentileLessThanZero, "percentile value must be in the range [0, 1]")), ), Entry("with invalid recommendationUpperBoundMemoryPercentile field", - core.VerticalPodAutoscaler{RecommendationUpperBoundMemoryPercentile: &percentileGreaterThanOne}, "", + core.VerticalPodAutoscaler{RecommendationUpperBoundMemoryPercentile: &percentileGreaterThanOne}, ConsistOf(field.Invalid(field.NewPath("recommendationUpperBoundMemoryPercentile"), percentileGreaterThanOne, "percentile value must be in the range [0, 1]")), ), Entry("with invalid cpuHistogramDecayHalfLife field", - core.VerticalPodAutoscaler{CPUHistogramDecayHalfLife: &negativeDuration}, "", + core.VerticalPodAutoscaler{CPUHistogramDecayHalfLife: &negativeDuration}, ConsistOf(field.Invalid(field.NewPath("cpuHistogramDecayHalfLife"), negativeDuration.Duration.String(), "must be non-negative")), ), Entry("with invalid memoryHistogramDecayHalfLife field", - core.VerticalPodAutoscaler{MemoryHistogramDecayHalfLife: &negativeDuration}, "", + core.VerticalPodAutoscaler{MemoryHistogramDecayHalfLife: &negativeDuration}, ConsistOf(field.Invalid(field.NewPath("memoryHistogramDecayHalfLife"), negativeDuration.Duration.String(), "must be non-negative")), ), Entry("with invalid memoryAggregationInterval field", - core.VerticalPodAutoscaler{MemoryAggregationInterval: &negativeDuration}, "", + core.VerticalPodAutoscaler{MemoryAggregationInterval: &negativeDuration}, ConsistOf(field.Invalid(field.NewPath("memoryAggregationInterval"), negativeDuration.Duration.String(), "must be non-negative")), ), Entry("with invalid memoryAggregationIntervalCount field", - core.VerticalPodAutoscaler{MemoryAggregationIntervalCount: ptr.To[int64](-1)}, "", + core.VerticalPodAutoscaler{MemoryAggregationIntervalCount: ptr.To[int64](-1)}, ConsistOf(field.Invalid(field.NewPath("memoryAggregationIntervalCount"), int64(-1), "must be greater than or equal to 0").WithOrigin("minimum")), ), Entry("with unknown feature gate", - core.VerticalPodAutoscaler{FeatureGates: map[string]bool{"Foo": true}}, "", + core.VerticalPodAutoscaler{FeatureGates: map[string]bool{"Foo": true}}, ConsistOf(field.Invalid(field.NewPath("featureGates.Foo"), "Foo", "unknown feature gate")), ), - Entry("with invalid InPlaceOrRecreate feature gate for Kubernetes 1.32", - core.VerticalPodAutoscaler{FeatureGates: map[string]bool{"InPlaceOrRecreate": true}}, "1.32", - ConsistOf(field.Invalid(field.NewPath("featureGates"), "InPlaceOrRecreate", "for Kubernetes versions < 1.33, feature gate is not supported")), - ), Entry("with unsupported maxAllowed resource", - core.VerticalPodAutoscaler{MaxAllowed: map[corev1.ResourceName]resource.Quantity{"storage": {}}}, "", + core.VerticalPodAutoscaler{MaxAllowed: map[corev1.ResourceName]resource.Quantity{"storage": {}}}, ConsistOf(field.NotSupported(field.NewPath("maxAllowed.storage"), corev1.ResourceName("storage"), []corev1.ResourceName{"cpu", "memory"})), ), Entry("with invalid maxAllowed resource quantity value", - core.VerticalPodAutoscaler{MaxAllowed: map[corev1.ResourceName]resource.Quantity{"cpu": resource.MustParse("-100m"), "memory": resource.MustParse("-100Mi")}}, "", + core.VerticalPodAutoscaler{MaxAllowed: map[corev1.ResourceName]resource.Quantity{"cpu": resource.MustParse("-100m"), "memory": resource.MustParse("-100Mi")}}, ConsistOf( field.Invalid(field.NewPath("maxAllowed.cpu"), "-100m", "must be greater than or equal to 0"), field.Invalid(field.NewPath("maxAllowed.memory"), "-100Mi", "must be greater than or equal to 0"), @@ -3940,7 +3936,6 @@ var _ = Describe("Shoot Validation Tests", func() { "memory": resource.MustParse("32Gi"), }, }, - "1.33", BeEmpty(), ), ) diff --git a/pkg/component/autoscaling/vpa/updater.go b/pkg/component/autoscaling/vpa/updater.go index 968819e501b..5340becad14 100644 --- a/pkg/component/autoscaling/vpa/updater.go +++ b/pkg/component/autoscaling/vpa/updater.go @@ -78,6 +78,10 @@ func (v *vpa) updaterResourceConfigs() component.ResourceConfigs { configs := component.ResourceConfigs{ {Obj: clusterRole, Class: component.Application, MutateFn: func() { v.reconcileUpdaterClusterRole(clusterRole) }}, {Obj: clusterRoleBinding, Class: component.Application, MutateFn: func() { v.reconcileUpdaterClusterRoleBinding(clusterRoleBinding, clusterRole, updater) }}, + {Obj: clusterRoleInPlace, Class: component.Application, MutateFn: func() { v.reconcileUpdaterClusterRoleInPlace(clusterRoleInPlace) }}, + {Obj: clusterRoleBindingInPlace, Class: component.Application, MutateFn: func() { + v.reconcileUpdaterClusterRoleBindingInPlace(clusterRoleBindingInPlace, clusterRoleInPlace, updater) + }}, {Obj: roleLeaderLocking, Class: component.Application, MutateFn: func() { v.reconcileUpdaterRoleLeaderLocking(roleLeaderLocking) }}, {Obj: roleBindingLeaderLocking, Class: component.Application, MutateFn: func() { v.reconcileUpdaterRoleBindingLeaderLocking(roleBindingLeaderLocking, roleLeaderLocking, updater) @@ -87,16 +91,6 @@ func (v *vpa) updaterResourceConfigs() component.ResourceConfigs { {Obj: serviceMonitor, Class: component.Runtime, MutateFn: func() { v.reconcileUpdaterServiceMonitor(serviceMonitor) }}, } - if isEnabled, ok := v.values.FeatureGates["InPlaceOrRecreate"]; ok && isEnabled { - inPlaceConfigs := component.ResourceConfigs{ - {Obj: clusterRoleInPlace, Class: component.Application, MutateFn: func() { v.reconcileUpdaterClusterRoleInPlace(clusterRoleInPlace) }}, - {Obj: clusterRoleBindingInPlace, Class: component.Application, MutateFn: func() { - v.reconcileUpdaterClusterRoleBindingInPlace(clusterRoleBindingInPlace, clusterRoleInPlace, updater) - }}, - } - configs = append(configs, inPlaceConfigs...) - } - if v.values.ClusterType == component.ClusterTypeSeed { serviceAccount := v.emptyServiceAccount(updater) configs = append(configs, diff --git a/pkg/component/autoscaling/vpa/vpa_test.go b/pkg/component/autoscaling/vpa/vpa_test.go index 686aaf7feed..b1004c87a5b 100644 --- a/pkg/component/autoscaling/vpa/vpa_test.go +++ b/pkg/component/autoscaling/vpa/vpa_test.go @@ -1695,6 +1695,8 @@ var _ = Describe("VPA", func() { serviceAccountUpdater, clusterRoleUpdater, clusterRoleBindingUpdater, + clusterRoleUpdaterInPlace, + clusterRoleBindingUpdaterInPlace, roleLeaderLockingUpdater, roleBindingLeaderLockingUpdater, deploymentUpdater, @@ -1926,61 +1928,39 @@ var _ = Describe("VPA", func() { args := getContainerArgs(deployment, "recommender") Expect(args).ShouldNot(ContainElement(HavePrefix("--feature-gates="))) }) - - It("should not deploy in-place allowing RBAC resources", func() { - Expect(c.Get(ctx, client.ObjectKeyFromObject(managedResource), managedResource)).To(Succeed()) - - clusterRoleBindingUpdaterInPlace.Subjects[0].Namespace = "kube-system" - resources := []client.Object{ - clusterRoleUpdaterInPlace, - clusterRoleBindingUpdaterInPlace, - } - Expect(managedResource).NotTo(contain(resources...)) - }) }) - Context("with InPlaceOrRecreate enabled", func() { + Context("with feature gate enabled", func() { BeforeEach(func() { featureGates := map[string]bool{ - "InPlaceOrRecreate": true, + "Foo": true, } vpa = vpaFor(component.ClusterTypeShoot, false, featureGates) Expect(vpa.Deploy(ctx)).To(Succeed()) }) - It("should add InPlaceOrRecreate=true feature gate to vpa-admission-controller container", func() { + It("should add feature gate to vpa-admission-controller container", func() { deployment := &appsv1.Deployment{} Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: "vpa-admission-controller"}, deployment)).To(Succeed()) args := getContainerArgs(deployment, "admission-controller") - Expect(args).Should(ContainElement(ContainSubstring("--feature-gates=InPlaceOrRecreate=true"))) + Expect(args).Should(ContainElement(ContainSubstring("--feature-gates=Foo=true"))) }) - It("should add InPlaceOrRecreate=true feature gate to vpa-updater container", func() { + It("should add feature gate to vpa-updater container", func() { deployment := &appsv1.Deployment{} Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: "vpa-updater"}, deployment)).To(Succeed()) args := getContainerArgs(deployment, "updater") - Expect(args).Should(ContainElement(ContainSubstring("--feature-gates=InPlaceOrRecreate=true"))) + Expect(args).Should(ContainElement(ContainSubstring("--feature-gates=Foo=true"))) }) - It("should add InPlaceOrRecreate=true feature gate to vpa-recommender container", func() { + It("should add feature gate to vpa-recommender container", func() { deployment := &appsv1.Deployment{} Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: "vpa-recommender"}, deployment)).To(Succeed()) args := getContainerArgs(deployment, "recommender") - Expect(args).Should(ContainElement(ContainSubstring("--feature-gates=InPlaceOrRecreate=true"))) - }) - - It("should deploy in-place allowing RBAC resources", func() { - Expect(c.Get(ctx, client.ObjectKeyFromObject(managedResource), managedResource)).To(Succeed()) - - clusterRoleBindingUpdaterInPlace.Subjects[0].Namespace = "kube-system" - resources := []client.Object{ - clusterRoleUpdaterInPlace, - clusterRoleBindingUpdaterInPlace, - } - Expect(managedResource).To(contain(resources...)) + Expect(args).Should(ContainElement(ContainSubstring("--feature-gates=Foo=true"))) }) }) }) diff --git a/pkg/gardenlet/operation/botanist/resource_manager_test.go b/pkg/gardenlet/operation/botanist/resource_manager_test.go index d29aa832014..daca3840e3e 100644 --- a/pkg/gardenlet/operation/botanist/resource_manager_test.go +++ b/pkg/gardenlet/operation/botanist/resource_manager_test.go @@ -151,6 +151,27 @@ var _ = Describe("ResourceManager", func() { DeferCleanup(test.WithFeatureGate(features.DefaultFeatureGate, features.VPAInPlaceUpdates, true)) }) + Context("without feature gates", func() { + BeforeEach(func() { + botanist.Shoot.SetInfo(&gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Kubernetes: gardencorev1beta1.Kubernetes{ + VerticalPodAutoscaler: &gardencorev1beta1.VerticalPodAutoscaler{ + FeatureGates: nil, + }, + }, + }, + }) + }) + + It("should set VPAInPlaceUpdatesEnabled=true", func() { + resourceManager, err := botanist.DefaultResourceManager() + Expect(resourceManager).NotTo(BeNil()) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceManager.GetValues().VPAInPlaceUpdatesEnabled).To(BeTrue()) + }) + }) + Context("with Shoot Vertical Pod Autoscaler InPlaceOrRecreate feature gate enabled", func() { BeforeEach(func() { botanist.Shoot.SetInfo(&gardencorev1beta1.Shoot{ From e07aedcd611924f8a60af58ee9284883141497ea Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 21 Nov 2025 07:52:14 +0100 Subject: [PATCH 149/176] Update dependency credativ/vali to v2.2.29 (#13501) --- imagevector/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagevector/containers.yaml b/imagevector/containers.yaml index 0ec79ef9ea2..48217db3df2 100644 --- a/imagevector/containers.yaml +++ b/imagevector/containers.yaml @@ -631,7 +631,7 @@ images: - name: vali sourceRepository: github.com/credativ/vali repository: europe-docker.pkg.dev/gardener-project/releases/3rd/credativ/vali - tag: "v2.2.28" + tag: "v2.2.29" labels: - name: 'gardener.cloud/cve-categorisation' value: From 38e44e4804f160eac51e9052a830a9b80e02a37d Mon Sep 17 00:00:00 2001 From: Tim Usner Date: Fri, 21 Nov 2025 16:13:15 +0100 Subject: [PATCH 150/176] Add resources to Garden API (#13464) * Add `resources` to Garden API * Adjust `garden/reference` controller * Address feedback --- .../operator/templates/crd-gardens.yaml | 33 ++++++ docs/api-reference/operator.md | 24 ++++ docs/extensions/referenced-resources.md | 4 +- ...0-crd-operator.gardener.cloud_gardens.yaml | 33 ++++++ example/operator/20-garden.yaml | 7 ++ pkg/apis/core/validation/seed.go | 2 +- pkg/apis/core/validation/shoot.go | 2 +- pkg/apis/core/validation/shootstate.go | 2 +- pkg/apis/core/validation/utils.go | 3 +- pkg/apis/core/validation/utils_test.go | 106 ++++++++++++++++++ pkg/apis/operator/v1alpha1/types_garden.go | 3 + .../operator/v1alpha1/validation/garden.go | 14 +++ .../v1alpha1/validation/garden_test.go | 37 ++++++ .../v1alpha1/zz_generated.deepcopy.go | 5 + .../controller/garden/reference/add.go | 20 +++- .../controller/garden/reference/add_test.go | 16 +++ .../garden/reference/reference_test.go | 47 ++++++-- 17 files changed, 340 insertions(+), 18 deletions(-) diff --git a/charts/gardener/operator/templates/crd-gardens.yaml b/charts/gardener/operator/templates/crd-gardens.yaml index 134597cb6eb..7bf0ea35996 100644 --- a/charts/gardener/operator/templates/crd-gardens.yaml +++ b/charts/gardener/operator/templates/crd-gardens.yaml @@ -144,6 +144,39 @@ spec: - type type: object type: array + resources: + description: Resources holds a list of named resource references that + can be referred to in extension configs by their names. + items: + description: NamedResourceReference is a named reference to a resource. + properties: + name: + description: Name of the resource reference. + type: string + resourceRef: + description: ResourceRef is a reference to a resource. + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + required: + - name + - resourceRef + type: object + type: array runtimeCluster: description: RuntimeCluster contains configuration for the runtime cluster. diff --git a/docs/api-reference/operator.md b/docs/api-reference/operator.md index 26fe312b313..04a5350fe9b 100644 --- a/docs/api-reference/operator.md +++ b/docs/api-reference/operator.md @@ -1617,6 +1617,18 @@ VirtualCluster

VirtualCluster contains configuration for the virtual cluster.

+ + +resources
+ +[]github.com/gardener/gardener/pkg/apis/core/v1beta1.NamedResourceReference + + + +(Optional) +

Resources holds a list of named resource references that can be referred to in extension configs by their names.

+ + @@ -1748,6 +1760,18 @@ VirtualCluster

VirtualCluster contains configuration for the virtual cluster.

+ + +resources
+ +[]github.com/gardener/gardener/pkg/apis/core/v1beta1.NamedResourceReference + + + +(Optional) +

Resources holds a list of named resource references that can be referred to in extension configs by their names.

+ +

GardenStatus diff --git a/docs/extensions/referenced-resources.md b/docs/extensions/referenced-resources.md index 396a35c186d..b82de683e9a 100644 --- a/docs/extensions/referenced-resources.md +++ b/docs/extensions/referenced-resources.md @@ -1,6 +1,6 @@ # Referenced Resources -`Shoot`s and `Seed`s can include a list of resources (usually secrets) that can be referenced by name in the extension `providerConfig` and other Shoot sections, for example: +`Shoot`s, `Seed`s and `Garden`s can include a list of resources (usually secrets) that can be referenced by name in the extension `providerConfig` and other Shoot sections, for example: ```yaml kind: Shoot @@ -26,7 +26,7 @@ spec: name: my-foobar-secret ``` -Gardener expects these referenced resources to be located in the project namespace (e.g., `garden-dev`) for `Shoot`s and in the `garden` namespace for `Seed`s. +Gardener expects these referenced resources to be located in the project namespace (e.g., `garden-dev`) for `Shoot`s and in the `garden` namespace for `Seed`s and `Garden`s. `Seed` resources are copied to the `garden` namespace in the seed cluster, while `Shoot` resources are copied to the control-plane namespace in the shoot cluster. To avoid conflicts with other resources in the shoot, all resources in the seed are prefixed with a static value. diff --git a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml index 134597cb6eb..7bf0ea35996 100644 --- a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml +++ b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml @@ -144,6 +144,39 @@ spec: - type type: object type: array + resources: + description: Resources holds a list of named resource references that + can be referred to in extension configs by their names. + items: + description: NamedResourceReference is a named reference to a resource. + properties: + name: + description: Name of the resource reference. + type: string + resourceRef: + description: ResourceRef is a reference to a resource. + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + required: + - name + - resourceRef + type: object + type: array runtimeCluster: description: RuntimeCluster contains configuration for the runtime cluster. diff --git a/example/operator/20-garden.yaml b/example/operator/20-garden.yaml index 829fd259531..920fed36e8b 100644 --- a/example/operator/20-garden.yaml +++ b/example/operator/20-garden.yaml @@ -315,3 +315,10 @@ spec: networking: services: - 100.64.0.0/13 +# List resources referenced by providerConfig and other sections +# resources: +# - name: foobar-secret +# resourceRef: +# apiVersion: v1 +# kind: Secret +# name: my-foobar-secret diff --git a/pkg/apis/core/validation/seed.go b/pkg/apis/core/validation/seed.go index 8d4b824984e..9364adb3d8e 100644 --- a/pkg/apis/core/validation/seed.go +++ b/pkg/apis/core/validation/seed.go @@ -251,7 +251,7 @@ func ValidateSeedSpec(seedSpec *core.SeedSpec, fldPath *field.Path, inTemplate b } allErrs = append(allErrs, validateExtensions(seedSpec.Extensions, fldPath.Child("extensions"))...) - allErrs = append(allErrs, validateResources(seedSpec.Resources, fldPath.Child("resources"))...) + allErrs = append(allErrs, ValidateResources(seedSpec.Resources, fldPath.Child("resources"))...) return allErrs } diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index cab8b7e9b57..ced95de05a0 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -275,7 +275,7 @@ func ValidateShootSpec(meta metav1.ObjectMeta, spec *core.ShootSpec, fldPath *fi allErrs = append(allErrs, validateAddons(spec.Addons, spec.Purpose, workerless, fldPath.Child("addons"))...) allErrs = append(allErrs, validateDNS(spec.DNS, fldPath.Child("dns"))...) allErrs = append(allErrs, validateExtensions(spec.Extensions, fldPath.Child("extensions"))...) - allErrs = append(allErrs, validateResources(spec.Resources, fldPath.Child("resources"))...) + allErrs = append(allErrs, ValidateResources(spec.Resources, fldPath.Child("resources"))...) allErrs = append(allErrs, validateKubernetes(spec.Kubernetes, spec.Networking, workerless, fldPath.Child("kubernetes"))...) allErrs = append(allErrs, validateNetworking(spec.Networking, workerless, fldPath.Child("networking"))...) allErrs = append(allErrs, validateMaintenance(spec.Maintenance, fldPath.Child("maintenance"), workerless)...) diff --git a/pkg/apis/core/validation/shootstate.go b/pkg/apis/core/validation/shootstate.go index 4feafb52796..5817b2c6bcc 100644 --- a/pkg/apis/core/validation/shootstate.go +++ b/pkg/apis/core/validation/shootstate.go @@ -56,7 +56,7 @@ func ValidateShootStateSpec(shootStateSpec *core.ShootStateSpec, fldPath *field. if extension.Purpose != nil && len(*extension.Purpose) == 0 { allErrs = append(allErrs, field.Invalid(purposePath, extension.Purpose, "extension resource purpose cannot be empty")) } - allErrs = append(allErrs, validateResources(extension.Resources, fldPath.Child("resources"))...) + allErrs = append(allErrs, ValidateResources(extension.Resources, fldPath.Child("resources"))...) } for i, resource := range shootStateSpec.Resources { diff --git a/pkg/apis/core/validation/utils.go b/pkg/apis/core/validation/utils.go index dc1c35e4c56..069ab8b3924 100644 --- a/pkg/apis/core/validation/utils.go +++ b/pkg/apis/core/validation/utils.go @@ -532,7 +532,8 @@ func validateExtensions(extensions []core.Extension, fldPath *field.Path) field. return allErrs } -func validateResources(resources []core.NamedResourceReference, fldPath *field.Path) field.ErrorList { +// ValidateResources validates the given list of NamedResourceReference for valid values and combinations. +func ValidateResources(resources []core.NamedResourceReference, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} names := sets.Set[string]{} for i, resource := range resources { diff --git a/pkg/apis/core/validation/utils_test.go b/pkg/apis/core/validation/utils_test.go index 6b10a6a8b72..ffcfd70d6a4 100644 --- a/pkg/apis/core/validation/utils_test.go +++ b/pkg/apis/core/validation/utils_test.go @@ -9,6 +9,7 @@ import ( . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" gomegatypes "github.com/onsi/gomega/types" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" @@ -184,6 +185,111 @@ var _ = Describe("Utils tests", func() { }) }) + Describe("#ValidateResources", func() { + var fldPath *field.Path + + BeforeEach(func() { + fldPath = field.NewPath("resources") + }) + + It("should forbid resources w/o names or w/ invalid references", func() { + errorList := ValidateResources([]core.NamedResourceReference{{}}, fldPath) + + Expect(errorList).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("resources[0].name"), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("resources[0].resourceRef.kind"), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("resources[0].resourceRef.name"), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("resources[0].resourceRef.apiVersion"), + })), + )) + }) + + It("should forbid resources of kind other than Secret/ConfigMap", func() { + ref := core.NamedResourceReference{ + Name: "test", + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + Kind: "ServiceAccount", + Name: "test-sa", + APIVersion: "v1", + }, + } + + errorList := ValidateResources([]core.NamedResourceReference{ref}, fldPath) + + Expect(errorList).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeNotSupported), + "Field": Equal("resources[0].resourceRef.kind"), + "BadValue": Equal("ServiceAccount"), + })), + )) + }) + + It("should forbid resources with non-unique names", func() { + ref := core.NamedResourceReference{ + Name: "test", + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + Kind: "Secret", + Name: "test-secret", + APIVersion: "v1", + }, + } + + ref2 := core.NamedResourceReference{ + Name: "test", + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + Kind: "ConfigMap", + Name: ref.Name, + APIVersion: "v1", + }, + } + + errorList := ValidateResources([]core.NamedResourceReference{ref, ref2}, fldPath) + + Expect(errorList).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeDuplicate), + "Field": Equal("resources[1].name"), + })), + )) + }) + + It("should allow resources w/ names and valid references", func() { + ref := core.NamedResourceReference{ + Name: "test", + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + Kind: "Secret", + Name: "test-secret", + APIVersion: "v1", + }, + } + + ref2 := core.NamedResourceReference{ + Name: "test-cm", + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + Kind: "ConfigMap", + Name: "test-cm", + APIVersion: "v1", + }, + } + + errorList := ValidateResources([]core.NamedResourceReference{ref, ref2}, fldPath) + + Expect(errorList).To(BeEmpty()) + }) + }) + DescribeTable("#ValidateCredentialsRef", func(ref corev1.ObjectReference, matcher gomegatypes.GomegaMatcher) { fldPath := field.NewPath("credentialsRef") diff --git a/pkg/apis/operator/v1alpha1/types_garden.go b/pkg/apis/operator/v1alpha1/types_garden.go index 3a4983b99c3..9bc2fe4d49e 100644 --- a/pkg/apis/operator/v1alpha1/types_garden.go +++ b/pkg/apis/operator/v1alpha1/types_garden.go @@ -64,6 +64,9 @@ type GardenSpec struct { RuntimeCluster RuntimeCluster `json:"runtimeCluster"` // VirtualCluster contains configuration for the virtual cluster. VirtualCluster VirtualCluster `json:"virtualCluster"` + // Resources holds a list of named resource references that can be referred to in extension configs by their names. + // +optional + Resources []gardencorev1beta1.NamedResourceReference `json:"resources,omitempty"` } // DNSManagement contains specifications of DNS providers. diff --git a/pkg/apis/operator/v1alpha1/validation/garden.go b/pkg/apis/operator/v1alpha1/validation/garden.go index 5a563554093..b456508cd83 100644 --- a/pkg/apis/operator/v1alpha1/validation/garden.go +++ b/pkg/apis/operator/v1alpha1/validation/garden.go @@ -77,10 +77,24 @@ func ValidateGarden(garden *operatorv1alpha1.Garden, extensions []operatorv1alph allErrs = append(allErrs, validateExtensions(garden.Spec.Extensions, extensions, field.NewPath("spec", "extensions"))...) allErrs = append(allErrs, validateRuntimeCluster(garden.Spec.DNS, garden.Spec.RuntimeCluster, helper.HighAvailabilityEnabled(garden), field.NewPath("spec", "runtimeCluster"))...) allErrs = append(allErrs, validateVirtualCluster(garden.Spec.DNS, garden.Spec.VirtualCluster, garden.Spec.RuntimeCluster, field.NewPath("spec", "virtualCluster"))...) + allErrs = append(allErrs, validateResources(garden.Spec.Resources, field.NewPath("spec", "resources"))...) return allErrs } +func validateResources(resources []gardencorev1beta1.NamedResourceReference, path *field.Path) field.ErrorList { + coreResources := make([]gardencore.NamedResourceReference, 0, len(resources)) + for i, res := range resources { + coreResource := &gardencore.NamedResourceReference{} + if err := gardenCoreScheme.Convert(&res, coreResource, nil); err != nil { + return field.ErrorList{field.InternalError(path.Index(i), err)} + } + coreResources = append(coreResources, *coreResource) + } + + return gardencorevalidation.ValidateResources(coreResources, path) +} + // ValidateGardenUpdate contains functionality for performing extended validation of a Garden object under update which // is not possible with standard CRD validation, see https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules. func ValidateGardenUpdate(oldGarden, newGarden *operatorv1alpha1.Garden, extensions []operatorv1alpha1.Extension) field.ErrorList { diff --git a/pkg/apis/operator/v1alpha1/validation/garden_test.go b/pkg/apis/operator/v1alpha1/validation/garden_test.go index cef2de0ab5e..94319a2c094 100644 --- a/pkg/apis/operator/v1alpha1/validation/garden_test.go +++ b/pkg/apis/operator/v1alpha1/validation/garden_test.go @@ -13,6 +13,7 @@ import ( . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" gomegatypes "github.com/onsi/gomega/types" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -2586,6 +2587,42 @@ var _ = Describe("Validation Tests", func() { }) }) }) + + Context("resources", func() { + BeforeEach(func() { + garden.Spec.Resources = []gardencorev1beta1.NamedResourceReference{ + { + Name: "test", + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + Kind: "Secret", + Name: "test-secret", + APIVersion: "v1", + }, + }, + } + }) + + It("should not complain for valid resource names", func() { + Expect(ValidateGarden(garden, extensions)).To(BeEmpty()) + }) + + It("should complain about duplicate resource names", func() { + garden.Spec.Resources = append(garden.Spec.Resources, gardencorev1beta1.NamedResourceReference{ + Name: garden.Spec.Resources[0].Name, + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + Kind: "ConfigMap", + Name: "some-other-name", + APIVersion: "v1", + }, + }) + + Expect(ValidateGarden(garden, extensions)).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeDuplicate), + "Field": Equal("spec.resources[1].name"), + })))) + }) + }) }) Describe("#ValidateGardenUpdate", func() { diff --git a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index dd1528a9554..a7da109e3ec 100644 --- a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -890,6 +890,11 @@ func (in *GardenSpec) DeepCopyInto(out *GardenSpec) { } in.RuntimeCluster.DeepCopyInto(&out.RuntimeCluster) in.VirtualCluster.DeepCopyInto(&out.VirtualCluster) + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]v1beta1.NamedResourceReference, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/operator/controller/garden/reference/add.go b/pkg/operator/controller/garden/reference/add.go index 3d9be933528..336cc8393a5 100644 --- a/pkg/operator/controller/garden/reference/add.go +++ b/pkg/operator/controller/garden/reference/add.go @@ -13,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" "github.com/gardener/gardener/pkg/controller/reference" @@ -55,7 +56,8 @@ func Predicate(oldObj, newObj client.Object) bool { gardenerAPIServerAdmissionPluginSecretChanged(oldGarden.Spec.VirtualCluster.Gardener.APIServer, newGarden.Spec.VirtualCluster.Gardener.APIServer) || kubeAPIServerStructuredAuthenticationConfigMapChanged(oldGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer, newGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer) || kubeAPIServerStructuredAuthorizationConfigMapChanged(oldGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer, newGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer) || - kubeAPIServerStructuredAuthorizationSecretsChanged(oldGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer, newGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer) + kubeAPIServerStructuredAuthorizationSecretsChanged(oldGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer, newGarden.Spec.VirtualCluster.Kubernetes.KubeAPIServer) || + resourceReferencesChanged(oldGarden.Spec.Resources, newGarden.Spec.Resources) } func kubeAPIServerAuditPolicyConfigMapChanged(oldKubeAPIServer, newKubeAPIServer *operatorv1alpha1.KubeAPIServerConfig) bool { @@ -312,6 +314,12 @@ func getReferencedSecretNames(obj client.Object) []string { } } + for _, ref := range garden.Spec.Resources { + if ref.ResourceRef.APIVersion == "v1" && ref.ResourceRef.Kind == "Secret" { + out = append(out, ref.ResourceRef.Name) + } + } + return out } @@ -342,5 +350,15 @@ func getReferencedConfigMapNames(obj client.Object) []string { out = append(out, virtualCluster.Gardener.APIServer.AuditConfig.AuditPolicy.ConfigMapRef.Name) } + for _, ref := range garden.Spec.Resources { + if ref.ResourceRef.APIVersion == "v1" && ref.ResourceRef.Kind == "ConfigMap" { + out = append(out, ref.ResourceRef.Name) + } + } + return out } + +func resourceReferencesChanged(oldRefs, newRefs []gardencorev1beta1.NamedResourceReference) bool { + return !v1beta1helper.ResourceReferencesEqual(oldRefs, newRefs) +} diff --git a/pkg/operator/controller/garden/reference/add_test.go b/pkg/operator/controller/garden/reference/add_test.go index bc21ac936f8..1262ce9dc80 100644 --- a/pkg/operator/controller/garden/reference/add_test.go +++ b/pkg/operator/controller/garden/reference/add_test.go @@ -7,6 +7,7 @@ package reference_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -142,5 +143,20 @@ var _ = Describe("Add", func() { garden.Spec.VirtualCluster.Gardener.APIServer.AdmissionPlugins = []gardencorev1beta1.AdmissionPlugin{{KubeconfigSecretName: ptr.To("foo")}} Expect(Predicate(oldShoot, garden)).To(BeTrue()) }) + + It("should return true because the resource references changed", func() { + oldShoot := garden.DeepCopy() + garden.Spec.Resources = []gardencorev1beta1.NamedResourceReference{ + { + Name: "foo", + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + APIVersion: "v1", + Kind: "Secret", + Name: "bar", + }, + }, + } + Expect(Predicate(oldShoot, garden)).To(BeTrue()) + }) }) }) diff --git a/test/integration/operator/garden/reference/reference_test.go b/test/integration/operator/garden/reference/reference_test.go index 4136b405b3b..2ef5a9f0d2b 100644 --- a/test/integration/operator/garden/reference/reference_test.go +++ b/test/integration/operator/garden/reference/reference_test.go @@ -9,6 +9,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,20 +21,22 @@ import ( var _ = Describe("Garden Reference controller tests", func() { var ( - secret1 *corev1.Secret - secret2 *corev1.Secret - secret3 *corev1.Secret - secret4 *corev1.Secret - secret5 *corev1.Secret - secret6 *corev1.Secret - secret7 *corev1.Secret - secret8 *corev1.Secret - secret9 *corev1.Secret + secret1 *corev1.Secret + secret2 *corev1.Secret + secret3 *corev1.Secret + secret4 *corev1.Secret + secret5 *corev1.Secret + secret6 *corev1.Secret + secret7 *corev1.Secret + secret8 *corev1.Secret + secret9 *corev1.Secret + secret10 *corev1.Secret configMap1 *corev1.ConfigMap configMap2 *corev1.ConfigMap configMap3 *corev1.ConfigMap configMap4 *corev1.ConfigMap + configMap5 *corev1.ConfigMap allReferencedObjects []client.Object garden *operatorv1alpha1.Garden @@ -49,14 +52,16 @@ var _ = Describe("Garden Reference controller tests", func() { secret7 = initializeObject("secret").(*corev1.Secret) secret8 = initializeObject("secret").(*corev1.Secret) secret9 = initializeObject("secret").(*corev1.Secret) + secret10 = initializeObject("secret").(*corev1.Secret) configMap1 = initializeObject("configMap").(*corev1.ConfigMap) configMap2 = initializeObject("configMap").(*corev1.ConfigMap) configMap3 = initializeObject("configMap").(*corev1.ConfigMap) configMap4 = initializeObject("configMap").(*corev1.ConfigMap) + configMap5 = initializeObject("configMap").(*corev1.ConfigMap) - allReferencedObjects = append([]client.Object{}, secret1, secret2, secret3, secret4, secret5, secret6, secret7, secret8, secret9) - allReferencedObjects = append(allReferencedObjects, configMap1, configMap2, configMap3, configMap4) + allReferencedObjects = append([]client.Object{}, secret1, secret2, secret3, secret4, secret5, secret6, secret7, secret8, secret9, secret10) + allReferencedObjects = append(allReferencedObjects, configMap1, configMap2, configMap3, configMap4, configMap5) garden = &operatorv1alpha1.Garden{ ObjectMeta: metav1.ObjectMeta{ @@ -163,6 +168,24 @@ var _ = Describe("Garden Reference controller tests", func() { Services: []string{"100.64.0.0/13"}, }, }, + Resources: []gardencorev1beta1.NamedResourceReference{ + { + Name: "secret-resource", + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + Kind: "Secret", + APIVersion: "v1", + Name: secret10.Name, + }, + }, + { + Name: "configmap-resource", + ResourceRef: autoscalingv1.CrossVersionObjectReference{ + Kind: "ConfigMap", + APIVersion: "v1", + Name: configMap5.Name, + }, + }, + }, }, } }) @@ -189,6 +212,7 @@ var _ = Describe("Garden Reference controller tests", func() { garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer = nil garden.Spec.VirtualCluster.Gardener.APIServer = nil garden.Spec.DNS = nil + garden.Spec.Resources = nil }) It("should not add the finalizer to the garden", func() { @@ -222,6 +246,7 @@ var _ = Describe("Garden Reference controller tests", func() { garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer = nil garden.Spec.VirtualCluster.Gardener.APIServer = nil garden.Spec.DNS = nil + garden.Spec.Resources = nil Expect(testClient.Patch(ctx, garden, patch)).To(Succeed()) for _, obj := range append([]client.Object{garden}, allReferencedObjects...) { From f52cf8dd6199ab1d026b6e1569b1ba190163fcef Mon Sep 17 00:00:00 2001 From: Dimitar Mirchev Date: Fri, 21 Nov 2025 17:13:22 +0200 Subject: [PATCH 151/176] Improve extension health check documentation (#13510) https://github.com/gardener/gardener/pull/13329 did not address the existing documentation, which is fixed with the current commit --- docs/extensions/healthcheck-library.md | 46 ++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/docs/extensions/healthcheck-library.md b/docs/extensions/healthcheck-library.md index 995de033c84..cde4e1f0ed4 100644 --- a/docs/extensions/healthcheck-library.md +++ b/docs/extensions/healthcheck-library.md @@ -52,17 +52,51 @@ You can find implementation examples in the [healtcheck folder](../../extensions type HealthCheck interface { // Check is the function that executes the actual health check Check(context.Context, types.NamespacedName) (*SingleCheckResult, error) - // InjectSeedClient injects the seed client - InjectSeedClient(client.Client) - // InjectShootClient injects the shoot client - InjectShootClient(client.Client) // SetLoggerSuffix injects the logger SetLoggerSuffix(string, string) - // DeepCopy clones the healthCheck - DeepCopy() HealthCheck } ``` +### Client Injection + +Health check implementations can optionally implement one or both of the following interfaces to receive the appropriate Kubernetes clients: + +```go +type ShootClient interface { + // InjectShootClient injects the shoot client + InjectShootClient(client.Client) +} + +type SeedClient interface { + // InjectSeedClient injects the seed client + InjectSeedClient(client.Client) +} +``` + +The health check controller automatically detects if a health checker implements these interfaces and injects the corresponding clients before executing the health checks. +This allows health checks to interact with resources in the shoot cluster (via `ShootClient`) or the seed cluster (via `SeedClient`) as needed. + +For example, a health check that needs to verify resources in the shoot cluster would implement the `ShootClient` interface: + +```go +type MyShootHealthCheck struct { + shootClient client.Client +} + +func (h *MyShootHealthCheck) InjectShootClient(c client.Client) { + h.shootClient = c +} + +func (h *MyShootHealthCheck) Check(ctx context.Context, request types.NamespacedName) (*SingleCheckResult, error) { + // Use h.shootClient to check resources in the shoot cluster + // ... +} +``` + +Similarly, if a health check needs to access both shoot and seed resources, it can implement both interfaces. + +### Reconciliation + The health check controller regularly (default: `30s`) reconciles the extension resource and executes the registered health checks for the dependent objects. As a result, the controller writes condition(s) to the status of the extension containing the health check result. In our example, two checks are mapped to `ShootEveryNodeReady` and one to `ShootSystemComponentsHealthy`, leading to conditions with two distinct `HealthConditionTypes` (condition.type): From 9d121983041a20a1adb57d340d806aa168fc7480 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Fri, 21 Nov 2025 19:43:15 +0100 Subject: [PATCH 152/176] Update module github.com/prometheus/common to v0.67.4 (#13517) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 517eecb32b0..5aa9adaafe0 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.86.2 github.com/prometheus/blackbox_exporter v0.27.0 github.com/prometheus/client_golang v1.23.2 - github.com/prometheus/common v0.67.3 + github.com/prometheus/common v0.67.4 github.com/robfig/cron v1.2.0 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.1 diff --git a/go.sum b/go.sum index 8e9d486ac83..835b47dbbe0 100644 --- a/go.sum +++ b/go.sum @@ -639,8 +639,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.67.3 h1:shd26MlnwTw5jksTDhC7rTQIteBxy+ZZDr3t7F2xN2Q= -github.com/prometheus/common v0.67.3/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ= github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= From 66e2ce3e5b194e4f4c54514c11a656b89252deee Mon Sep 17 00:00:00 2001 From: Vladimir Nachev Date: Mon, 24 Nov 2025 10:46:16 +0200 Subject: [PATCH 153/176] Fix gardener access component to not deploy shoot resource into garden cluster (#13492) * Fix gardener access component to not deploy shoot resource into garden cluster Shoot's adminkubeconfig and viewerkubeconfig permissions are not needed in the Garden cluster * Address review feedback --- pkg/component/gardener/access/access.go | 9 ++++++-- pkg/component/gardener/access/access_test.go | 21 +++++++++++++++++++ .../operation/botanist/gardeneraccess.go | 1 + .../controller/garden/garden/components.go | 1 + 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pkg/component/gardener/access/access.go b/pkg/component/gardener/access/access.go index b9f63605e83..c3907573613 100644 --- a/pkg/component/gardener/access/access.go +++ b/pkg/component/gardener/access/access.go @@ -59,6 +59,8 @@ type Values struct { ServerInCluster string // ManagedResourceLabels are labels added to the ManagedResource. ManagedResourceLabels map[string]string + // IsGardenCluster is specifying whether the component is deployed to a Garden cluster. + IsGardenCluster bool } type accessNameToServer struct { @@ -157,8 +159,11 @@ func (g *gardener) computeResourcesData(serviceAccountNames ...string) (map[stri }) } - resources := append(adminClusterRoleBindings(), viewerClusterRoleBindings()...) - resources = append(resources, gardenerSystemClusterRoleBinding) + resources := []client.Object{gardenerSystemClusterRoleBinding} + if !g.values.IsGardenCluster { + resources = append(resources, adminClusterRoleBindings()...) + resources = append(resources, viewerClusterRoleBindings()...) + } return registry.AddAllAndSerialize(resources...) } diff --git a/pkg/component/gardener/access/access_test.go b/pkg/component/gardener/access/access_test.go index e0ff81fae83..951e02c591f 100644 --- a/pkg/component/gardener/access/access_test.go +++ b/pkg/component/gardener/access/access_test.go @@ -168,6 +168,7 @@ var _ = Describe("Access", func() { ServerOutOfCluster: serverOutOfCluster, ServerInCluster: serverInCluster, ManagedResourceLabels: map[string]string{"foo": "bar"}, + IsGardenCluster: false, }) expectedGardenerSecret = &corev1.Secret{ @@ -322,6 +323,26 @@ users: Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledGardenerInternalSecret), reconciledGardenerInternalSecret)).To(Succeed()) Expect(reconciledGardenerInternalSecret).To(DeepEqual(expectedGardenerInternalSecret)) }) + + It("should not deploy shootAccess resource when configured so", func() { + access = New(fakeClient, namespace, sm, Values{ + ServerOutOfCluster: serverOutOfCluster, + ServerInCluster: serverInCluster, + ManagedResourceLabels: map[string]string{"foo": "bar"}, + IsGardenCluster: true, + }) + Expect(access.Deploy(ctx)).To(Succeed()) + + reconciledManagedResource := &resourcesv1alpha1.ManagedResource{ObjectMeta: metav1.ObjectMeta{Name: managedResourceName, Namespace: namespace}} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledManagedResource), reconciledManagedResource)).To(Succeed()) + Expect(reconciledManagedResource).To(consistOf(gardenerSystemClusterRoleBinding)) + Expect(reconciledManagedResource).ToNot(consistOf( + systemViewersClusterRoleBinding, + projectViewersClusterRoleBinding, + systemAdminClusterRoleBinding, + projectAdminClusterRoleBinding, + )) + }) }) Describe("#Destroy", func() { diff --git a/pkg/gardenlet/operation/botanist/gardeneraccess.go b/pkg/gardenlet/operation/botanist/gardeneraccess.go index 9179f4cba7c..83dfacca0e8 100644 --- a/pkg/gardenlet/operation/botanist/gardeneraccess.go +++ b/pkg/gardenlet/operation/botanist/gardeneraccess.go @@ -19,6 +19,7 @@ func (b *Botanist) DefaultGardenerAccess() component.Deployer { gardeneraccess.Values{ ServerInCluster: b.Shoot.ComputeInClusterAPIServerAddress(false), ServerOutOfCluster: b.Shoot.ComputeOutOfClusterAPIServerAddress(true), + IsGardenCluster: false, }, ) } diff --git a/pkg/operator/controller/garden/garden/components.go b/pkg/operator/controller/garden/garden/components.go index 05f5aa7f139..3d7d28cd666 100644 --- a/pkg/operator/controller/garden/garden/components.go +++ b/pkg/operator/controller/garden/garden/components.go @@ -988,6 +988,7 @@ func (r *Reconciler) newGardenerAccess(garden *operatorv1alpha1.Garden, secretsM ServerInCluster: fmt.Sprintf("%s%s.%s.svc.cluster.local", namePrefix, v1beta1constants.DeploymentNameKubeAPIServer, r.GardenNamespace), ServerOutOfCluster: v1beta1helper.GetAPIServerDomain(garden.Spec.VirtualCluster.DNS.Domains[0].Name), ManagedResourceLabels: map[string]string{v1beta1constants.LabelCareConditionType: string(operatorv1alpha1.VirtualComponentsHealthy)}, + IsGardenCluster: true, }, ) } From 9f27c926d0de5684d908245204e2ae23a2111cc8 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> Date: Mon, 24 Nov 2025 12:31:17 +0200 Subject: [PATCH 154/176] Make the `.spec.provider.workers[].machine.image` field required for the Shoot resource (part 8) (#13399) * Make the `.spec.provider.workers[].machine.image` field required for the Shoot resource * Fix integration tests --- pkg/apis/core/validation/shoot.go | 20 +++++-- pkg/apis/core/validation/shoot_test.go | 56 +++++++++++++++++-- .../controllermanager/bastion/bastion_test.go | 4 ++ .../credentialsbinding_test.go | 8 ++- .../exposureclass/exposureclass_test.go | 8 ++- .../gardenletlifecycle/lifecycle_test.go | 4 ++ .../managedseedset/managedseedset_test.go | 6 +- .../project/activity/activity_test.go | 4 ++ .../project/project/project_test.go | 4 ++ .../project/stale/stale_test.go | 4 ++ .../secretbinding/secretbinding_test.go | 8 ++- .../shoot/conditions/conditions_test.go | 4 ++ .../shoot/hibernation/hibernation_test.go | 4 ++ .../shoot/migration/migration_test.go | 4 ++ .../shoot/quota/quota_test.go | 4 ++ .../shoot/reference/reference_test.go | 4 ++ .../shoot/retry/retry_test.go | 4 ++ .../shoot/statuslabel/statuslabel_test.go | 4 ++ .../shootstate/shootstate_test.go | 4 ++ .../gardenlet/backupentry/backupentry_test.go | 4 ++ .../gardenlet/bastion/bastion_test.go | 4 ++ .../gardenlet/managedseed/managedseed_test.go | 6 +- .../gardenlet/shoot/care/care_test.go | 4 ++ .../gardenlet/shoot/lease/lease_suite_test.go | 4 ++ 24 files changed, 164 insertions(+), 16 deletions(-) diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index ced95de05a0..256a915b892 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -2033,12 +2033,22 @@ func ValidateWorker(worker core.Worker, kubernetes core.Kubernetes, shootNamespa if len(worker.Machine.Type) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("machine", "type"), "must specify a machine type")) } - if worker.Machine.Image != nil { - if len(worker.Machine.Image.Name) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("machine", "image", "name"), "must specify a machine image name")) + if inTemplate { + if worker.Machine.Image != nil { + if len(worker.Machine.Image.Name) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("machine", "image", "name"), "must specify a machine image name")) + } } - if !inTemplate && len(worker.Machine.Image.Version) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("machine", "image", "version"), "must specify a machine image version")) + } else { + if worker.Machine.Image == nil { + allErrs = append(allErrs, field.Required(fldPath.Child("machine", "image"), "must specify a machine image")) + } else { + if len(worker.Machine.Image.Name) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("machine", "image", "name"), "must specify a machine image name")) + } + if len(worker.Machine.Image.Version) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("machine", "image", "version"), "must specify a machine image version")) + } } } if worker.Minimum < 0 { diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index 9b1a9e4c7b0..64eb44ddc9d 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -1395,6 +1395,11 @@ var _ = Describe("Shoot Validation Tests", func() { "Field": Equal("spec.provider.workers[0].machine.type"), "Detail": ContainSubstring("must specify a machine type"), })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("spec.provider.workers[0].machine.image"), + "Detail": ContainSubstring("must specify a machine image"), + })), PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("spec.provider.workers[0].minimum"), @@ -4859,6 +4864,10 @@ var _ = Describe("Shoot Validation Tests", func() { Name: "worker-1", Machine: core.Machine{ Type: "xlarge", + Image: &core.ShootMachineImage{ + Name: "image-name", + Version: "1.0.0", + }, }, Maximum: 1, Minimum: 0, @@ -4868,6 +4877,10 @@ var _ = Describe("Shoot Validation Tests", func() { Name: "worker-2", Machine: core.Machine{ Type: "xlarge", + Image: &core.ShootMachineImage{ + Name: "image-name", + Version: "1.0.0", + }, }, Maximum: 1, Minimum: 0, @@ -7411,8 +7424,21 @@ var _ = Describe("Shoot Validation Tests", func() { Architecture: ptr.To("amd64"), }, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeRequired), - "Field": Equal("machine.type"), + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("machine.type"), + "Detail": Equal("must specify a machine type"), + }))), + ), + Entry("nil machine image", + core.Machine{ + Type: "large", + Image: nil, + Architecture: ptr.To("amd64"), + }, + ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("machine.image"), + "Detail": Equal("must specify a machine image"), }))), ), Entry("empty machine image name", @@ -7425,8 +7451,9 @@ var _ = Describe("Shoot Validation Tests", func() { Architecture: ptr.To("amd64"), }, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeRequired), - "Field": Equal("machine.image.name"), + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("machine.image.name"), + "Detail": Equal("must specify a machine image name"), }))), ), Entry("empty machine image version", @@ -7439,8 +7466,9 @@ var _ = Describe("Shoot Validation Tests", func() { Architecture: ptr.To("amd64"), }, ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeRequired), - "Field": Equal("machine.image.version"), + "Type": Equal(field.ErrorTypeRequired), + "Field": Equal("machine.image.version"), + "Detail": Equal("must specify a machine image version"), }))), ), Entry("nil machine architecture", @@ -7977,6 +8005,10 @@ var _ = Describe("Shoot Validation Tests", func() { MaxUnavailable: ptr.To(intstr.FromInt32(1)), Machine: core.Machine{ Type: "xlarge", + Image: &core.ShootMachineImage{ + Name: "image-name", + Version: "1.0.0", + }, }, } fldPath = field.NewPath("workers").Index(0) @@ -8150,6 +8182,10 @@ var _ = Describe("Shoot Validation Tests", func() { MaxUnavailable: ptr.To(intstr.FromInt32(1)), Machine: core.Machine{ Type: "xlarge", + Image: &core.ShootMachineImage{ + Name: "image-name", + Version: "1.0.0", + }, }, } @@ -8237,6 +8273,10 @@ var _ = Describe("Shoot Validation Tests", func() { MaxUnavailable: ptr.To(intstr.FromInt32(1)), Machine: core.Machine{ Type: "xlarge", + Image: &core.ShootMachineImage{ + Name: "image-name", + Version: "1.0.0", + }, }, } @@ -8384,6 +8424,10 @@ var _ = Describe("Shoot Validation Tests", func() { Name: "worker", Machine: core.Machine{ Type: "xlarge", + Image: &core.ShootMachineImage{ + Name: "image-name", + Version: "1.0.0", + }, }, MaxUnavailable: ptr.To(intstr.FromInt(1)), Priority: ptr.To(int32(-2)), diff --git a/test/integration/controllermanager/bastion/bastion_test.go b/test/integration/controllermanager/bastion/bastion_test.go index 45bc5c44845..f0583a8af10 100644 --- a/test/integration/controllermanager/bastion/bastion_test.go +++ b/test/integration/controllermanager/bastion/bastion_test.go @@ -59,6 +59,10 @@ var _ = Describe("Bastion controller tests", func() { Maximum: 2, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/credentialsbinding/credentialsbinding_test.go b/test/integration/controllermanager/credentialsbinding/credentialsbinding_test.go index 68e4817a309..61d97a23c9e 100644 --- a/test/integration/controllermanager/credentialsbinding/credentialsbinding_test.go +++ b/test/integration/controllermanager/credentialsbinding/credentialsbinding_test.go @@ -92,7 +92,13 @@ var _ = Describe("CredentialsBinding controller test", func() { Name: "cpu-worker", Minimum: 2, Maximum: 2, - Machine: gardencorev1beta1.Machine{Type: "large"}, + Machine: gardencorev1beta1.Machine{ + Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, + }, }, }, }, diff --git a/test/integration/controllermanager/exposureclass/exposureclass_test.go b/test/integration/controllermanager/exposureclass/exposureclass_test.go index 9c1cfd42e7d..74644c80661 100644 --- a/test/integration/controllermanager/exposureclass/exposureclass_test.go +++ b/test/integration/controllermanager/exposureclass/exposureclass_test.go @@ -57,7 +57,13 @@ var _ = Describe("ExposureClass controller test", func() { Name: "cpu-worker", Minimum: 2, Maximum: 2, - Machine: gardencorev1beta1.Machine{Type: "large"}, + Machine: gardencorev1beta1.Machine{ + Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, + }, }, }, }, diff --git a/test/integration/controllermanager/gardenletlifecycle/lifecycle_test.go b/test/integration/controllermanager/gardenletlifecycle/lifecycle_test.go index 2dc7b45d7a2..81de24ed52c 100644 --- a/test/integration/controllermanager/gardenletlifecycle/lifecycle_test.go +++ b/test/integration/controllermanager/gardenletlifecycle/lifecycle_test.go @@ -138,6 +138,10 @@ var _ = Describe("Gardenlet Lifecycle controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/managedseedset/managedseedset_test.go b/test/integration/controllermanager/managedseedset/managedseedset_test.go index 87a9d9ce1a1..3d846791aec 100644 --- a/test/integration/controllermanager/managedseedset/managedseedset_test.go +++ b/test/integration/controllermanager/managedseedset/managedseedset_test.go @@ -191,7 +191,11 @@ var _ = Describe("ManagedSeedSet controller test", func() { { Name: "some-worker", Machine: gardencorev1beta1.Machine{ - Type: "some-machine-type", + Type: "some-machine-type", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, Architecture: ptr.To("amd64"), }, Maximum: 2, diff --git a/test/integration/controllermanager/project/activity/activity_test.go b/test/integration/controllermanager/project/activity/activity_test.go index 613f8c33018..b52152aa0b6 100644 --- a/test/integration/controllermanager/project/activity/activity_test.go +++ b/test/integration/controllermanager/project/activity/activity_test.go @@ -206,6 +206,10 @@ var _ = Describe("Project Activity controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/project/project/project_test.go b/test/integration/controllermanager/project/project/project_test.go index 3eab28ddfdb..063cac11cba 100644 --- a/test/integration/controllermanager/project/project/project_test.go +++ b/test/integration/controllermanager/project/project/project_test.go @@ -70,6 +70,10 @@ var _ = Describe("Project controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/project/stale/stale_test.go b/test/integration/controllermanager/project/stale/stale_test.go index f2b8cacbc15..5364aeb1e95 100644 --- a/test/integration/controllermanager/project/stale/stale_test.go +++ b/test/integration/controllermanager/project/stale/stale_test.go @@ -111,6 +111,10 @@ var _ = Describe("Project Stale controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/secretbinding/secretbinding_test.go b/test/integration/controllermanager/secretbinding/secretbinding_test.go index d55a353ff2a..bedabeb06f9 100644 --- a/test/integration/controllermanager/secretbinding/secretbinding_test.go +++ b/test/integration/controllermanager/secretbinding/secretbinding_test.go @@ -85,7 +85,13 @@ var _ = Describe("SecretBinding controller test", func() { Name: "cpu-worker", Minimum: 2, Maximum: 2, - Machine: gardencorev1beta1.Machine{Type: "large"}, + Machine: gardencorev1beta1.Machine{ + Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, + }, }, }, }, diff --git a/test/integration/controllermanager/shoot/conditions/conditions_test.go b/test/integration/controllermanager/shoot/conditions/conditions_test.go index f82bd180e54..2041e5031c9 100644 --- a/test/integration/controllermanager/shoot/conditions/conditions_test.go +++ b/test/integration/controllermanager/shoot/conditions/conditions_test.go @@ -45,6 +45,10 @@ var _ = Describe("Shoot Conditions controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/shoot/hibernation/hibernation_test.go b/test/integration/controllermanager/shoot/hibernation/hibernation_test.go index 584bb94f22a..ac0c8d11066 100644 --- a/test/integration/controllermanager/shoot/hibernation/hibernation_test.go +++ b/test/integration/controllermanager/shoot/hibernation/hibernation_test.go @@ -36,6 +36,10 @@ var _ = Describe("Shoot Hibernation controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/shoot/migration/migration_test.go b/test/integration/controllermanager/shoot/migration/migration_test.go index 316126be7c9..1d95fd5e6f2 100644 --- a/test/integration/controllermanager/shoot/migration/migration_test.go +++ b/test/integration/controllermanager/shoot/migration/migration_test.go @@ -46,6 +46,10 @@ var _ = Describe("Shoot Migration controller tests", Ordered, func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/shoot/quota/quota_test.go b/test/integration/controllermanager/shoot/quota/quota_test.go index 115cc228009..1b648c8d67b 100644 --- a/test/integration/controllermanager/shoot/quota/quota_test.go +++ b/test/integration/controllermanager/shoot/quota/quota_test.go @@ -96,6 +96,10 @@ var _ = Describe("Shoot Quota controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/shoot/reference/reference_test.go b/test/integration/controllermanager/shoot/reference/reference_test.go index c954763876d..221dd989cd0 100644 --- a/test/integration/controllermanager/shoot/reference/reference_test.go +++ b/test/integration/controllermanager/shoot/reference/reference_test.go @@ -69,6 +69,10 @@ var _ = Describe("Shoot Reference controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/shoot/retry/retry_test.go b/test/integration/controllermanager/shoot/retry/retry_test.go index 02c58c75484..6cf1e0da143 100644 --- a/test/integration/controllermanager/shoot/retry/retry_test.go +++ b/test/integration/controllermanager/shoot/retry/retry_test.go @@ -35,6 +35,10 @@ var _ = Describe("Shoot retry controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/shoot/statuslabel/statuslabel_test.go b/test/integration/controllermanager/shoot/statuslabel/statuslabel_test.go index 8d1bcea55ef..ca826fc0ca5 100644 --- a/test/integration/controllermanager/shoot/statuslabel/statuslabel_test.go +++ b/test/integration/controllermanager/shoot/statuslabel/statuslabel_test.go @@ -37,6 +37,10 @@ var _ = Describe("Shoot StatusLabel controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/controllermanager/shootstate/shootstate_test.go b/test/integration/controllermanager/shootstate/shootstate_test.go index b8a8fc44f82..654f568f01d 100644 --- a/test/integration/controllermanager/shootstate/shootstate_test.go +++ b/test/integration/controllermanager/shootstate/shootstate_test.go @@ -44,6 +44,10 @@ var _ = Describe("ShootState controller test", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/gardenlet/backupentry/backupentry_test.go b/test/integration/gardenlet/backupentry/backupentry_test.go index 0e8436e82d8..390ca0621d4 100644 --- a/test/integration/gardenlet/backupentry/backupentry_test.go +++ b/test/integration/gardenlet/backupentry/backupentry_test.go @@ -218,6 +218,10 @@ var _ = Describe("BackupEntry controller tests", func() { Maximum: 2, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/gardenlet/bastion/bastion_test.go b/test/integration/gardenlet/bastion/bastion_test.go index 65927d5fffc..d8503764914 100644 --- a/test/integration/gardenlet/bastion/bastion_test.go +++ b/test/integration/gardenlet/bastion/bastion_test.go @@ -121,6 +121,10 @@ var _ = Describe("Bastion controller tests", func() { Maximum: 2, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/gardenlet/managedseed/managedseed_test.go b/test/integration/gardenlet/managedseed/managedseed_test.go index c9e5a03f036..73cf7e94274 100644 --- a/test/integration/gardenlet/managedseed/managedseed_test.go +++ b/test/integration/gardenlet/managedseed/managedseed_test.go @@ -132,7 +132,11 @@ var _ = Describe("ManagedSeed controller test", func() { { Name: "some-worker", Machine: gardencorev1beta1.Machine{ - Type: "some-machine-type", + Type: "some-machine-type", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, Architecture: ptr.To("amd64"), }, Maximum: 2, diff --git a/test/integration/gardenlet/shoot/care/care_test.go b/test/integration/gardenlet/shoot/care/care_test.go index 8f1c04d38c2..c0bc96353a6 100644 --- a/test/integration/gardenlet/shoot/care/care_test.go +++ b/test/integration/gardenlet/shoot/care/care_test.go @@ -94,6 +94,10 @@ var _ = Describe("Shoot Care controller tests", func() { Maximum: 3, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, diff --git a/test/integration/gardenlet/shoot/lease/lease_suite_test.go b/test/integration/gardenlet/shoot/lease/lease_suite_test.go index cf1843c6191..226007a5fa9 100644 --- a/test/integration/gardenlet/shoot/lease/lease_suite_test.go +++ b/test/integration/gardenlet/shoot/lease/lease_suite_test.go @@ -129,6 +129,10 @@ var _ = BeforeSuite(func() { Maximum: 2, Machine: gardencorev1beta1.Machine{ Type: "large", + Image: &gardencorev1beta1.ShootMachineImage{ + Name: "some-image", + Version: ptr.To("1.0.0"), + }, }, }, }, From 84c20c00795555159ad5d4b9dcd1aae41bda051e Mon Sep 17 00:00:00 2001 From: Sebastian Stauch Date: Mon, 24 Nov 2025 15:06:17 +0100 Subject: [PATCH 155/176] Add `CustomDNSServerInNodeLocalDNS` feature gate (#13511) * add ServerBlockSupportForNodeLocalDNS feature gate * rename feature gate to CustomDNSServerInNodeLocalDNS * address feedback * address feedback * Address PR review feedback from ialidzhikov (2) --------- Co-authored-by: Ismail Alidzhikov <9372594+ialidzhikov@users.noreply.github.com> --- docs/deployment/feature_gates.md | 2 + .../networking/nodelocaldns/nodelocaldns.go | 2 + .../nodelocaldns/nodelocaldns_test.go | 405 +++++++++++++++--- .../networking/nodelocaldns/resources.go | 111 ++--- pkg/features/features.go | 26 +- pkg/gardenlet/features/features.go | 1 + .../operation/botanist/nodelocaldns.go | 18 +- 7 files changed, 445 insertions(+), 120 deletions(-) diff --git a/docs/deployment/feature_gates.md b/docs/deployment/feature_gates.md index 57e42e1a341..7c265fbcf57 100644 --- a/docs/deployment/feature_gates.md +++ b/docs/deployment/feature_gates.md @@ -33,6 +33,7 @@ The following tables are a summary of the feature gates that you can set on diff | OpenTelemetryCollector | `false` | `Alpha` | `1.124` | | | UseUnifiedHTTPProxyPort | `false` | `Alpha` | `1.130` | | | VPAInPlaceUpdates | `false` | `Alpha` | `1.133` | | +| CustomDNSServerInNodeLocalDNS | `true` | `Beta` | `1.133` | | ## Feature Gates for Graduated or Deprecated Features @@ -257,3 +258,4 @@ A *General Availability* (GA) feature is also referred to as a *stable* feature. | OpenTelemetryCollector | `gardenlet` | Routes logs through an instance of an `OpenTelemetry Collector` in the control-plane of `Shoots`. | | UseUnifiedHTTPProxyPort | `gardenlet` | Enables the gardenlet to set up the unified HTTP proxy network infrastructure. Gardenlet will also reconfigure the API server proxy and shoot VPN client to connect to the unified port using the new X-Gardener-Destination header. | | VPAInPlaceUpdates | `gardenlet`, `gardener-operator` | Enables the usage of in-place Pod resource updates in `Shoot`, `Seed` and `Garden` cluster's Vertical Pod Autoscaler deployments. | +| CustomDNSServerInNodeLocalDNS | `gardenlet` | Enables custom server block support for NodeLocalDNS in the custom CoreDNS configuration of Shoot clusters. | diff --git a/pkg/component/networking/nodelocaldns/nodelocaldns.go b/pkg/component/networking/nodelocaldns/nodelocaldns.go index 52891283d63..6271e8e918e 100644 --- a/pkg/component/networking/nodelocaldns/nodelocaldns.go +++ b/pkg/component/networking/nodelocaldns/nodelocaldns.go @@ -126,6 +126,8 @@ type Values struct { Workers []gardencorev1beta1.Worker // KubeProxyConfig is the kube-proxy configuration for the shoot. KubeProxyConfig *gardencorev1beta1.KubeProxyConfig + // CustomDNSServerInNodeLocalDNS indicates whether server block support is enabled for node-local-dns. + CustomDNSServerInNodeLocalDNS bool } // New creates a new instance of DeployWaiter for node-local-dns. diff --git a/pkg/component/networking/nodelocaldns/nodelocaldns_test.go b/pkg/component/networking/nodelocaldns/nodelocaldns_test.go index 2a0ff6eeed3..dbc4be8beae 100644 --- a/pkg/component/networking/nodelocaldns/nodelocaldns_test.go +++ b/pkg/component/networking/nodelocaldns/nodelocaldns_test.go @@ -236,6 +236,7 @@ var _ = Describe("NodeLocalDNS", func() { Name: "worker-aaaa", }, }, + CustomDNSServerInNodeLocalDNS: true, } managedResource = &resourcesv1alpha1.ManagedResource{ @@ -297,7 +298,10 @@ metadata: namespace: kube-system ` configMapYAMLFor = func() string { - + serverBlockImport := "" + if values.CustomDNSServerInNodeLocalDNS { + serverBlockImport = "\n import generated-config/custom-server-block.server" + } out := `apiVersion: v1 data: Corefile: | @@ -350,8 +354,7 @@ data: errors cache 30 reload - } - import generated-config/custom-server-block.server + }` + serverBlockImport + ` immutable: true kind: ConfigMap metadata: @@ -390,7 +393,7 @@ status: maxUnavailable = intstr.FromString("10%") hostPathFileOrCreate = corev1.HostPathFileOrCreate daemonSetYAMLFor = func() *appsv1.DaemonSet { - daemonset := &appsv1.DaemonSet{ + daemonSet := &appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{ APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "DaemonSet", @@ -453,41 +456,6 @@ status: Type: corev1.SeccompProfileTypeRuntimeDefault, }, }, - InitContainers: []corev1.Container{ - { - Name: "coredns-config-adapter", - Image: values.CorednsConfigAdapterImage, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("5m"), - corev1.ResourceMemory: resource.MustParse("10Mi"), - }, - }, - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: ptr.To(false), - RunAsNonRoot: ptr.To(true), - RunAsUser: ptr.To[int64](65532), - RunAsGroup: ptr.To[int64](65532), - }, - Args: []string{ - "-inputDir=/etc/custom", - "-outputDir=/etc/generated-config", - "-bind=bind " + bindIP(values), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "custom-config-volume", - MountPath: "/etc/custom", - ReadOnly: true, - }, - { - MountPath: "/etc/generated-config", - Name: "generated-config", - }, - }, - RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), - }, - }, Containers: []corev1.Container{ { Name: "node-cache", @@ -569,10 +537,6 @@ status: MountPath: "/etc/custom", ReadOnly: true, }, - { - MountPath: "/etc/generated-config", - Name: "generated-config", - }, }, }, }, @@ -625,18 +589,60 @@ status: }, }, }, - { - Name: "generated-config", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, }, }, }, }, } - return daemonset + + if values.CustomDNSServerInNodeLocalDNS { + daemonSet.Spec.Template.Spec.InitContainers = append(daemonSet.Spec.Template.Spec.InitContainers, corev1.Container{ + Name: "coredns-config-adapter", + Image: values.CorednsConfigAdapterImage, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To[int64](65532), + RunAsGroup: ptr.To[int64](65532), + }, + Args: []string{ + "-inputDir=/etc/custom", + "-outputDir=/etc/generated-config", + "-bind=bind " + bindIP(values), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "custom-config-volume", + MountPath: "/etc/custom", + ReadOnly: true, + }, + { + MountPath: "/etc/generated-config", + Name: "generated-config", + }, + }, + RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + }) + + daemonSet.Spec.Template.Spec.Volumes = append(daemonSet.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "generated-config", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + + daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts = append(daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + MountPath: "/etc/generated-config", + Name: "generated-config", + }) + } + return daemonSet } vpaYAML = `apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler @@ -646,7 +652,7 @@ metadata: spec: resourcePolicy: containerPolicies: - - containerName: '*' + - containerName: node-cache controlledValues: RequestsOnly - containerName: coredns-config-adapter mode: "Off" @@ -1280,6 +1286,303 @@ import generated-config/custom-server-block.server }) }) }) + + Context("NodeLocalDNS with ipvsEnabled not enabled and featureGate CustomDNSServerInNodeLocalDNS disabled", func() { + BeforeEach(func() { + values.IPFamilies = []gardencorev1beta1.IPFamily{gardencorev1beta1.IPFamilyIPv4} + ipvsAddress = "169.254.20.10" + values.ClusterDNS = []string{"__PILLAR__CLUSTER__DNS__"} + values.DNSServers = []string{"1.2.3.4", "2001:db8::1"} + values.CustomDNSServerInNodeLocalDNS = false + vpaYAML = `apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: node-local-dns-worker-aaaa + namespace: kube-system +spec: + resourcePolicy: + containerPolicies: + - containerName: node-cache + controlledValues: RequestsOnly + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: node-local-dns-worker-aaaa + updatePolicy: + updateMode: Recreate +status: {} +` + }) + + Context("ConfigMap", func() { + JustBeforeEach(func() { + configMapData := map[string]string{ + "Corefile": `cluster.local:53 { + loop + bind ` + bindIP(values) + ` + forward . ` + strings.Join(values.ClusterDNS, " ") + ` { + ` + forceTcpToClusterDNS + ` + } + prometheus :` + strconv.Itoa(prometheusPort) + ` + health ` + healthAddress(values) + `:` + strconv.Itoa(livenessProbePort) + ` + import custom/*.override + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + } +in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind ` + bindIP(values) + ` + forward . ` + strings.Join(values.ClusterDNS, " ") + ` { + ` + forceTcpToClusterDNS + ` + } + prometheus :` + strconv.Itoa(prometheusPort) + ` + } +ip6.arpa:53 { + errors + cache 30 + reload + loop + bind ` + bindIP(values) + ` + forward . ` + strings.Join(values.ClusterDNS, " ") + ` { + ` + forceTcpToClusterDNS + ` + } + prometheus :` + strconv.Itoa(prometheusPort) + ` + } +.:53 { + loop + bind ` + bindIP(values) + ` + forward . ` + strings.Join(upstreamDNSAddress, " ") + ` { + ` + forceTcpToUpstreamDNS + ` + } + prometheus :` + strconv.Itoa(prometheusPort) + ` + import custom/*.override + errors + cache 30 + reload + } +`, + } + configMapHash = utils.ComputeConfigMapChecksum(configMapData)[:8] + }) + + Context("ForceTcpToClusterDNS : true and ForceTcpToUpstreamDNS : true", func() { + BeforeEach(func() { + values.Config = &gardencorev1beta1.NodeLocalDNS{ + Enabled: true, + ForceTCPToClusterDNS: ptr.To(true), + ForceTCPToUpstreamDNS: ptr.To(true), + DisableForwardToUpstreamDNS: ptr.To(false), + } + }) + + Context("w/o VPA", func() { + BeforeEach(func() { + values.VPAEnabled = false + }) + + It("should successfully deploy all resources", func() { + expectedManifests = append(expectedManifests, configMapYAMLFor()) + Expect(manifests).To(ContainElements(expectedManifests)) + + managedResourceDaemonset, err := extractDaemonSet(manifests, kubernetes.ShootCodec.UniversalDeserializer()) + Expect(err).ToNot(HaveOccurred()) + daemonset := daemonSetYAMLFor() + utilruntime.Must(references.InjectAnnotations(daemonset)) + Expect(daemonset).To(DeepEqual(managedResourceDaemonset)) + }) + }) + + Context("w/ VPA", func() { + BeforeEach(func() { + values.VPAEnabled = true + }) + + It("should successfully deploy all resources", func() { + expectedManifests = append(expectedManifests, configMapYAMLFor(), vpaYAML) + Expect(manifests).To(ContainElements(expectedManifests)) + + managedResourceDaemonset, err := extractDaemonSet(manifests, kubernetes.ShootCodec.UniversalDeserializer()) + Expect(err).ToNot(HaveOccurred()) + daemonset := daemonSetYAMLFor() + utilruntime.Must(references.InjectAnnotations(daemonset)) + Expect(daemonset).To(DeepEqual(managedResourceDaemonset)) + }) + }) + }) + Context("ForceTcpToClusterDNS : true and ForceTcpToUpstreamDNS : false", func() { + BeforeEach(func() { + values.Config = &gardencorev1beta1.NodeLocalDNS{ + Enabled: true, + ForceTCPToClusterDNS: ptr.To(true), + ForceTCPToUpstreamDNS: ptr.To(false), + DisableForwardToUpstreamDNS: ptr.To(false), + } + forceTcpToClusterDNS = "force_tcp" + forceTcpToUpstreamDNS = "prefer_udp" + }) + + Context("w/o VPA", func() { + BeforeEach(func() { + values.VPAEnabled = false + }) + + It("should successfully deploy all resources", func() { + expectedManifests = append(expectedManifests, configMapYAMLFor()) + Expect(manifests).To(ContainElements(expectedManifests)) + + managedResourceDaemonset, err := extractDaemonSet(manifests, kubernetes.ShootCodec.UniversalDeserializer()) + Expect(err).ToNot(HaveOccurred()) + daemonset := daemonSetYAMLFor() + utilruntime.Must(references.InjectAnnotations(daemonset)) + Expect(daemonset).To(DeepEqual(managedResourceDaemonset)) + }) + }) + + Context("w/ VPA", func() { + BeforeEach(func() { + values.VPAEnabled = true + }) + + It("should successfully deploy all resources", func() { + expectedManifests = append(expectedManifests, configMapYAMLFor(), vpaYAML) + Expect(manifests).To(ContainElements(expectedManifests)) + + managedResourceDaemonset, err := extractDaemonSet(manifests, kubernetes.ShootCodec.UniversalDeserializer()) + Expect(err).ToNot(HaveOccurred()) + daemonset := daemonSetYAMLFor() + utilruntime.Must(references.InjectAnnotations(daemonset)) + Expect(daemonset).To(DeepEqual(managedResourceDaemonset)) + }) + }) + }) + + Context("ForceTcpToClusterDNS : false and ForceTcpToUpstreamDNS : true", func() { + BeforeEach(func() { + values.Config = &gardencorev1beta1.NodeLocalDNS{ + Enabled: true, + ForceTCPToClusterDNS: ptr.To(false), + ForceTCPToUpstreamDNS: ptr.To(true), + DisableForwardToUpstreamDNS: ptr.To(false), + } + forceTcpToClusterDNS = "prefer_udp" + forceTcpToUpstreamDNS = "force_tcp" + }) + + Context("w/o VPA", func() { + BeforeEach(func() { + values.VPAEnabled = false + }) + + It("should successfully deploy all resources", func() { + expectedManifests = append(expectedManifests, configMapYAMLFor()) + Expect(manifests).To(ContainElements(expectedManifests)) + + managedResourceDaemonset, err := extractDaemonSet(manifests, kubernetes.ShootCodec.UniversalDeserializer()) + Expect(err).ToNot(HaveOccurred()) + daemonset := daemonSetYAMLFor() + utilruntime.Must(references.InjectAnnotations(daemonset)) + Expect(daemonset).To(DeepEqual(managedResourceDaemonset)) + }) + }) + + Context("w/ VPA", func() { + BeforeEach(func() { + values.VPAEnabled = true + }) + + It("should successfully deploy all resources", func() { + expectedManifests = append(expectedManifests, configMapYAMLFor(), vpaYAML) + Expect(manifests).To(ContainElements(expectedManifests)) + + managedResourceDaemonset, err := extractDaemonSet(manifests, kubernetes.ShootCodec.UniversalDeserializer()) + Expect(err).ToNot(HaveOccurred()) + daemonset := daemonSetYAMLFor() + utilruntime.Must(references.InjectAnnotations(daemonset)) + Expect(daemonset).To(DeepEqual(managedResourceDaemonset)) + }) + }) + }) + + Context("ForceTcpToClusterDNS : false and ForceTcpToUpstreamDNS : false", func() { + BeforeEach(func() { + values.Config = &gardencorev1beta1.NodeLocalDNS{ + Enabled: true, + ForceTCPToClusterDNS: ptr.To(false), + ForceTCPToUpstreamDNS: ptr.To(false), + DisableForwardToUpstreamDNS: ptr.To(false), + } + forceTcpToClusterDNS = "prefer_udp" + forceTcpToUpstreamDNS = "prefer_udp" + }) + Context("w/o VPA", func() { + BeforeEach(func() { + values.VPAEnabled = false + }) + + It("should successfully deploy all resources", func() { + expectedManifests = append(expectedManifests, configMapYAMLFor()) + Expect(manifests).To(ContainElements(expectedManifests)) + + managedResourceDaemonset, err := extractDaemonSet(manifests, kubernetes.ShootCodec.UniversalDeserializer()) + Expect(err).ToNot(HaveOccurred()) + daemonset := daemonSetYAMLFor() + utilruntime.Must(references.InjectAnnotations(daemonset)) + Expect(daemonset).To(DeepEqual(managedResourceDaemonset)) + }) + }) + + Context("w/ VPA", func() { + BeforeEach(func() { + values.VPAEnabled = true + }) + + It("should successfully deploy all resources", func() { + expectedManifests = append(expectedManifests, configMapYAMLFor(), vpaYAML) + Expect(manifests).To(ContainElements(expectedManifests)) + + managedResourceDaemonset, err := extractDaemonSet(manifests, kubernetes.ShootCodec.UniversalDeserializer()) + Expect(err).ToNot(HaveOccurred()) + daemonset := daemonSetYAMLFor() + utilruntime.Must(references.InjectAnnotations(daemonset)) + Expect(daemonset).To(DeepEqual(managedResourceDaemonset)) + }) + }) + }) + + Context("DisableForwardToUpstreamDNS true", func() { + BeforeEach(func() { + values.Config = &gardencorev1beta1.NodeLocalDNS{ + Enabled: true, + ForceTCPToClusterDNS: ptr.To(true), + ForceTCPToUpstreamDNS: ptr.To(true), + DisableForwardToUpstreamDNS: ptr.To(true), + } + values.VPAEnabled = true + upstreamDNSAddress = values.ClusterDNS + forceTcpToClusterDNS = "force_tcp" + forceTcpToUpstreamDNS = "force_tcp" + }) + + It("should successfully deploy all resources", func() { + expectedManifests = append(expectedManifests, configMapYAMLFor()) + Expect(manifests).To(ContainElements(expectedManifests)) + + managedResourceDaemonset, err := extractDaemonSet(manifests, kubernetes.ShootCodec.UniversalDeserializer()) + Expect(err).ToNot(HaveOccurred()) + daemonset := daemonSetYAMLFor() + utilruntime.Must(references.InjectAnnotations(daemonset)) + Expect(daemonset).To(DeepEqual(managedResourceDaemonset)) + }) + }) + }) + }) }) }) diff --git a/pkg/component/networking/nodelocaldns/resources.go b/pkg/component/networking/nodelocaldns/resources.go index b184a9ef61b..2ce81e260fe 100644 --- a/pkg/component/networking/nodelocaldns/resources.go +++ b/pkg/component/networking/nodelocaldns/resources.go @@ -101,12 +101,15 @@ ip6.arpa:53 { cache 30 reload } -import generated-config/custom-server-block.server `, }, } ) + if n.values.CustomDNSServerInNodeLocalDNS { + configMap.Data[configDataKey] = configMap.Data[configDataKey] + "import generated-config/custom-server-block.server\n" + } + utilruntime.Must(kubernetesutils.MakeUnique(configMap)) var ( @@ -210,41 +213,6 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc v1beta1constants.LabelNodeLocalDNS: "true", v1beta1constants.LabelWorkerPool: worker.Name, }, - InitContainers: []corev1.Container{ - { - Name: sideCarName, - Image: n.values.CorednsConfigAdapterImage, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("5m"), - corev1.ResourceMemory: resource.MustParse("10Mi"), - }, - }, - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: ptr.To(false), - RunAsNonRoot: ptr.To(true), - RunAsUser: ptr.To[int64](65532), - RunAsGroup: ptr.To[int64](65532), - }, - Args: []string{ - "-inputDir=" + volumeMountPathCustomConfig, - "-outputDir=" + volumeMountPathGeneratedConfig, - "-bind=bind " + n.bindIP(), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: volumeMountNameCustomConfig, - MountPath: volumeMountPathCustomConfig, - ReadOnly: true, - }, - { - MountPath: volumeMountPathGeneratedConfig, - Name: volumeMountNameGeneratedConfig, - }, - }, - RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), - }, - }, Containers: []corev1.Container{ { Name: containerName, @@ -326,10 +294,6 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc MountPath: volumeMountPathCustomConfig, ReadOnly: true, }, - { - MountPath: volumeMountPathGeneratedConfig, - Name: volumeMountNameGeneratedConfig, - }, }, }, }, @@ -382,17 +346,60 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc }, }, }, - { - Name: volumeMountNameGeneratedConfig, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, }, }, }, }, } + + if n.values.CustomDNSServerInNodeLocalDNS { + daemonSet.Spec.Template.Spec.InitContainers = append(daemonSet.Spec.Template.Spec.InitContainers, corev1.Container{ + Name: sideCarName, + Image: n.values.CorednsConfigAdapterImage, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To[int64](65532), + RunAsGroup: ptr.To[int64](65532), + }, + Args: []string{ + "-inputDir=" + volumeMountPathCustomConfig, + "-outputDir=" + volumeMountPathGeneratedConfig, + "-bind=bind " + n.bindIP(), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: volumeMountNameCustomConfig, + MountPath: volumeMountPathCustomConfig, + ReadOnly: true, + }, + { + MountPath: volumeMountPathGeneratedConfig, + Name: volumeMountNameGeneratedConfig, + }, + }, + RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + }) + + daemonSet.Spec.Template.Spec.Volumes = append(daemonSet.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: volumeMountNameGeneratedConfig, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + + daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts = append(daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + MountPath: volumeMountPathGeneratedConfig, + Name: volumeMountNameGeneratedConfig, + }) + } + utilruntime.Must(references.InjectAnnotations(daemonSet)) clientObjects = append(clientObjects, daemonSet) @@ -415,17 +422,19 @@ func (n *nodeLocalDNS) computePoolResourcesData(serviceAccount *corev1.ServiceAc ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ { - ContainerName: vpaautoscalingv1.DefaultContainerResourcePolicy, + ContainerName: containerName, ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), }, - { - ContainerName: sideCarName, - Mode: ptr.To(vpaautoscalingv1.ContainerScalingModeOff), - }, }, }, }, } + if n.values.CustomDNSServerInNodeLocalDNS { + vpa.Spec.ResourcePolicy.ContainerPolicies = append(vpa.Spec.ResourcePolicy.ContainerPolicies, vpaautoscalingv1.ContainerResourcePolicy{ + ContainerName: sideCarName, + Mode: ptr.To(vpaautoscalingv1.ContainerScalingModeOff), + }) + } clientObjects = append(clientObjects, vpa) } } diff --git a/pkg/features/features.go b/pkg/features/features.go index 53d1b36185b..fd1306f2b2b 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -80,6 +80,11 @@ const ( // owner: @vitanovs @ialidzhikov // alpha: v1.133.0 VPAInPlaceUpdates featuregate.Feature = "VPAInPlaceUpdates" + + // CustomDNSServerInNodeLocalDNS enables custom server block support for NodeLocalDNS in the custom CoreDNS configuration of Shoot clusters. + // owner: @docktofuture + // beta: v1.133.0 + CustomDNSServerInNodeLocalDNS featuregate.Feature = "CustomDNSServerInNodeLocalDNS" ) // DefaultFeatureGate is the central feature gate map used by all gardener components. @@ -107,16 +112,17 @@ var DefaultFeatureGate = utilfeature.DefaultMutableFeatureGate // AllFeatureGates is the list of all feature gates. var AllFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - DefaultSeccompProfile: {Default: false, PreRelease: featuregate.Alpha}, - ShootCredentialsBinding: {Default: true, PreRelease: featuregate.Beta}, - NewWorkerPoolHash: {Default: true, PreRelease: featuregate.Beta}, - InPlaceNodeUpdates: {Default: false, PreRelease: featuregate.Alpha}, - IstioTLSTermination: {Default: false, PreRelease: featuregate.Alpha}, - CloudProfileCapabilities: {Default: false, PreRelease: featuregate.Alpha}, - DoNotCopyBackupCredentials: {Default: true, PreRelease: featuregate.Beta}, - OpenTelemetryCollector: {Default: false, PreRelease: featuregate.Alpha}, - UseUnifiedHTTPProxyPort: {Default: false, PreRelease: featuregate.Alpha}, - VPAInPlaceUpdates: {Default: false, PreRelease: featuregate.Alpha}, + DefaultSeccompProfile: {Default: false, PreRelease: featuregate.Alpha}, + ShootCredentialsBinding: {Default: true, PreRelease: featuregate.Beta}, + NewWorkerPoolHash: {Default: true, PreRelease: featuregate.Beta}, + InPlaceNodeUpdates: {Default: false, PreRelease: featuregate.Alpha}, + IstioTLSTermination: {Default: false, PreRelease: featuregate.Alpha}, + CloudProfileCapabilities: {Default: false, PreRelease: featuregate.Alpha}, + DoNotCopyBackupCredentials: {Default: true, PreRelease: featuregate.Beta}, + OpenTelemetryCollector: {Default: false, PreRelease: featuregate.Alpha}, + UseUnifiedHTTPProxyPort: {Default: false, PreRelease: featuregate.Alpha}, + VPAInPlaceUpdates: {Default: false, PreRelease: featuregate.Alpha}, + CustomDNSServerInNodeLocalDNS: {Default: true, PreRelease: featuregate.Beta}, } // GetFeatures returns a feature gate map with the respective specifications. Non-existing feature gates are ignored. diff --git a/pkg/gardenlet/features/features.go b/pkg/gardenlet/features/features.go index 8ace7dac960..f75b3fc0e51 100644 --- a/pkg/gardenlet/features/features.go +++ b/pkg/gardenlet/features/features.go @@ -26,5 +26,6 @@ func GetFeatures() []featuregate.Feature { features.OpenTelemetryCollector, features.UseUnifiedHTTPProxyPort, features.VPAInPlaceUpdates, + features.CustomDNSServerInNodeLocalDNS, } } diff --git a/pkg/gardenlet/operation/botanist/nodelocaldns.go b/pkg/gardenlet/operation/botanist/nodelocaldns.go index 4691d8715e3..067ca6317d4 100644 --- a/pkg/gardenlet/operation/botanist/nodelocaldns.go +++ b/pkg/gardenlet/operation/botanist/nodelocaldns.go @@ -18,6 +18,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" "github.com/gardener/gardener/pkg/component/networking/nodelocaldns" + "github.com/gardener/gardener/pkg/features" imagevectorutils "github.com/gardener/gardener/pkg/utils/imagevector" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" versionutils "github.com/gardener/gardener/pkg/utils/version" @@ -44,14 +45,15 @@ func (b *Botanist) DefaultNodeLocalDNS() (nodelocaldns.Interface, error) { b.SeedClientSet.Client(), b.Shoot.ControlPlaneNamespace, nodelocaldns.Values{ - Image: image.String(), - AlpineImage: imageAlpine.String(), - CorednsConfigAdapterImage: imageCorednsConfigAdapter.String(), - VPAEnabled: b.Shoot.WantsVerticalPodAutoscaler, - Config: v1beta1helper.GetNodeLocalDNS(b.Shoot.GetInfo().Spec.SystemComponents), - Workers: b.Shoot.GetInfo().Spec.Provider.Workers, - KubeProxyConfig: b.Shoot.GetInfo().Spec.Kubernetes.KubeProxy, - Log: b.Logger, + Image: image.String(), + AlpineImage: imageAlpine.String(), + CorednsConfigAdapterImage: imageCorednsConfigAdapter.String(), + VPAEnabled: b.Shoot.WantsVerticalPodAutoscaler, + Config: v1beta1helper.GetNodeLocalDNS(b.Shoot.GetInfo().Spec.SystemComponents), + Workers: b.Shoot.GetInfo().Spec.Provider.Workers, + KubeProxyConfig: b.Shoot.GetInfo().Spec.Kubernetes.KubeProxy, + Log: b.Logger, + CustomDNSServerInNodeLocalDNS: features.DefaultFeatureGate.Enabled(features.CustomDNSServerInNodeLocalDNS), }, ), nil } From 0932f63c694a15b21b45c658acc007024e6505c1 Mon Sep 17 00:00:00 2001 From: Gardener Prow Robot Date: Tue, 25 Nov 2025 01:43:17 +0100 Subject: [PATCH 156/176] Update module github.com/docker/cli to v29.0.4+incompatible (#13521) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5aa9adaafe0..581b6083aaf 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/containerd/errdefs v1.0.0 github.com/coreos/go-systemd/v22 v22.6.0 github.com/distribution/distribution/v3 v3.0.0 - github.com/docker/cli v29.0.2+incompatible + github.com/docker/cli v29.0.4+incompatible github.com/fluent/fluent-operator/v3 v3.5.0 github.com/gardener/cert-management v0.19.0 github.com/gardener/dependency-watchdog v1.6.0 diff --git a/go.sum b/go.sum index 835b47dbbe0..7ce3252462f 100644 --- a/go.sum +++ b/go.sum @@ -188,8 +188,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/cli v29.0.2+incompatible h1:iLuKy2GWOSLXGp8feLYBJQVDv7m/8xoofz6lPq41x6A= -github.com/docker/cli v29.0.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.0.4+incompatible h1:mffN/hPqaI39vx/4QiSkdldHeM0rP1ZZBIXRUOPI5+I= +github.com/docker/cli v29.0.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= From cf6796b403dc8db69f564f171bdb5c526a0f6481 Mon Sep 17 00:00:00 2001 From: Raphael Vogel Date: Tue, 25 Nov 2025 07:15:18 +0100 Subject: [PATCH 157/176] Add privacy notice to templates (#13527) --- .github/ISSUE_TEMPLATE/bug.md | 2 ++ .github/ISSUE_TEMPLATE/feature.md | 2 ++ .github/ISSUE_TEMPLATE/flaking-test.md | 2 ++ .github/pull_request_template.md | 2 ++ 4 files changed, 8 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 682f0df59da..2a96dce7e65 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -4,6 +4,8 @@ about: Report a bug encountered while operating Gardener --- + + **How to categorize this issue?** + **How to categorize this issue?** + **How to categorize this issue?** diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 25a50e144bd..b4f5bddd70f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,5 @@ + + **How to categorize this PR?**