cec: support for explicit control of Cilium Envoy filter injection

pkg/annotation/k8s.go

@@ -37,6 +37,9 @@ const (
 	// CNIPrefix is the common prefix for CNI related annotations.
 	CNIPrefix = "cni.cilium.io"
 
+	// CECPrefix is the common prefix for CEC related annotations.
+	CECPrefix = "cec.cilium.io"
+
 	// PodAnnotationMAC is used to store the MAC address of the Pod.
 	PodAnnotationMAC = CNIPrefix + "/mac-address"
 
@@ -198,13 +201,13 @@ const (
 	LBIPAMSharingKeyAlias             = Prefix + "/lb-ipam-sharing-key"
 	LBIPAMSharingAcrossNamespace      = LBIPAMPrefix + "/sharing-cross-namespace"
 	LBIPAMSharingAcrossNamespaceAlias = Prefix + "/lb-ipam-sharing-cross-namespace"
-)
 
-var (
-	// CiliumPrefixRegex is a regex matching Cilium specific annotations.
-	CiliumPrefixRegex = regexp.MustCompile(`^([A-Za-z0-9]+\.)*cilium.io/`)
+	CECInjectCiliumFilters = CECPrefix + "/inject-cilium-filters"
 )
 
+// CiliumPrefixRegex is a regex matching Cilium specific annotations.
+var CiliumPrefixRegex = regexp.MustCompile(`^([A-Za-z0-9]+\.)*cilium.io/`)
+
 // Get returns the annotation value associated with the given key, or any of
 // the additional aliases if not found.
 func Get(obj metav1.Object, key string, aliases ...string) (value string, ok bool) {

pkg/ciliumenvoyconfig/cec_manager.go

@@ -13,6 +13,7 @@ import (
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/cilium/cilium/pkg/annotation"
 	"github.com/cilium/cilium/pkg/envoy"
 	"github.com/cilium/cilium/pkg/k8s"
 	ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
@@ -89,6 +90,7 @@ func (r *cecManager) addCiliumEnvoyConfig(cecObjectMeta metav1.ObjectMeta, cecSp
 		cecObjectMeta.GetName(),
 		cecSpec.Resources,
 		len(cecSpec.Services) > 0,
+		len(cecSpec.Services) > 0,
 		useOriginalSourceAddress(&cecObjectMeta),
 		true,
 	)
@@ -180,7 +182,6 @@ func (r *cecManager) syncAllHeadlessService(name string, namespace string, spec
 		}
 	}
 	return nil
-
 }
 
 func (r *cecManager) syncHeadlessService(name string, namespace string, serviceName loadbalancer.ServiceName, servicePorts []string) error {
@@ -352,6 +353,7 @@ func (r *cecManager) updateCiliumEnvoyConfig(
 		oldCECObjectMeta.GetName(),
 		oldCECSpec.Resources,
 		len(oldCECSpec.Services) > 0,
+		injectCiliumEnvoyFilters(&oldCECObjectMeta, oldCECSpec),
 		useOriginalSourceAddress(&oldCECObjectMeta),
 		false,
 	)
@@ -363,6 +365,7 @@ func (r *cecManager) updateCiliumEnvoyConfig(
 		newCECObjectMeta.GetName(),
 		newCECSpec.Resources,
 		len(newCECSpec.Services) > 0,
+		injectCiliumEnvoyFilters(&newCECObjectMeta, newCECSpec),
 		useOriginalSourceAddress(&newCECObjectMeta),
 		true,
 	)
@@ -466,6 +469,7 @@ func (r *cecManager) deleteCiliumEnvoyConfig(cecObjectMeta metav1.ObjectMeta, ce
 		cecObjectMeta.GetName(),
 		cecSpec.Resources,
 		len(cecSpec.Services) > 0,
+		injectCiliumEnvoyFilters(&cecObjectMeta, cecSpec),
 		useOriginalSourceAddress(&cecObjectMeta),
 		false,
 	)
@@ -591,3 +595,18 @@ func useOriginalSourceAddress(meta *metav1.ObjectMeta) bool {
 
 	return true
 }
+
+// injectCiliumEnvoyFilters returns true if the given object indicates that Cilium Envoy Network- and L7 filters
+// should be added to all non-internal Listeners.
+// This can be an explicit annotation or the implicit presence of a L7LB service via the Services property.
+func injectCiliumEnvoyFilters(meta *metav1.ObjectMeta, spec *ciliumv2.CiliumEnvoyConfigSpec) bool {
+	if meta.GetAnnotations() != nil {
+		if v, ok := meta.GetAnnotations()[annotation.CECInjectCiliumFilters]; ok {
+			if boolValue, err := strconv.ParseBool(v); err == nil {
+				return boolValue
+			}
+		}
+	}
+
+	return len(spec.Services) > 0
+}

pkg/ciliumenvoyconfig/cec_manager_test.go

@@ -13,8 +13,10 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 
+	"github.com/cilium/cilium/pkg/annotation"
 	cmtypes "github.com/cilium/cilium/pkg/clustermesh/types"
 	"github.com/cilium/cilium/pkg/k8s"
 	cilium_v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
@@ -76,7 +78,7 @@ func TestParseEnvoySpec(t *testing.T) {
 	assert.Equal(t, "type.googleapis.com/envoy.config.listener.v3.Listener", cec.Spec.Resources[0].TypeUrl)
 	assert.True(t, useOriginalSourceAddress(&cec.ObjectMeta))
 
-	resources, err := parser.parseResources("", "name", cec.Spec.Resources, len(cec.Spec.Services) > 0, useOriginalSourceAddress(&cec.ObjectMeta), true)
+	resources, err := parser.parseResources("", "name", cec.Spec.Resources, len(cec.Spec.Services) > 0, injectCiliumEnvoyFilters(&cec.ObjectMeta, &cec.Spec), useOriginalSourceAddress(&cec.ObjectMeta), true)
 	assert.NoError(t, err)
 	assert.Len(t, resources.Listeners, 1)
 	assert.Equal(t, uint32(10000), resources.Listeners[0].Address.GetSocketAddress().GetPortValue())
@@ -156,7 +158,7 @@ func TestParseEnvoySpecWithService(t *testing.T) {
 	assert.Equal(t, "type.googleapis.com/envoy.config.listener.v3.Listener", cec.Spec.Resources[0].TypeUrl)
 	assert.True(t, useOriginalSourceAddress(&cec.ObjectMeta))
 
-	resources, err := parser.parseResources("", "name", cec.Spec.Resources, len(cec.Spec.Services) > 0, useOriginalSourceAddress(&cec.ObjectMeta), true)
+	resources, err := parser.parseResources("", "name", cec.Spec.Resources, len(cec.Spec.Services) > 0, injectCiliumEnvoyFilters(&cec.ObjectMeta, &cec.Spec), useOriginalSourceAddress(&cec.ObjectMeta), true)
 	assert.NoError(t, err)
 	assert.Len(t, resources.Listeners, 1)
 	assert.Equal(t, uint32(1025), resources.Listeners[0].Address.GetSocketAddress().GetPortValue())
@@ -508,7 +510,7 @@ func Test_convertToLBService(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			svcs := convertToLBService(tt.args.svc, tt.args.ep)
 			require.Len(t, svcs, len(tt.want))
-			for i := 0; i < len(svcs); i++ {
+			for i := range svcs {
 				require.Equal(t, tt.want[i].Name, svcs[i].Name)
 				require.Equal(t, tt.want[i].Frontend, svcs[i].Frontend)
 				require.Len(t, svcs[i].Backends, len(tt.want[i].Backends))
@@ -517,3 +519,91 @@ func Test_convertToLBService(t *testing.T) {
 		})
 	}
 }
+
+func Test_injectCiliumEnvoyFilters(t *testing.T) {
+	tests := []struct {
+		name string
+		meta *metav1.ObjectMeta
+		spec *cilium_v2.CiliumEnvoyConfigSpec
+		want bool
+	}{
+		{
+			name: "L7LB services defined",
+			meta: &metav1.ObjectMeta{},
+			spec: &cilium_v2.CiliumEnvoyConfigSpec{
+				Services: []*cilium_v2.ServiceListener{{
+					Name: "test",
+				}},
+			},
+			want: true,
+		},
+		{
+			name: "L7LB services defined but override via annotation",
+			meta: &metav1.ObjectMeta{
+				Annotations: map[string]string{
+					annotation.CECInjectCiliumFilters: "false",
+				},
+			},
+			spec: &cilium_v2.CiliumEnvoyConfigSpec{
+				Services: []*cilium_v2.ServiceListener{{
+					Name: "test",
+				}},
+			},
+			want: false,
+		},
+		{
+			name: "No L7LB services but explicit inject via annotation",
+			meta: &metav1.ObjectMeta{
+				Annotations: map[string]string{
+					annotation.CECInjectCiliumFilters: "true",
+				},
+			},
+			spec: &cilium_v2.CiliumEnvoyConfigSpec{
+				Services: []*cilium_v2.ServiceListener{},
+			},
+			want: true,
+		},
+		{
+			name: "L7LB services defined and invalid annotation value",
+			meta: &metav1.ObjectMeta{
+				Annotations: map[string]string{
+					annotation.CECInjectCiliumFilters: "invalid",
+				},
+			},
+			spec: &cilium_v2.CiliumEnvoyConfigSpec{
+				Services: []*cilium_v2.ServiceListener{{
+					Name: "test",
+				}},
+			},
+			want: true,
+		},
+		{
+			name: "No L7LB services and invalid annotation value",
+			meta: &metav1.ObjectMeta{
+				Annotations: map[string]string{
+					annotation.CECInjectCiliumFilters: "invalid",
+				},
+			},
+			spec: &cilium_v2.CiliumEnvoyConfigSpec{
+				Services: []*cilium_v2.ServiceListener{},
+			},
+			want: false,
+		},
+		{
+			name: "No L7LB services and no annotation",
+			meta: &metav1.ObjectMeta{
+				Annotations: map[string]string{},
+			},
+			spec: &cilium_v2.CiliumEnvoyConfigSpec{
+				Services: []*cilium_v2.ServiceListener{},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := injectCiliumEnvoyFilters(tt.meta, tt.spec)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

pkg/ciliumenvoyconfig/cec_resource_parser.go

@@ -119,12 +119,13 @@ type PortAllocator interface {
 // Parameters:
 //   - `cecNamespace` and `cecName` will be prepended to the Envoy resource names.
 //   - `xdsResources` are the resources from the CiliumEnvoyConfig or CiliumClusterwideEnvoyConfig.
-//   - `isL7LB` defines whether these resources are used for L7 loadbalancing. If `true`, the Envoy Cilium Network- and L7 filters are always
-//     added to all non-internal Listeners. In addition, the info gets passed to the Envoy Cilium BPF Metadata listener filter on all Listeners.
+//   - `isL7LB` defines whether these resources are used for L7 loadbalancing. If `true`, the info gets passed to
+//     the Envoy Cilium BPF Metadata listener filter on all Listeners.
+//   - `injecCiliumEnvoyFilters` defines whether the Envoy Cilium Network- and L7 filters should always be added to all non-internal Listeners.
 //   - `useOriginalSourceAddr` is passed to the Envoy Cilium BPF Metadata listener filter on all Listeners.
 //   - `newResources` is passed as `true` when parsing resources that are being added or are the new version of the resources being updated,
 //     and as `false` if the resources are being removed or are the old version of the resources being updated. Only 'new' resources are validated.
-func (r *cecResourceParser) parseResources(cecNamespace string, cecName string, xdsResources []cilium_v2.XDSResource, isL7LB bool, useOriginalSourceAddr bool, newResources bool) (envoy.Resources, error) {
+func (r *cecResourceParser) parseResources(cecNamespace string, cecName string, xdsResources []cilium_v2.XDSResource, isL7LB bool, injectCiliumEnvoyFilters bool, useOriginalSourceAddr bool, newResources bool) (envoy.Resources, error) {
 	// only validate new resources - old ones are already applied
 	validate := newResources
 
@@ -162,13 +163,13 @@ func (r *cecResourceParser) parseResources(cecNamespace string, cecName string,
 			}
 
 			// Only inject Cilium downstream filters if all of the following conditions are fulfilled
-			// * Cilium allocates listener address or it's a listener for a L7 loadbalancer
+			// * Cilium allocates listener address or it's configured to do so
 			// * It's not an internal listener
-			injectCiliumDownstreamFilters := (listener.GetAddress() == nil || isL7LB) && listener.GetInternalListener() == nil
+			injectCiliumDownstreamFilters := (listener.GetAddress() == nil || injectCiliumEnvoyFilters) && listener.GetInternalListener() == nil
 
-			// Also inject upstream filters for L7 LB when injecting the downstream
-			// HTTP enforcement filter for at least one listener
-			if injectCiliumDownstreamFilters && isL7LB {
+			// Also inject upstream filters when injecting the downstream
+			// HTTP enforcement filter for at least one listener.
+			if injectCiliumDownstreamFilters && injectCiliumEnvoyFilters {
 				injectCiliumUpstreamFilters = true
 			}