Dual network filter

WORKSPACE

@@ -37,13 +37,14 @@ git_repository(
     patch_args = ["apply"],
     patch_tool = "git",
     patches = [
-        "@//patches:0001-network-Add-callback-for-upstream-authorization.patch",
-        "@//patches:0002-listener-add-socket-options.patch",
-        "@//patches:0003-original_dst_cluster-Avoid-multiple-hosts-for-the-sa.patch",
-        "@//patches:0004-tcp_proxy-Check-for-nullptr-in-watermark-ASSERTs.patch",
-        "@//patches:0005-thread_local-reset-slot-in-worker-threads-first.patch",
-        "@//patches:0006-http-header-expose-attribute.patch",
-        "@//patches:0007-liburing-arm-build.patch",
+        "@//patches:0001-listener-add-socket-options.patch",
+        "@//patches:0002-original_dst_cluster-Avoid-multiple-hosts-for-the-sa.patch",
+        "@//patches:0003-tcp_proxy-Check-for-nullptr-in-watermark-ASSERTs.patch",
+        "@//patches:0004-thread_local-reset-slot-in-worker-threads-first.patch",
+        "@//patches:0005-Expose-HTTP-Header-matcher-attribute.patch",
+        "@//patches:0006-build-Fix-arm-build-for-liburing.patch",
+        "@//patches:0007-network-Add-filter-callback-onDestinationSelected.patch",
+        "@//patches:0008-network-Compat-for-missing-upstream-filter.patch",
     ],
     # // clang-format off: Envoy's format check: Only repository_locations.bzl may contains URL references
     remote = "https://github.com/envoyproxy/envoy.git",

cilium/filter_state_cilium_policy.cc

@@ -10,6 +10,7 @@
 #include "source/common/common/macros.h"
 
 #include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
 #include "cilium/accesslog.h"
 
 namespace Envoy {
@@ -19,120 +20,170 @@ const std::string& CiliumPolicyFilterState::key() {
   CONSTRUCT_ON_FIRST_USE(std::string, "cilium.policy");
 }
 
-bool CiliumPolicyFilterState::enforceNetworkPolicy(const Network::Connection& conn,
-                                                   uint32_t destination_identity,
-                                                   uint16_t destination_port,
-                                                   const absl::string_view sni,
-                                                   /* OUT */ bool& use_proxy_lib,
-                                                   /* OUT */ std::string& l7_proto,
-                                                   /* INOUT */ AccessLog::Entry& log_entry) const {
+bool CiliumPolicyFilterState::enforcePodNetworkPolicy(const Network::Connection& conn,
+                                                      uint32_t destination_identity,
+                                                      uint16_t destination_port,
+                                                      const absl::string_view sni,
+                                                      /* OUT */ bool& use_proxy_lib,
+                                                      /* OUT */ std::string& l7_proto) const {
+  auto remote_id = ingress_ ? source_identity_ : destination_identity;
+  const auto& policy = policy_resolver_->getPolicy(pod_ip_);
+  auto port = ingress_ ? port_ : destination_port;
+  auto port_policy = policy.findPortPolicy(ingress_, port);
+
   use_proxy_lib = false;
   l7_proto = "";
 
-  // enforce pod policy first, if any
-  if (pod_ip_.length() > 0) {
-    const auto& policy = policy_resolver_->getPolicy(pod_ip_);
-    auto remote_id = ingress_ ? source_identity_ : destination_identity;
-    auto port = ingress_ ? port_ : destination_port;
-
-    auto port_policy = policy.findPortPolicy(ingress_, port);
-
-    if (!port_policy.allowed(proxy_id_, remote_id, sni)) {
-      ENVOY_CONN_LOG(debug, "Pod policy DENY on proxy_id: {} id: {} port: {} sni: \"{}\"", conn,
-                     proxy_id_, remote_id, port, sni);
-      return false;
-    }
-
-    // populate l7proto_ if available
-    use_proxy_lib = port_policy.useProxylib(proxy_id_, remote_id, l7_proto);
+  if (!port_policy.allowed(proxy_id_, remote_id, sni)) {
+    ENVOY_CONN_LOG(debug,
+                   "cilium.network: Pod {} network {} policy DENY on proxy_id: {} id: {} port: {} "
+                   "sni: \"{}\"",
+                   conn, pod_ip_, ingress_ ? "ingress" : "egress", proxy_id_, remote_id,
+                   destination_port, sni);
+    return false;
   }
 
-  // enforce Ingress policy 2nd, if any
-  if (ingress_policy_name_.length() > 0) {
-    log_entry.entry_.set_policy_name(ingress_policy_name_);
-    const auto& policy = policy_resolver_->getPolicy(ingress_policy_name_);
+  // populate l7proto_ if available
+  use_proxy_lib = port_policy.useProxylib(proxy_id_, remote_id, l7_proto);
 
-    // Enforce ingress policy for Ingress, on the original destination port
-    if (ingress_source_identity_ != 0) {
-      auto ingress_port_policy = policy.findPortPolicy(true, port_);
-      if (!ingress_port_policy.allowed(proxy_id_, ingress_source_identity_, sni)) {
-        ENVOY_CONN_LOG(debug,
-                       "Ingress network policy {} DROP for source identity and destination "
-                       "reserved ingress identity: {} proxy_id: {} port: {} sni: \"{}\"",
-                       conn, ingress_policy_name_, ingress_source_identity_, proxy_id_, port_, sni);
-        return false;
-      }
-    }
+  ENVOY_CONN_LOG(debug,
+                 "cilium.network: Pod {} network {} policy ALLOW on proxy_id: {} id: {} port: {} "
+                 "sni: \"{}\"",
+                 conn, pod_ip_, ingress_ ? "ingress" : "egress", proxy_id_, remote_id,
+                 destination_port, sni);
+  return true;
+}
 
-    // Enforce egress policy for Ingress
-    auto egress_port_policy = policy.findPortPolicy(false, destination_port);
-    if (!egress_port_policy.allowed(proxy_id_, destination_identity, sni)) {
-      ENVOY_CONN_LOG(debug,
-                     "Egress network policy {} DROP for reserved ingress identity and destination "
-                     "identity: {} proxy_id: {} port: {} sni: \"{}\"",
-                     conn, ingress_policy_name_, destination_identity, proxy_id_, destination_port,
-                     sni);
+bool CiliumPolicyFilterState::enforceIngressNetworkPolicy(const Network::Connection& conn,
+                                                          uint32_t destination_identity,
+                                                          uint16_t destination_port,
+                                                          const absl::string_view sni) const {
+  const auto& policy = policy_resolver_->getPolicy(ingress_policy_name_);
+
+  // Enforce ingress policy for Ingress, on the original destination port
+  if (ingress_source_identity_ != 0) {
+    auto ingress_port_policy = policy.findPortPolicy(true, port_);
+    if (!ingress_port_policy.allowed(proxy_id_, ingress_source_identity_, sni)) {
+      ENVOY_CONN_LOG(
+          debug,
+          "cilium.network: Ingress {} network ingress policy DENY on proxy_id: {} id: {} "
+          "port: {} sni: \"{}\"",
+          conn, ingress_policy_name_, proxy_id_, ingress_source_identity_, port_, sni);
       return false;
     }
   }
 
-  // Connection allowed by policy
+  // Enforce egress policy for Ingress
+  auto egress_port_policy = policy.findPortPolicy(false, destination_port);
+  if (!egress_port_policy.allowed(proxy_id_, destination_identity, sni)) {
+    ENVOY_CONN_LOG(debug,
+                   "cilium.network: Ingress {} network egress policy DENY on proxy_id: {} "
+                   "id: {} port: {} sni: \"{}\"",
+                   conn, ingress_policy_name_, proxy_id_, destination_identity, destination_port,
+                   sni);
+    return false;
+  }
+
+  ENVOY_CONN_LOG(debug,
+                 "cilium.network: Ingress {} network policy ALLOW on proxy_id: {} id: {} port: {} "
+                 "sni: \"{}\"",
+                 conn, ingress_policy_name_, proxy_id_, destination_identity, destination_port,
+                 sni);
   return true;
 }
 
 bool CiliumPolicyFilterState::enforcePodHTTPPolicy(const Network::Connection& conn,
                                                    uint32_t destination_identity,
                                                    uint16_t destination_port,
                                                    /* INOUT */ Http::RequestHeaderMap& headers,
+                                                   /* INOUT */ LastNetworkPolicyCache& policy_cache,
                                                    /* INOUT */ AccessLog::Entry& log_entry) const {
   const auto& policy = policy_resolver_->getPolicy(pod_ip_);
   auto remote_id = ingress_ ? source_identity_ : destination_identity;
   auto port = ingress_ ? port_ : destination_port;
 
+  uint64_t version = policy.version();
   const auto port_policy = policy.findPortPolicy(ingress_, port);
-  if (!port_policy.hasHttpRules()) {
-    ENVOY_CONN_LOG(debug,
-                   "cilium.l7policy: Pod {} HTTP {} policy enforcement skipped (no HTTP rules) on "
-                   "proxy_id: {} id: {} port: {}",
-                   conn, pod_ip_, ingress_ ? "ingress" : "egress", proxy_id_, remote_id, port);
-    return true;
+  bool has_http_rules = port_policy.hasHttpRules();
+
+  if (!has_http_rules) {
+    absl::optional<bool> opt = policy_cache.previousVerdict(version, remote_id, port);
+    if (opt.has_value()) {
+      bool verdict = opt.value();
+      ENVOY_CONN_LOG(debug,
+                     "cilium.l7policy: Pod {} HTTP {} policy enforcement using cached verdict {} "
+                     "(no HTTP rules) on "
+                     "proxy_id: {} id: {} port: {}",
+                     conn, pod_ip_, ingress_ ? "ingress" : "egress", verdict ? "ALLOW" : "DENY",
+                     proxy_id_, remote_id, port);
+      return verdict;
+    } else {
+      ENVOY_CONN_LOG(debug,
+                     "cilium.l7policy: Pod {} HTTP {} policy cache MISS "
+                     "(version: {}/{}, id: {}/{}, port: {}/{}), not skipping.",
+                     conn, pod_ip_, ingress_ ? "ingress" : "egress", version, policy_cache.version_,
+                     remote_id, policy_cache.identity_, port, policy_cache.port_);
+    }
+  } else {
+    ENVOY_CONN_LOG(debug, "cilium.l7policy: Pod {} HTTP {} policy has HTTP rules, not skipping.",
+                   conn, pod_ip_, ingress_ ? "ingress" : "egress");
   }
 
-  if (!port_policy.allowed(proxy_id_, remote_id, headers, log_entry)) {
+  bool verdict = port_policy.allowed(proxy_id_, remote_id, headers, log_entry);
+  if (!has_http_rules) {
+    policy_cache.update(version, remote_id, port, verdict);
+  }
+  if (verdict == false) {
     ENVOY_CONN_LOG(debug,
                    "cilium.l7policy: Pod {} HTTP {} policy DENY on proxy_id: {} id: {} port: {}",
                    conn, pod_ip_, ingress_ ? "ingress" : "egress", proxy_id_, remote_id, port);
-    return false;
+  } else {
+    ENVOY_CONN_LOG(debug,
+                   "cilium.l7policy: Pod {} HTTP {} policy ALLOW on proxy_id: {} id: {} port: {}",
+                   conn, pod_ip_, ingress_ ? "ingress" : "egress", proxy_id_, remote_id, port);
   }
-
-  // Connection allowed by policy
-  ENVOY_CONN_LOG(debug,
-                 "cilium.l7policy: Pod {} HTTP {} policy ALLOW on proxy_id: {} id: {} port: {}",
-                 conn, pod_ip_, ingress_ ? "ingress" : "egress", proxy_id_, remote_id, port);
-  return true;
+  return verdict;
 }
 
 bool CiliumPolicyFilterState::enforceIngressHTTPPolicy(
     const Network::Connection& conn, uint32_t destination_identity, uint16_t destination_port,
     /* INOUT */ Http::RequestHeaderMap& headers,
+    /* INOUT */ LastNetworkPolicyCache policy_cache[2],
     /* INOUT */ AccessLog::Entry& log_entry) const {
   log_entry.entry_.set_policy_name(ingress_policy_name_);
   log_entry.request_logged_ = false; // we reuse the same entry we used for the pod policy
 
   const auto& policy = policy_resolver_->getPolicy(ingress_policy_name_);
+  uint64_t version = policy.version();
 
   // Enforce ingress policy for Ingress, on the original destination port
   if (ingress_source_identity_ != 0) {
     const auto port_policy = policy.findPortPolicy(true, port_);
-    if (!port_policy.hasHttpRules()) {
-      ENVOY_CONN_LOG(debug,
-                     "cilium.l7policy: Ingress {} HTTP ingress policy enforcement skipped (no HTTP "
-                     "rules) on proxy_id: {} id: {} port: {}",
-                     conn, ingress_policy_name_, proxy_id_, ingress_source_identity_, port_);
-      return true;
+    bool has_http_rules = port_policy.hasHttpRules();
+
+    bool have_verdict = false;
+    bool verdict;
+    if (!has_http_rules) {
+      absl::optional<bool> opt =
+          policy_cache[0].previousVerdict(version, ingress_source_identity_, port_);
+      if (opt.has_value()) {
+        have_verdict = true;
+        verdict = opt.value();
+        ENVOY_CONN_LOG(debug,
+                       "cilium.l7policy: Ingress {} HTTP ingress policy enforcement using using "
+                       "cached verdict (no HTTP rules)",
+                       conn, ingress_policy_name_);
+      }
+    }
+
+    if (!have_verdict) {
+      verdict = port_policy.allowed(proxy_id_, ingress_source_identity_, headers, log_entry);
+      if (!has_http_rules) {
+        policy_cache[0].update(version, ingress_source_identity_, port_, verdict);
+      }
     }
 
-    if (!port_policy.allowed(proxy_id_, ingress_source_identity_, headers, log_entry)) {
+    if (verdict == false) {
       ENVOY_CONN_LOG(
           debug,
           "cilium.l7policy: Ingress {} HTTP ingress policy DROP on proxy_id: {} id: {} port: {}",
@@ -143,15 +194,31 @@ bool CiliumPolicyFilterState::enforceIngressHTTPPolicy(
 
   // Enforce egress policy for Ingress on the upstream destination identity and port
   const auto port_policy = policy.findPortPolicy(false, destination_port);
-  if (!port_policy.hasHttpRules()) {
-    ENVOY_CONN_LOG(debug,
-                   "cilium.l7policy: Ingress {} HTTP egress policy enforcement skipped (no HTTP "
-                   "rules) on proxy_id: {} id: {} port: {}",
-                   conn, ingress_policy_name_, proxy_id_, destination_identity, destination_port);
-    return true;
+  bool has_http_rules = port_policy.hasHttpRules();
+
+  bool have_verdict = false;
+  bool verdict;
+  if (!has_http_rules) {
+    absl::optional<bool> opt =
+        policy_cache[1].previousVerdict(version, destination_identity, destination_port);
+    if (opt.has_value()) {
+      have_verdict = true;
+      verdict = opt.value();
+      ENVOY_CONN_LOG(debug,
+                     "cilium.l7policy: Ingress {} HTTP egress policy enforcement using using "
+                     "cached verdict (no HTTP rules)",
+                     conn, ingress_policy_name_);
+    }
+  }
+
+  if (!have_verdict) {
+    verdict = port_policy.allowed(proxy_id_, destination_identity, headers, log_entry);
+    if (!has_http_rules) {
+      policy_cache[1].update(version, destination_identity, destination_port, verdict);
+    }
   }
 
-  if (!port_policy.allowed(proxy_id_, destination_identity, headers, log_entry)) {
+  if (verdict == false) {
     ENVOY_CONN_LOG(
         debug,
         "cilium.l7policy: Ingress {} HTTP egress policy DROP on proxy_id: {} id: {}  port: {}",

cilium/filter_state_cilium_policy.h

@@ -14,6 +14,7 @@
 #include "source/common/common/logger.h"
 
 #include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
 #include "cilium/accesslog.h"
 #include "cilium/network_policy.h"
 
@@ -29,6 +30,29 @@ class PolicyResolver {
 };
 using PolicyResolverSharedPtr = std::shared_ptr<PolicyResolver>;
 
+// local cache for policy verdicts, which is only used when the policy has no HTTP rules.
+// This eliminates policy lookups when the destination identity and port remain the same
+struct LastNetworkPolicyCache {
+  absl::optional<bool> previousVerdict(uint64_t version, uint32_t identity, uint16_t port) const {
+    if (version == version_ && identity == identity_ && port == port_) {
+      return verdict_;
+    }
+    return absl::nullopt;
+  }
+
+  void update(uint64_t version, uint32_t identity, uint16_t port, bool verdict) {
+    version_ = version;
+    identity_ = identity;
+    port_ = port;
+    verdict_ = verdict;
+  }
+
+  uint64_t version_ = 0;
+  uint32_t identity_ = 0;
+  uint16_t port_ = 0;
+  bool verdict_ = false;
+};
+
 // FilterState that holds relevant connection & policy information that can be retrieved
 // by the Cilium network- and HTTP policy filters via filter state.
 class CiliumPolicyFilterState : public StreamInfo::FilterState::Object,
@@ -56,20 +80,24 @@ class CiliumPolicyFilterState : public StreamInfo::FilterState::Object,
 
   const PolicyInstance& getPolicy() const { return policy_resolver_->getPolicy(pod_ip_); }
 
-  bool enforceNetworkPolicy(const Network::Connection& conn, uint32_t destination_identity,
-                            uint16_t destination_port, const absl::string_view sni,
-                            /* OUT */ bool& use_proxy_lib,
-                            /* OUT */ std::string& l7_proto,
-                            /* INOUT */ AccessLog::Entry& log_entry) const;
+  bool enforcePodNetworkPolicy(const Network::Connection& conn, uint32_t destination_identity,
+                               uint16_t destination_port, const absl::string_view sni,
+                               /* OUT */ bool& use_proxy_lib,
+                               /* OUT */ std::string& l7_proto) const;
+
+  bool enforceIngressNetworkPolicy(const Network::Connection& conn, uint32_t destination_identity,
+                                   uint16_t destination_port, const absl::string_view sni) const;
 
   bool enforcePodHTTPPolicy(const Network::Connection& conn, uint32_t destination_identity,
                             uint16_t destination_port,
                             /* INOUT */ Http::RequestHeaderMap& headers,
+                            /* INOUT */ LastNetworkPolicyCache& policy_cache,
                             /* INOUT */ AccessLog::Entry& log_entry) const;
 
   bool enforceIngressHTTPPolicy(const Network::Connection& conn, uint32_t destination_identity,
                                 uint16_t destination_port,
                                 /* INOUT */ Http::RequestHeaderMap& headers,
+                                /* INOUT */ LastNetworkPolicyCache ingress_cache[2],
                                 /* INOUT */ AccessLog::Entry& log_entry) const;
 
   // policyUseUpstreamDestinationAddress returns 'true' if policy enforcement should be done on the

cilium/ipcache.h

@@ -41,7 +41,7 @@ PACKED_STRUCT(struct IpCacheKey {
   std::string asString() const {
     if (family == ENDPOINT_KEY_IPV4) {
       auto ip = ntohl(ip4);
-      return fmt::format("%d.%d.%d.%d/%d", uint8_t(ip >> 24), uint8_t(ip >> 16), uint8_t(ip >> 8),
+      return fmt::format("{}.{}.{}.{}/{}", uint8_t(ip >> 24), uint8_t(ip >> 16), uint8_t(ip >> 8),
                          uint8_t(ip), lpm_key.prefixlen - 32);
     } else if (family == ENDPOINT_KEY_IPV6) {
       return fmt::format("{:x}:{:x}:{:x}:{:x}/{}", ntohl(ip6[0]), ntohl(ip6[1]), ntohl(ip6[2]),