From c0b1d14e2d798d0026951781b71786167c21749f Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Fri, 26 Jun 2026 08:28:13 +0100 Subject: [PATCH 1/6] feat(gatewayapi): calico-system policy for namespaced data-plane proxies Since deploy.type=GatewayNamespace (#4690) the data-plane envoy proxies run in each Gateway's own namespace, not calico-system. The only calico-system gateway policy selects the controller/certgen pods in calico-system, so the proxies match nothing and have no policy punching through a default-deny tier in the namespaces they now run in. Add a GlobalNetworkPolicy selecting the EG proxy pods (label gateway.envoyproxy.io/owning-gateway-name) so it covers every Gateway namespace with no re-render: DNS + xDS(18000)/Wasm(18002) egress to the controller in calico-system, and all inbound TCP so a managed Gateway serves traffic out of the box under a default-deny tier. Backend egress is left to the user, matching the controller policy. --- pkg/render/gatewayapi/gateway_api.go | 65 +++++++++++++++++++++++ pkg/render/gatewayapi/gateway_api_test.go | 5 +- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index 145b9daaac..391ae38842 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -75,6 +75,12 @@ const ( ControllerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "envoy-gateway" EnvoyGatewayPolicySelector = "k8s-app == '" + GatewayControllerLabel + "' || k8s-app == '" + GatewayCertgenLabel + "'" + + // Data-plane proxies run in each Gateway's own namespace (deploy.type=GatewayNamespace), + // not calico-system, so they need their own policy in the calico-system tier. EnvoyProxy + // stamps gateway.envoyproxy.io/owning-gateway-name on every proxy pod, so use it as the selector. + ProxyPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "envoy-gateway-proxy" + EnvoyProxyPolicySelector = "has(gateway.envoyproxy.io/owning-gateway-name)" ) // gatewayAPIResources defines all of the resources that we expect to read from the rendered Envoy Gateway @@ -486,6 +492,9 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien // Installation's default-deny. if pr.cfg.IncludeV3NetworkPolicy { objs = append(objs, gatewayAPIControllerPolicy(common.CalicoNamespace, openShift)) + // Data-plane proxies live in each Gateway's own namespace; a GlobalNetworkPolicy + // covers them all and auto-extends to new Gateway namespaces without a re-render. + objs = append(objs, gatewayAPIProxyPolicy(openShift)) } // Helm-rendered envoy-gateway controller in calico-system. @@ -1397,3 +1406,59 @@ func gatewayAPIControllerPolicy(namespace string, openShift bool) *v3.NetworkPol }, } } + +// gatewayAPIProxyPolicy lets the data-plane envoy proxies — which run in each +// Gateway's own namespace (deploy.type=GatewayNamespace), not calico-system — +// punch through any default-deny in those namespaces. It is a GlobalNetworkPolicy +// rather than a per-namespace NetworkPolicy fanned out over GatewayNamespaces so +// that it automatically covers new Gateway namespaces with no re-render. +func gatewayAPIProxyPolicy(openShift bool) *v3.GlobalNetworkPolicy { + egress := networkpolicy.AppendDNSEgressRules(nil, openShift) + egress = append(egress, + // xDS config (18000) and Wasm module fetch (18002) from the envoy-gateway + // controller in calico-system. The proxy dials the controller — see + // envoyproxy/gateway internal/infrastructure/kubernetes/proxy/resource.go + // (XdsServerHost = ..svc) and internal/xds/bootstrap + // (DefaultXdsServerPort=18000, wasmHTTPServicePort=18002). 18001 is the + // ratelimit→controller SotW path, not a proxy path, so it is omitted here. + v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + NamespaceSelector: "kubernetes.io/metadata.name == '" + common.CalicoNamespace + "'", + Selector: EnvoyGatewayPolicySelector, + Ports: networkpolicy.Ports(18000, 18002), + }, + }, + // Backend/application egress is left to the user. Under a default-deny tier + // the user must allow proxy->backend themselves; until they do, the proxy is + // reachable and configured but returns 503 on the upstream connection. + v3.Rule{Action: v3.Pass}, + ) + + return &v3.GlobalNetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "GlobalNetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{Name: ProxyPolicyName}, + Spec: v3.GlobalNetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: EnvoyProxyPolicySelector, + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + // Allow all inbound TCP from any source (this also covers the 19001 metrics + // scrape). Gateway listener ports are user-defined and dynamic, so a managed + // Gateway has to accept arbitrary ports to serve traffic out of the box, + // including under a default-deny tier. Verified on a cluster: the narrower + // alternative (allow only 19001, then Pass) lets listener ingress fall + // through to the user's default-deny and silently breaks every Gateway in a + // default-deny namespace. The cost of allowing all TCP is that an Allow is + // terminal in this tier, so a user cannot narrow ingress to the proxy with + // their own policy. + Ingress: []v3.Rule{ + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"0.0.0.0/0"}}}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"::/0"}}}, + {Action: v3.Pass}, + }, + Egress: egress, + }, + } +} diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index a29ef6cf3c..9b271e9753 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -160,9 +160,12 @@ var _ = Describe("Gateway API rendering tests", func() { &gapi.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: GatewayClassName}}, } - // V3 NetworkPolicy allowing the controller under the calico-system default-deny tier. + // V3 policies under the calico-system default-deny tier: the controller in + // calico-system, and a cluster-scoped policy for the data-plane proxies that + // now run in each Gateway's own namespace (deploy.type=GatewayNamespace). bootstrapExpected := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: ControllerPolicyName, Namespace: common.CalicoNamespace}}, + &v3.GlobalNetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: ProxyPolicyName}}, } It("should render Gateway API resources from helm chart", func() { From a7b817e7173a43f4a30e1d4dd132220552e19e89 Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Tue, 14 Jul 2026 15:51:38 +0100 Subject: [PATCH 2/6] refactor(gatewayapi): select proxies by a Calico-owned label Per review: instead of selecting the data-plane proxy pods on Envoy Gateway's gateway.envoyproxy.io/owning-gateway-name label, which we do not own and which could change upstream without notice, stamp our own k8s-app=calico-gateway-api-proxy label on the proxy pods through the EnvoyProxy pod spec and point the calico-system-tier proxy policy at that. Any user-supplied pod labels from a custom EnvoyProxy are kept. --- pkg/render/gatewayapi/gateway_api.go | 24 ++++++++++++++++++++--- pkg/render/gatewayapi/gateway_api_test.go | 14 +++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index 391ae38842..5b5eee6e5f 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -77,10 +77,13 @@ const ( EnvoyGatewayPolicySelector = "k8s-app == '" + GatewayControllerLabel + "' || k8s-app == '" + GatewayCertgenLabel + "'" // Data-plane proxies run in each Gateway's own namespace (deploy.type=GatewayNamespace), - // not calico-system, so they need their own policy in the calico-system tier. EnvoyProxy - // stamps gateway.envoyproxy.io/owning-gateway-name on every proxy pod, so use it as the selector. + // not calico-system, so they need their own policy in the calico-system tier. We stamp a + // Calico-owned k8s-app label on every proxy pod (via the EnvoyProxy pod spec, see + // envoyProxyConfig) and select on that, rather than Envoy Gateway's + // gateway.envoyproxy.io/owning-gateway-name label which we do not own and which could + // change upstream without notice. ProxyPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "envoy-gateway-proxy" - EnvoyProxyPolicySelector = "has(gateway.envoyproxy.io/owning-gateway-name)" + EnvoyProxyPolicySelector = "k8s-app == '" + GatewayProxyLabel + "'" ) // gatewayAPIResources defines all of the resources that we expect to read from the rendered Envoy Gateway @@ -112,6 +115,7 @@ const ( GatewayAPIName = "calico-gateway-api" GatewayControllerLabel = GatewayAPIName + "-controller" GatewayCertgenLabel = GatewayAPIName + "-certgen" + GatewayProxyLabel = GatewayAPIName + "-proxy" EnvoyGatewayConfigName = "envoy-gateway-config" EnvoyGatewayConfigKey = "envoy-gateway.yaml" EnvoyGatewayDeploymentContainerName = "envoy-gateway" @@ -865,6 +869,18 @@ func ensureExtraArg(args []string, flag, value string) []string { return append(out, args[sep:]...) } +// ensureGatewayProxyLabel stamps the Calico-owned k8s-app label on the data-plane +// proxy pods so that calico-system-tier policy (see gatewayAPIProxyPolicy) can select +// them by our own label rather than Envoy Gateway's gateway.envoyproxy.io/owning-gateway-name, +// which we do not control and which could change upstream. Any user-supplied pod labels +// carried over from a custom EnvoyProxy are preserved. +func ensureGatewayProxyLabel(pod *envoyapi.KubernetesPodSpec) { + if pod.Labels == nil { + pod.Labels = map[string]string{} + } + pod.Labels["k8s-app"] = GatewayProxyLabel +} + func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className, ns string, envoyProxy *envoyapi.EnvoyProxy, classSpec *operatorv1.GatewayClassSpec) *envoyapi.EnvoyProxy { // Ensure the minimal structure that we need for basic correctness and for the following // customizations. Note, we always create the running EnvoyProxy in our own namespace, even @@ -917,6 +933,7 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className, ns stri envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Pod = &envoyapi.KubernetesPodSpec{} } envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Pod.ImagePullSecrets = secret.GetReferenceList(pr.cfg.PullSecrets) + ensureGatewayProxyLabel(envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Pod) if envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Container == nil { envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Container = &envoyapi.KubernetesContainerSpec{} } @@ -930,6 +947,7 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className, ns stri envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod = &envoyapi.KubernetesPodSpec{} } envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.ImagePullSecrets = secret.GetReferenceList(pr.cfg.PullSecrets) + ensureGatewayProxyLabel(envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod) if envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Container == nil { envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Container = &envoyapi.KubernetesContainerSpec{} } diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index 9b271e9753..c7abeb99b4 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -365,6 +365,9 @@ var _ = Describe("Gateway API rendering tests", func() { proxy, err := rtest.GetResourceOfType[*envoyapi.EnvoyProxy](objsToCreate, GatewayClassName, common.CalicoNamespace) Expect(err).NotTo(HaveOccurred()) Expect(proxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Labels).To(HaveKeyWithValue("g-rural", "urban")) + // Our Calico-owned label is stamped alongside any user-supplied pod labels, so the + // calico-system-tier proxy policy can select the proxy pods by a label we control. + Expect(proxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Labels).To(HaveKeyWithValue("k8s-app", GatewayProxyLabel)) Expect(proxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Annotations).To(HaveKeyWithValue("g-haste", "speed")) Expect(proxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Affinity).To(Equal(affinity)) Expect(proxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.NodeSelector).To(HaveKeyWithValue("g-fast", "slow")) @@ -1677,6 +1680,17 @@ value: Expect(err).NotTo(HaveOccurred()) Expect(policy.Spec.Tier).To(Equal("calico-system")) Expect(policy.Spec.Selector).To(Equal(EnvoyGatewayPolicySelector)) + + // The data-plane proxies run in their own Gateway namespaces, so a cluster-scoped + // GlobalNetworkPolicy covers them. It selects by our Calico-owned label rather than + // Envoy Gateway's owning-gateway-name label, which we do not control. + proxyPolicy, err := rtest.GetResourceOfType[*v3.GlobalNetworkPolicy](objsToCreate, ProxyPolicyName, "") + Expect(err).NotTo(HaveOccurred()) + Expect(proxyPolicy.Spec.Tier).To(Equal("calico-system")) + Expect(proxyPolicy.Spec.Selector).To(Equal(EnvoyProxyPolicySelector)) + Expect(proxyPolicy.Spec.Selector).NotTo(ContainSubstring("owning-gateway-name"), + "proxy policy must select by our Calico label, not Envoy Gateway's owning-gateway-name") + _, err = rtest.GetResourceOfType[*v3.NetworkPolicy](objsToCreate, "calico-system.default-deny", common.CalicoNamespace) Expect(err).To(HaveOccurred(), "must not render default-deny in calico-system") }) From 1d372cdee96d62989e9e8dbd48412cd8b16b18ba Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Tue, 14 Jul 2026 15:58:55 +0100 Subject: [PATCH 3/6] refactor(gatewayapi): consolidate envoy-gateway component labels Put the Calico-owned k8s-app labeling for all three envoy-gateway components in one documented place. The operator-rendered controller and certgen pods go through setGatewayComponentLabel; the runtime-created proxy pods, which the operator never renders, go through the EnvoyProxy pod spec (ensureGatewayProxyLabel). Documents why this lives in the gateway render and not the standard labeler in component.go, which keys off the object name and cannot reach a runtime-created pod. --- pkg/render/gatewayapi/gateway_api.go | 36 ++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index 5b5eee6e5f..c273fac7de 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -794,7 +794,7 @@ func (pr *gatewayAPIImplementationComponent) controllerObjects() []client.Object controllerDeployment.Spec.Template.Spec.ImagePullSecrets = append( controllerDeployment.Spec.Template.Spec.ImagePullSecrets, secret.GetReferenceList(pr.cfg.PullSecrets)...) - controllerDeployment.Spec.Template.Labels["k8s-app"] = GatewayControllerLabel + setGatewayComponentLabel(&controllerDeployment.Spec.Template, GatewayControllerLabel) // Mount the trust bundle on the envoy-gateway controller. The controller pulls // wasm OCI images and may call out to JWT/OIDC providers, both of which need @@ -828,10 +828,7 @@ func (pr *gatewayAPIImplementationComponent) controllerObjects() []client.Object certgenJob.Spec.Template.Spec.ImagePullSecrets = append( certgenJob.Spec.Template.Spec.ImagePullSecrets, secret.GetReferenceList(pr.cfg.PullSecrets)...) - if certgenJob.Spec.Template.Labels == nil { - certgenJob.Spec.Template.Labels = map[string]string{} - } - certgenJob.Spec.Template.Labels["k8s-app"] = GatewayCertgenLabel + setGatewayComponentLabel(&certgenJob.Spec.Template, GatewayCertgenLabel) rcomp.ApplyJobOverrides(certgenJob, pr.cfg.GatewayAPI.Spec.GatewayCertgenJob) objs = append(objs, certgenJob) @@ -869,11 +866,30 @@ func ensureExtraArg(args []string, flag, value string) []string { return append(out, args[sep:]...) } -// ensureGatewayProxyLabel stamps the Calico-owned k8s-app label on the data-plane -// proxy pods so that calico-system-tier policy (see gatewayAPIProxyPolicy) can select -// them by our own label rather than Envoy Gateway's gateway.envoyproxy.io/owning-gateway-name, -// which we do not control and which could change upstream. Any user-supplied pod labels -// carried over from a custom EnvoyProxy are preserved. +// The envoy-gateway components each carry a Calico-owned k8s-app label so the +// calico-system-tier policies can select them by a label we control (rather than a +// chart- or Envoy-Gateway-supplied label that could change upstream). The controller +// and certgen pods are operator-rendered, so we stamp the label on their pod templates +// directly (setGatewayComponentLabel). The proxy pods are created by the envoy-gateway +// controller at runtime, so the operator never renders them; the only operator-owned +// hook is the EnvoyProxy pod spec (ensureGatewayProxyLabel). This is why the label is +// set here in the gateway render rather than through the standard labeler in +// pkg/controller/utils/component.go, which keys off the object name and cannot reach a +// runtime-created pod. + +// setGatewayComponentLabel stamps the Calico-owned k8s-app label on an operator-rendered +// envoy-gateway component's pod template (controller, certgen). Existing template labels +// are preserved. +func setGatewayComponentLabel(template *corev1.PodTemplateSpec, label string) { + if template.Labels == nil { + template.Labels = map[string]string{} + } + template.Labels["k8s-app"] = label +} + +// ensureGatewayProxyLabel stamps the same Calico-owned k8s-app label on the data-plane +// proxy pods, via the EnvoyProxy pod spec, so gatewayAPIProxyPolicy can select them. +// Any user-supplied pod labels carried over from a custom EnvoyProxy are preserved. func ensureGatewayProxyLabel(pod *envoyapi.KubernetesPodSpec) { if pod.Labels == nil { pod.Labels = map[string]string{} From 5123b636f10072a6f6b0f72d1f09afd11d937eab Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Thu, 20 Aug 2026 20:28:55 +0100 Subject: [PATCH 4/6] refactor(gatewayapi): trim comments to the non-obvious why Drop the narration around the proxy label helpers and the proxy policy, keeping only the parts a reader can't get from the code: why we select on our own label, why the proxy pod spec is the only labelling hook, and why the egress Pass is deliberate. --- pkg/render/gatewayapi/gateway_api.go | 66 ++++++----------------- pkg/render/gatewayapi/gateway_api_test.go | 8 --- 2 files changed, 16 insertions(+), 58 deletions(-) diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index c273fac7de..6727d99e93 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -76,12 +76,8 @@ const ( ControllerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "envoy-gateway" EnvoyGatewayPolicySelector = "k8s-app == '" + GatewayControllerLabel + "' || k8s-app == '" + GatewayCertgenLabel + "'" - // Data-plane proxies run in each Gateway's own namespace (deploy.type=GatewayNamespace), - // not calico-system, so they need their own policy in the calico-system tier. We stamp a - // Calico-owned k8s-app label on every proxy pod (via the EnvoyProxy pod spec, see - // envoyProxyConfig) and select on that, rather than Envoy Gateway's - // gateway.envoyproxy.io/owning-gateway-name label which we do not own and which could - // change upstream without notice. + // Selects on a label we own, rather than Envoy Gateway's + // gateway.envoyproxy.io/owning-gateway-name, which upstream is free to change. ProxyPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "envoy-gateway-proxy" EnvoyProxyPolicySelector = "k8s-app == '" + GatewayProxyLabel + "'" ) @@ -492,12 +488,10 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien var objs, objsToDelete []client.Object openShift := pr.cfg.Installation.KubernetesProvider.IsOpenShift() - // Allow policy for the controller in calico-system to punch through the core - // Installation's default-deny. + // Allow policies for the controller and the data-plane proxies, to punch through the + // core Installation's default-deny. if pr.cfg.IncludeV3NetworkPolicy { objs = append(objs, gatewayAPIControllerPolicy(common.CalicoNamespace, openShift)) - // Data-plane proxies live in each Gateway's own namespace; a GlobalNetworkPolicy - // covers them all and auto-extends to new Gateway namespaces without a re-render. objs = append(objs, gatewayAPIProxyPolicy(openShift)) } @@ -866,20 +860,6 @@ func ensureExtraArg(args []string, flag, value string) []string { return append(out, args[sep:]...) } -// The envoy-gateway components each carry a Calico-owned k8s-app label so the -// calico-system-tier policies can select them by a label we control (rather than a -// chart- or Envoy-Gateway-supplied label that could change upstream). The controller -// and certgen pods are operator-rendered, so we stamp the label on their pod templates -// directly (setGatewayComponentLabel). The proxy pods are created by the envoy-gateway -// controller at runtime, so the operator never renders them; the only operator-owned -// hook is the EnvoyProxy pod spec (ensureGatewayProxyLabel). This is why the label is -// set here in the gateway render rather than through the standard labeler in -// pkg/controller/utils/component.go, which keys off the object name and cannot reach a -// runtime-created pod. - -// setGatewayComponentLabel stamps the Calico-owned k8s-app label on an operator-rendered -// envoy-gateway component's pod template (controller, certgen). Existing template labels -// are preserved. func setGatewayComponentLabel(template *corev1.PodTemplateSpec, label string) { if template.Labels == nil { template.Labels = map[string]string{} @@ -887,9 +867,8 @@ func setGatewayComponentLabel(template *corev1.PodTemplateSpec, label string) { template.Labels["k8s-app"] = label } -// ensureGatewayProxyLabel stamps the same Calico-owned k8s-app label on the data-plane -// proxy pods, via the EnvoyProxy pod spec, so gatewayAPIProxyPolicy can select them. -// Any user-supplied pod labels carried over from a custom EnvoyProxy are preserved. +// Proxy pods are created by the envoy-gateway controller at runtime, so the EnvoyProxy +// pod spec is the only place the operator gets to label them. func ensureGatewayProxyLabel(pod *envoyapi.KubernetesPodSpec) { if pod.Labels == nil { pod.Labels = map[string]string{} @@ -1441,20 +1420,13 @@ func gatewayAPIControllerPolicy(namespace string, openShift bool) *v3.NetworkPol } } -// gatewayAPIProxyPolicy lets the data-plane envoy proxies — which run in each -// Gateway's own namespace (deploy.type=GatewayNamespace), not calico-system — -// punch through any default-deny in those namespaces. It is a GlobalNetworkPolicy -// rather than a per-namespace NetworkPolicy fanned out over GatewayNamespaces so -// that it automatically covers new Gateway namespaces with no re-render. +// gatewayAPIProxyPolicy lets the data-plane envoy proxies punch through any default-deny +// in the Gateway namespaces they run in (deploy.type=GatewayNamespace). func gatewayAPIProxyPolicy(openShift bool) *v3.GlobalNetworkPolicy { egress := networkpolicy.AppendDNSEgressRules(nil, openShift) egress = append(egress, - // xDS config (18000) and Wasm module fetch (18002) from the envoy-gateway - // controller in calico-system. The proxy dials the controller — see - // envoyproxy/gateway internal/infrastructure/kubernetes/proxy/resource.go - // (XdsServerHost = ..svc) and internal/xds/bootstrap - // (DefaultXdsServerPort=18000, wasmHTTPServicePort=18002). 18001 is the - // ratelimit→controller SotW path, not a proxy path, so it is omitted here. + // xDS config (18000) and Wasm module fetch (18002) on the envoy-gateway controller. + // 18001 is the ratelimit->controller path, which no proxy dials. v3.Rule{ Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, @@ -1464,9 +1436,8 @@ func gatewayAPIProxyPolicy(openShift bool) *v3.GlobalNetworkPolicy { Ports: networkpolicy.Ports(18000, 18002), }, }, - // Backend/application egress is left to the user. Under a default-deny tier - // the user must allow proxy->backend themselves; until they do, the proxy is - // reachable and configured but returns 503 on the upstream connection. + // Pass, not an end-of-tier deny: proxy->backend egress is the user's call, so hand + // it down to their tier. Until they allow it the proxy answers 503. v3.Rule{Action: v3.Pass}, ) @@ -1478,15 +1449,10 @@ func gatewayAPIProxyPolicy(openShift bool) *v3.GlobalNetworkPolicy { Tier: networkpolicy.CalicoTierName, Selector: EnvoyProxyPolicySelector, Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - // Allow all inbound TCP from any source (this also covers the 19001 metrics - // scrape). Gateway listener ports are user-defined and dynamic, so a managed - // Gateway has to accept arbitrary ports to serve traffic out of the box, - // including under a default-deny tier. Verified on a cluster: the narrower - // alternative (allow only 19001, then Pass) lets listener ingress fall - // through to the user's default-deny and silently breaks every Gateway in a - // default-deny namespace. The cost of allowing all TCP is that an Allow is - // terminal in this tier, so a user cannot narrow ingress to the proxy with - // their own policy. + // Listener ports are user-defined and dynamic, so the proxy has to accept + // arbitrary inbound TCP to serve traffic at all (this also covers the 19001 + // metrics scrape). The Allow is terminal in this tier, so a user cannot + // narrow proxy ingress with a policy of their own. Ingress: []v3.Rule{ {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"0.0.0.0/0"}}}, {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"::/0"}}}, diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index c7abeb99b4..d82b93fa88 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -160,9 +160,6 @@ var _ = Describe("Gateway API rendering tests", func() { &gapi.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: GatewayClassName}}, } - // V3 policies under the calico-system default-deny tier: the controller in - // calico-system, and a cluster-scoped policy for the data-plane proxies that - // now run in each Gateway's own namespace (deploy.type=GatewayNamespace). bootstrapExpected := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: ControllerPolicyName, Namespace: common.CalicoNamespace}}, &v3.GlobalNetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: ProxyPolicyName}}, @@ -365,8 +362,6 @@ var _ = Describe("Gateway API rendering tests", func() { proxy, err := rtest.GetResourceOfType[*envoyapi.EnvoyProxy](objsToCreate, GatewayClassName, common.CalicoNamespace) Expect(err).NotTo(HaveOccurred()) Expect(proxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Labels).To(HaveKeyWithValue("g-rural", "urban")) - // Our Calico-owned label is stamped alongside any user-supplied pod labels, so the - // calico-system-tier proxy policy can select the proxy pods by a label we control. Expect(proxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Labels).To(HaveKeyWithValue("k8s-app", GatewayProxyLabel)) Expect(proxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Annotations).To(HaveKeyWithValue("g-haste", "speed")) Expect(proxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Affinity).To(Equal(affinity)) @@ -1681,9 +1676,6 @@ value: Expect(policy.Spec.Tier).To(Equal("calico-system")) Expect(policy.Spec.Selector).To(Equal(EnvoyGatewayPolicySelector)) - // The data-plane proxies run in their own Gateway namespaces, so a cluster-scoped - // GlobalNetworkPolicy covers them. It selects by our Calico-owned label rather than - // Envoy Gateway's owning-gateway-name label, which we do not control. proxyPolicy, err := rtest.GetResourceOfType[*v3.GlobalNetworkPolicy](objsToCreate, ProxyPolicyName, "") Expect(err).NotTo(HaveOccurred()) Expect(proxyPolicy.Spec.Tier).To(Equal("calico-system")) From 3b16b346efefcdd389d6a425f180561fff1f7ddb Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Thu, 3 Sep 2026 16:09:44 +0100 Subject: [PATCH 5/6] refactor(gatewayapi): scope the proxy policy per namespace and per listener port Replace the cluster-wide GlobalNetworkPolicy with a NetworkPolicy in each Gateway namespace, and allow ingress only on the listener ports the Gateways there declare plus the fixed metrics port. The controller already lists every Gateway to build GatewayNamespaces, so the ports come off the same list. --- .../gatewayapi/gatewayapi_controller.go | 37 +++++++++++--- .../gatewayapi/gatewayapi_controller_test.go | 42 ++++++++++++++++ pkg/render/gatewayapi/gateway_api.go | 43 ++++++++++------ pkg/render/gatewayapi/gateway_api_test.go | 49 ++++++++++++++++++- 4 files changed, 148 insertions(+), 23 deletions(-) diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index 06819a95ca..d4be79f6c5 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -506,13 +506,7 @@ func (r *ReconcileGatewayAPI) Reconcile(ctx context.Context, request reconcile.R ownedClass[gcList.Items[i].Name] = true } } - nsSet := set.New[string]() - for i := range gwList.Items { - if ownedClass[string(gwList.Items[i].Spec.GatewayClassName)] { - nsSet.Insert(gwList.Items[i].Namespace) - } - } - gatewayConfig.GatewayNamespaces = nsSet.SortedList() + gatewayConfig.GatewayNamespaces, gatewayConfig.GatewayListenerPorts = gatewayNamespacesAndPorts(gwList.Items, ownedClass) // Legacy tigera-gateway teardown (TODO: remove once upgrades from 3.x are unsupported). // Foreground-delete the old controller so its Deployment lingers until its pods are gone — only @@ -687,6 +681,35 @@ func (r *ReconcileGatewayAPI) maintainFinalizer(ctx context.Context, gatewayAPI return utils.MaintainInstallationFinalizer(ctx, r.client, gatewayAPI, render.GatewayAPIFinalizer, &gatewayAPIDeployment) } +// gatewayNamespacesAndPorts reports the namespaces holding a Gateway of one of our +// classes, and for each the sorted listener ports its Gateways declare. The proxy policy +// allows ingress on exactly those ports, so a user editing a listener is what widens or +// narrows it. Both results are sorted so an unchanged cluster renders an unchanged policy. +func gatewayNamespacesAndPorts(gateways []gapi.Gateway, ownedClass map[string]bool) ([]string, map[string][]uint16) { + nsSet := set.New[string]() + portSets := map[string]set.Set[uint16]{} + for i := range gateways { + gw := &gateways[i] + if !ownedClass[string(gw.Spec.GatewayClassName)] { + continue + } + nsSet.Insert(gw.Namespace) + ports, ok := portSets[gw.Namespace] + if !ok { + ports = set.New[uint16]() + portSets[gw.Namespace] = ports + } + for j := range gw.Spec.Listeners { + ports.Insert(uint16(gw.Spec.Listeners[j].Port)) + } + } + listenerPorts := make(map[string][]uint16, len(portSets)) + for ns, ports := range portSets { + listenerPorts[ns] = ports.SortedList() + } + return nsSet.SortedList(), listenerPorts +} + // reconcileGatewayNamespaceResources writes the per-namespace resources owned by the namespace's // Gateways, so the GC removes them once the last Gateway is gone (and the GatewayAPI CR's deletion // doesn't strand them). Reserved namespaces are skipped; the trust bundle and operator-secrets diff --git a/pkg/controller/gatewayapi/gatewayapi_controller_test.go b/pkg/controller/gatewayapi/gatewayapi_controller_test.go index 2389bdb29e..139aa663f1 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller_test.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller_test.go @@ -748,6 +748,48 @@ var _ = Describe("Gateway API controller tests", func() { Expect(c.Get(ctx, client.ObjectKey{Name: gatewayapi.GatewayClassName}, &gapi.GatewayClass{})).NotTo(HaveOccurred()) }) + It("collects each Gateway namespace's listener ports, deduplicated and sorted", func() { + gateways := []gapi.Gateway{ + {ObjectMeta: metav1.ObjectMeta{Namespace: "app-ns", Name: "gw1"}, Spec: gapi.GatewaySpec{ + GatewayClassName: gatewayapi.GatewayClassName, + Listeners: []gapi.Listener{{Port: 443}, {Port: 80}}, + }}, + // A second Gateway in the same namespace adds to the set, and the port it + // shares with gw1 must not appear twice. + {ObjectMeta: metav1.ObjectMeta{Namespace: "app-ns", Name: "gw2"}, Spec: gapi.GatewaySpec{ + GatewayClassName: gatewayapi.GatewayClassName, + Listeners: []gapi.Listener{{Port: 443}, {Port: 8080}}, + }}, + {ObjectMeta: metav1.ObjectMeta{Namespace: "blog-ns", Name: "gw3"}, Spec: gapi.GatewaySpec{ + GatewayClassName: gatewayapi.GatewayClassName, + Listeners: []gapi.Listener{{Port: 8443}}, + }}, + // Someone else's class, so neither its namespace nor its port is ours. + {ObjectMeta: metav1.ObjectMeta{Namespace: "other-ns", Name: "gw4"}, Spec: gapi.GatewaySpec{ + GatewayClassName: "not-ours", + Listeners: []gapi.Listener{{Port: 9999}}, + }}, + } + + namespaces, ports := gatewayNamespacesAndPorts(gateways, map[string]bool{gatewayapi.GatewayClassName: true}) + Expect(namespaces).To(Equal([]string{"app-ns", "blog-ns"})) + Expect(ports).To(Equal(map[string][]uint16{ + "app-ns": {80, 443, 8080}, + "blog-ns": {8443}, + })) + }) + + It("reports no namespaces and no ports when a Gateway has no listeners", func() { + gateways := []gapi.Gateway{ + {ObjectMeta: metav1.ObjectMeta{Namespace: "app-ns", Name: "gw1"}, Spec: gapi.GatewaySpec{ + GatewayClassName: gatewayapi.GatewayClassName, + }}, + } + namespaces, ports := gatewayNamespacesAndPorts(gateways, map[string]bool{gatewayapi.GatewayClassName: true}) + Expect(namespaces).To(Equal([]string{"app-ns"})) + Expect(ports).To(Equal(map[string][]uint16{"app-ns": {}})) + }) + It("writes the bundle and operator-secrets RoleBinding, but not WAF resources, on Calico", func() { r.newComponentHandler = utils.NewComponentHandler r.ext = extensions.Extensions{}.GatewayAPI() diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index fa6d01e9ee..662ea7b138 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -76,6 +76,10 @@ const ( // gateway.envoyproxy.io/owning-gateway-name, which upstream is free to change. ProxyPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "envoy-gateway-proxy" EnvoyProxyPolicySelector = "k8s-app == '" + GatewayProxyLabel + "'" + + // Envoy Gateway fixes the proxy's Prometheus port, and it is never a Gateway + // listener, so the proxy policy has to allow it by hand. + proxyMetricsPort = 19001 ) // Component names, which key the image overrides a variant resolves through. @@ -375,6 +379,12 @@ type GatewayAPIImplementationConfig struct { // this operator. GatewayNamespaces []string + // GatewayListenerPorts maps each Gateway namespace to the sorted, deduplicated + // listener ports the Gateways in it declare. The proxy policy allows ingress only + // on these, so a port the user never asked to serve on still falls through to + // their own tiers. + GatewayListenerPorts map[string][]uint16 + // TrustedBundle carries the public CA bundle (extracted from the operator's UBI // base image) plus Calico's internal CA. Mounted on the envoy-gateway controller // and on every provisioned envoy-proxy pod so outbound TLS (OCI wasm fetch, @@ -442,7 +452,9 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien // core Installation's default-deny. if pr.cfg.IncludeV3NetworkPolicy { objs = append(objs, gatewayAPIControllerPolicy(common.CalicoNamespace, openShift)) - objs = append(objs, gatewayAPIProxyPolicy(openShift)) + for _, ns := range pr.cfg.GatewayNamespaces { + objs = append(objs, gatewayAPIProxyPolicy(ns, pr.cfg.GatewayListenerPorts[ns], openShift)) + } } // Helm-rendered envoy-gateway controller in calico-system. @@ -1085,9 +1097,10 @@ func gatewayAPIControllerPolicy(namespace string, openShift bool) *v3.NetworkPol } } -// gatewayAPIProxyPolicy lets the data-plane envoy proxies punch through any default-deny -// in the Gateway namespaces they run in (deploy.type=GatewayNamespace). -func gatewayAPIProxyPolicy(openShift bool) *v3.GlobalNetworkPolicy { +// gatewayAPIProxyPolicy lets the data-plane envoy proxies in ns punch through any +// default-deny there (deploy.type=GatewayNamespace). It is namespaced, not global, so a +// pod outside a Gateway namespace cannot pick up these rules by wearing the proxy label. +func gatewayAPIProxyPolicy(ns string, listenerPorts []uint16, openShift bool) *v3.NetworkPolicy { egress := networkpolicy.AppendDNSEgressRules(nil, openShift) egress = append(egress, // xDS config (18000) and Wasm module fetch (18002) on the envoy-gateway controller. @@ -1106,21 +1119,23 @@ func gatewayAPIProxyPolicy(openShift bool) *v3.GlobalNetworkPolicy { v3.Rule{Action: v3.Pass}, ) - return &v3.GlobalNetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "GlobalNetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{Name: ProxyPolicyName}, - Spec: v3.GlobalNetworkPolicySpec{ + // The metrics port is not a listener, so it never appears in a Gateway spec. + ports := networkpolicy.Ports(append([]uint16{proxyMetricsPort}, listenerPorts...)...) + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{Name: ProxyPolicyName, Namespace: ns}, + Spec: v3.NetworkPolicySpec{ Order: &networkpolicy.HighPrecedenceOrder, Tier: networkpolicy.CalicoTierName, Selector: EnvoyProxyPolicySelector, Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - // Listener ports are user-defined and dynamic, so the proxy has to accept - // arbitrary inbound TCP to serve traffic at all (this also covers the 19001 - // metrics scrape). The Allow is terminal in this tier, so a user cannot - // narrow proxy ingress with a policy of their own. + // Allow only what a Gateway here actually listens on, then Pass, so the user + // keeps control of everything else. The Allow is terminal in this tier, so + // widening it any further would take that control away. Ingress: []v3.Rule{ - {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"0.0.0.0/0"}}}, - {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"::/0"}}}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"0.0.0.0/0"}}, Destination: v3.EntityRule{Ports: ports}}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"::/0"}}, Destination: v3.EntityRule{Ports: ports}}, {Action: v3.Pass}, }, Egress: egress, diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index ab3eba6368..cf151ec617 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -25,6 +25,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/render/common/networkpolicy" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/tls/certificatemanagement" admissionregv1 "k8s.io/api/admissionregistration/v1" @@ -116,9 +117,10 @@ var _ = Describe("Gateway API rendering tests", func() { &gapi.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: GatewayClassName}}, } + // No Gateway anywhere means no proxy pods, so the proxy policy is not part of this + // baseline. It is asserted per namespace instead. bootstrapExpected := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: ControllerPolicyName, Namespace: common.CalicoNamespace}}, - &v3.GlobalNetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: ProxyPolicyName}}, } It("should render Gateway API resources from helm chart", func() { @@ -983,6 +985,8 @@ value: Installation: installation, GatewayAPI: gatewayAPI, IncludeV3NetworkPolicy: true, + GatewayNamespaces: []string{"app-ns"}, + GatewayListenerPorts: map[string][]uint16{"app-ns": {80, 443}}, }) Expect(gatewayCompErr).NotTo(HaveOccurred()) @@ -1035,7 +1039,7 @@ value: Expect(policy.Spec.Tier).To(Equal("calico-system")) Expect(policy.Spec.Selector).To(Equal(EnvoyGatewayPolicySelector)) - proxyPolicy, err := rtest.GetResourceOfType[*v3.GlobalNetworkPolicy](objsToCreate, ProxyPolicyName, "") + proxyPolicy, err := rtest.GetResourceOfType[*v3.NetworkPolicy](objsToCreate, ProxyPolicyName, "app-ns") Expect(err).NotTo(HaveOccurred()) Expect(proxyPolicy.Spec.Tier).To(Equal("calico-system")) Expect(proxyPolicy.Spec.Selector).To(Equal(EnvoyProxyPolicySelector)) @@ -1046,6 +1050,47 @@ value: Expect(err).To(HaveOccurred(), "must not render default-deny in calico-system") }) + It("should scope proxy ingress to the listener ports in each Gateway namespace", func() { + gatewayComp, err := GatewayAPIImplementationComponent(&GatewayAPIImplementationConfig{ + Scheme: testScheme(), + Installation: &operatorv1.InstallationSpec{}, + GatewayAPI: &operatorv1.GatewayAPI{ + Spec: operatorv1.GatewayAPISpec{GatewayClasses: []operatorv1.GatewayClassSpec{{Name: "tigera-gateway-class"}}}, + }, + IncludeV3NetworkPolicy: true, + GatewayNamespaces: []string{"shop", "blog"}, + GatewayListenerPorts: map[string][]uint16{"shop": {80, 443}, "blog": {8080}}, + }) + Expect(err).NotTo(HaveOccurred()) + objsToCreate, _ := gatewayComp.Objects() + + // Nothing cluster-wide, so a pod outside a Gateway namespace cannot pick up + // these rules just by wearing the proxy label. + for _, obj := range objsToCreate { + Expect(obj).NotTo(BeAssignableToTypeOf(&v3.GlobalNetworkPolicy{})) + } + + // 19001 is the metrics port and is never a listener, so every namespace gets it + // on top of whatever its own Gateways declare. + for ns, want := range map[string][]uint16{"shop": {19001, 80, 443}, "blog": {19001, 8080}} { + policy, err := rtest.GetResourceOfType[*v3.NetworkPolicy](objsToCreate, ProxyPolicyName, ns) + Expect(err).NotTo(HaveOccurred(), ns) + Expect(policy.Spec.Ingress).To(HaveLen(3), ns) + + for i, nets := range [][]string{{"0.0.0.0/0"}, {"::/0"}} { + Expect(policy.Spec.Ingress[i].Action).To(Equal(v3.Allow), ns) + Expect(policy.Spec.Ingress[i].Source.Nets).To(Equal(nets), ns) + Expect(policy.Spec.Ingress[i].Destination.Ports).To(ConsistOf(networkpolicy.Ports(want...)), ns) + } + // Pass, so a port the user never asked to serve on still reaches their tiers. + Expect(policy.Spec.Ingress[2]).To(Equal(v3.Rule{Action: v3.Pass}), ns) + } + + // A namespace with no Gateway in it gets no policy at all. + _, err = rtest.GetResourceOfType[*v3.NetworkPolicy](objsToCreate, ProxyPolicyName, "other-ns") + Expect(err).To(HaveOccurred()) + }) + It("should not render any v3 NetworkPolicy when IncludeV3NetworkPolicy is false", func() { gatewayComp, err := GatewayAPIImplementationComponent(&GatewayAPIImplementationConfig{ Scheme: testScheme(), From a4e01ceb5d3bef2c53b5ffafe317830ac3b4972d Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Thu, 3 Sep 2026 16:26:50 +0100 Subject: [PATCH 6/6] fix(gatewayapi): sweep the proxy policy out of namespaces that lose their Gateways mergeState skips a v3.NetworkPolicy whose Spec has not changed, so a second owner reference never merges in and the policy cannot be Gateway-owned. Keep it owned by the GatewayAPI CR and delete the ones left in namespaces that no longer host a Gateway. The sweep reads through an uncached reader, so it does not start an informer over every Calico policy on the cluster. --- .../gatewayapi/gatewayapi_controller.go | 34 +++++++++++- .../gatewayapi/gatewayapi_controller_test.go | 21 +++++++ pkg/render/gatewayapi/gateway_api.go | 14 +++-- pkg/render/gatewayapi/gateway_api_test.go | 55 ++++++++++++------- 4 files changed, 99 insertions(+), 25 deletions(-) diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index d4be79f6c5..55b3935031 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -73,6 +73,7 @@ var log = logf.Log.WithName("controller_gatewayapi") func Add(mgr manager.Manager, opts options.ControllerOptions) error { r := &ReconcileGatewayAPI{ client: mgr.GetClient(), + apiReader: mgr.GetAPIReader(), scheme: mgr.GetScheme(), tierWatchReady: &utils.ReadyFlag{}, status: status.New(mgr.GetClient(), "gatewayapi", opts.KubernetesVersion), @@ -177,7 +178,11 @@ var _ reconcile.Reconciler = &ReconcileGatewayAPI{} // ReconcileGatewayAPI reconciles a GatewayAPI object type ReconcileGatewayAPI struct { - client client.Client + client client.Client + // apiReader reads straight from the API server. The sweep below needs to look at + // NetworkPolicies cluster-wide, and going through the cached client for that would + // start an informer holding every Calico policy on the cluster in memory. + apiReader client.Reader scheme *runtime.Scheme tierWatchReady *utils.ReadyFlag status status.StatusManager @@ -604,6 +609,11 @@ func (r *ReconcileGatewayAPI) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } + if err = r.sweepOrphanedProxyPolicies(ctx, gatewayConfig.GatewayNamespaces); err != nil { + r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error removing orphaned proxy policies", err, log) + return reconcile.Result{}, err + } + // Clear the degraded bit if we've reached this far. r.status.ClearDegraded() @@ -681,6 +691,28 @@ func (r *ReconcileGatewayAPI) maintainFinalizer(ctx context.Context, gatewayAPI return utils.MaintainInstallationFinalizer(ctx, r.client, gatewayAPI, render.GatewayAPIFinalizer, &gatewayAPIDeployment) } +// sweepOrphanedProxyPolicies deletes the proxy policy from namespaces that no longer host +// one of our Gateways. The render owns these by the GatewayAPI CR, not by the Gateways in +// the namespace, so nothing garbage-collects them when the last Gateway there goes away. +// Matching is by our own policy name, so a user's own policies are never touched. +func (r *ReconcileGatewayAPI) sweepOrphanedProxyPolicies(ctx context.Context, gatewayNamespaces []string) error { + policies := &v3.NetworkPolicyList{} + if err := r.apiReader.List(ctx, policies); err != nil { + return err + } + wanted := set.New(gatewayNamespaces...) + for i := range policies.Items { + policy := &policies.Items[i] + if policy.Name != gatewayapi.ProxyPolicyName || wanted.Has(policy.Namespace) { + continue + } + if err := r.client.Delete(ctx, policy); err != nil && !errors.IsNotFound(err) { + return err + } + } + return nil +} + // gatewayNamespacesAndPorts reports the namespaces holding a Gateway of one of our // classes, and for each the sorted listener ports its Gateways declare. The proxy policy // allows ingress on exactly those ports, so a user editing a listener is what widens or diff --git a/pkg/controller/gatewayapi/gatewayapi_controller_test.go b/pkg/controller/gatewayapi/gatewayapi_controller_test.go index 139aa663f1..33cd177592 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller_test.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller_test.go @@ -117,6 +117,7 @@ var _ = Describe("Gateway API controller tests", func() { fakeComponentHandlers = nil r = &ReconcileGatewayAPI{ client: c, + apiReader: c, scheme: scheme, status: mockStatus, variant: operatorv1.CalicoEnterprise, @@ -748,6 +749,26 @@ var _ = Describe("Gateway API controller tests", func() { Expect(c.Get(ctx, client.ObjectKey{Name: gatewayapi.GatewayClassName}, &gapi.GatewayClass{})).NotTo(HaveOccurred()) }) + It("sweeps the proxy policy out of a namespace that no longer hosts a Gateway", func() { + // The render owns these by the GatewayAPI CR, so nothing garbage-collects one when + // the last Gateway in its namespace goes away. Without the sweep a pod there could + // keep the allow by wearing the proxy label. + for _, ns := range []string{"app-ns", "gone-ns"} { + Expect(c.Create(ctx, gatewayapi.ProxyPolicy(ns, []uint16{80}, false))).NotTo(HaveOccurred()) + } + // A policy of the user's own in a namespace we are about to sweep, to prove the + // sweep matches on our name and does not touch anything else. + theirs := gatewayapi.ProxyPolicy("gone-ns", []uint16{80}, false) + theirs.Name = "their-own-policy" + Expect(c.Create(ctx, theirs)).NotTo(HaveOccurred()) + + Expect(r.sweepOrphanedProxyPolicies(ctx, []string{"app-ns"})).NotTo(HaveOccurred()) + + Expect(c.Get(ctx, client.ObjectKey{Namespace: "app-ns", Name: gatewayapi.ProxyPolicyName}, &v3.NetworkPolicy{})).NotTo(HaveOccurred()) + Expect(apierrors.IsNotFound(c.Get(ctx, client.ObjectKey{Namespace: "gone-ns", Name: gatewayapi.ProxyPolicyName}, &v3.NetworkPolicy{}))).To(BeTrue()) + Expect(c.Get(ctx, client.ObjectKey{Namespace: "gone-ns", Name: "their-own-policy"}, &v3.NetworkPolicy{})).NotTo(HaveOccurred()) + }) + It("collects each Gateway namespace's listener ports, deduplicated and sorted", func() { gateways := []gapi.Gateway{ {ObjectMeta: metav1.ObjectMeta{Namespace: "app-ns", Name: "gw1"}, Spec: gapi.GatewaySpec{ diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index 662ea7b138..ee0239a230 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -452,8 +452,12 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien // core Installation's default-deny. if pr.cfg.IncludeV3NetworkPolicy { objs = append(objs, gatewayAPIControllerPolicy(common.CalicoNamespace, openShift)) + // One per Gateway namespace. These are owned by the GatewayAPI CR rather than by + // the namespace's Gateways, because mergeState skips a v3.NetworkPolicy whose Spec + // has not changed (component.go), so a second owner reference would never merge in. + // A namespace that stops hosting Gateways is swept by the controller instead. for _, ns := range pr.cfg.GatewayNamespaces { - objs = append(objs, gatewayAPIProxyPolicy(ns, pr.cfg.GatewayListenerPorts[ns], openShift)) + objs = append(objs, ProxyPolicy(ns, pr.cfg.GatewayListenerPorts[ns], openShift)) } } @@ -1097,10 +1101,10 @@ func gatewayAPIControllerPolicy(namespace string, openShift bool) *v3.NetworkPol } } -// gatewayAPIProxyPolicy lets the data-plane envoy proxies in ns punch through any -// default-deny there (deploy.type=GatewayNamespace). It is namespaced, not global, so a -// pod outside a Gateway namespace cannot pick up these rules by wearing the proxy label. -func gatewayAPIProxyPolicy(ns string, listenerPorts []uint16, openShift bool) *v3.NetworkPolicy { +// ProxyPolicy lets the data-plane envoy proxies in ns punch through any default-deny +// there (deploy.type=GatewayNamespace). It is namespaced, not global, so a pod outside a +// Gateway namespace cannot pick up these rules by wearing the proxy label. +func ProxyPolicy(ns string, listenerPorts []uint16, openShift bool) *v3.NetworkPolicy { egress := networkpolicy.AppendDNSEgressRules(nil, openShift) egress = append(egress, // xDS config (18000) and Wasm module fetch (18002) on the envoy-gateway controller. diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index cf151ec617..88b8804ac0 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -1050,7 +1050,34 @@ value: Expect(err).To(HaveOccurred(), "must not render default-deny in calico-system") }) - It("should scope proxy ingress to the listener ports in each Gateway namespace", func() { + It("should scope proxy ingress to the namespace's own listener ports", func() { + // 19001 is the metrics port and is never a listener, so it comes on top of + // whatever the namespace's Gateways declare. + for _, tc := range []struct { + listeners []uint16 + want []uint16 + }{ + {listeners: []uint16{80, 443}, want: []uint16{19001, 80, 443}}, + {listeners: []uint16{8080}, want: []uint16{19001, 8080}}, + {listeners: nil, want: []uint16{19001}}, + } { + policy := ProxyPolicy("app-ns", tc.listeners, false) + Expect(policy.Namespace).To(Equal("app-ns")) + Expect(policy.Spec.Tier).To(Equal("calico-system")) + Expect(policy.Spec.Selector).To(Equal(EnvoyProxyPolicySelector)) + Expect(policy.Spec.Ingress).To(HaveLen(3)) + + for i, nets := range [][]string{{"0.0.0.0/0"}, {"::/0"}} { + Expect(policy.Spec.Ingress[i].Action).To(Equal(v3.Allow)) + Expect(policy.Spec.Ingress[i].Source.Nets).To(Equal(nets)) + Expect(policy.Spec.Ingress[i].Destination.Ports).To(ConsistOf(networkpolicy.Ports(tc.want...))) + } + // Pass, so a port the user never asked to serve on still reaches their tiers. + Expect(policy.Spec.Ingress[2]).To(Equal(v3.Rule{Action: v3.Pass})) + } + }) + + It("should render one proxy policy per Gateway namespace and nothing cluster-wide", func() { gatewayComp, err := GatewayAPIImplementationComponent(&GatewayAPIImplementationConfig{ Scheme: testScheme(), Installation: &operatorv1.InstallationSpec{}, @@ -1064,31 +1091,21 @@ value: Expect(err).NotTo(HaveOccurred()) objsToCreate, _ := gatewayComp.Objects() - // Nothing cluster-wide, so a pod outside a Gateway namespace cannot pick up - // these rules just by wearing the proxy label. - for _, obj := range objsToCreate { - Expect(obj).NotTo(BeAssignableToTypeOf(&v3.GlobalNetworkPolicy{})) - } - - // 19001 is the metrics port and is never a listener, so every namespace gets it - // on top of whatever its own Gateways declare. for ns, want := range map[string][]uint16{"shop": {19001, 80, 443}, "blog": {19001, 8080}} { policy, err := rtest.GetResourceOfType[*v3.NetworkPolicy](objsToCreate, ProxyPolicyName, ns) Expect(err).NotTo(HaveOccurred(), ns) - Expect(policy.Spec.Ingress).To(HaveLen(3), ns) - - for i, nets := range [][]string{{"0.0.0.0/0"}, {"::/0"}} { - Expect(policy.Spec.Ingress[i].Action).To(Equal(v3.Allow), ns) - Expect(policy.Spec.Ingress[i].Source.Nets).To(Equal(nets), ns) - Expect(policy.Spec.Ingress[i].Destination.Ports).To(ConsistOf(networkpolicy.Ports(want...)), ns) - } - // Pass, so a port the user never asked to serve on still reaches their tiers. - Expect(policy.Spec.Ingress[2]).To(Equal(v3.Rule{Action: v3.Pass}), ns) + Expect(policy.Spec.Ingress[0].Destination.Ports).To(ConsistOf(networkpolicy.Ports(want...)), ns) } - // A namespace with no Gateway in it gets no policy at all. + // A namespace with no Gateway in it gets nothing. _, err = rtest.GetResourceOfType[*v3.NetworkPolicy](objsToCreate, ProxyPolicyName, "other-ns") Expect(err).To(HaveOccurred()) + + // And nothing cluster-wide, so a pod outside a Gateway namespace cannot pick up + // the rules just by wearing the proxy label. + for _, obj := range objsToCreate { + Expect(obj).NotTo(BeAssignableToTypeOf(&v3.GlobalNetworkPolicy{})) + } }) It("should not render any v3 NetworkPolicy when IncludeV3NetworkPolicy is false", func() {