feat(gatewayapi): calico-system policy for namespaced data-plane proxies

pkg/controller/gatewayapi/gatewayapi_controller.go

@@ -73,6 +73,7 @@ var log = logf.Log.WithName("controller_gatewayapi")
 func Add(mgr manager.Manager, opts options.ControllerOptions) error {
 	r := &ReconcileGatewayAPI{
 		client:              mgr.GetClient(),
+		apiReader:           mgr.GetAPIReader(),
 		scheme:              mgr.GetScheme(),
 		tierWatchReady:      &utils.ReadyFlag{},
 		status:              status.New(mgr.GetClient(), "gatewayapi", opts.KubernetesVersion),
@@ -177,7 +178,11 @@ var _ reconcile.Reconciler = &ReconcileGatewayAPI{}
 
 // ReconcileGatewayAPI reconciles a GatewayAPI object
 type ReconcileGatewayAPI struct {
-	client              client.Client
+	client client.Client
+	// apiReader reads straight from the API server. The sweep below needs to look at
+	// NetworkPolicies cluster-wide, and going through the cached client for that would
+	// start an informer holding every Calico policy on the cluster in memory.
+	apiReader           client.Reader
 	scheme              *runtime.Scheme
 	tierWatchReady      *utils.ReadyFlag
 	status              status.StatusManager
@@ -506,13 +511,7 @@ func (r *ReconcileGatewayAPI) Reconcile(ctx context.Context, request reconcile.R
 			ownedClass[gcList.Items[i].Name] = true
 		}
 	}
-	nsSet := set.New[string]()
-	for i := range gwList.Items {
-		if ownedClass[string(gwList.Items[i].Spec.GatewayClassName)] {
-			nsSet.Insert(gwList.Items[i].Namespace)
-		}
-	}
-	gatewayConfig.GatewayNamespaces = nsSet.SortedList()
+	gatewayConfig.GatewayNamespaces, gatewayConfig.GatewayListenerPorts = gatewayNamespacesAndPorts(gwList.Items, ownedClass)
 
 	// Legacy tigera-gateway teardown (TODO: remove once upgrades from 3.x are unsupported).
 	// Foreground-delete the old controller so its Deployment lingers until its pods are gone — only
@@ -610,6 +609,11 @@ func (r *ReconcileGatewayAPI) Reconcile(ctx context.Context, request reconcile.R
 		return reconcile.Result{}, err
 	}
 
+	if err = r.sweepOrphanedProxyPolicies(ctx, gatewayConfig.GatewayNamespaces); err != nil {
+		r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error removing orphaned proxy policies", err, log)
+		return reconcile.Result{}, err
+	}
+
 	// Clear the degraded bit if we've reached this far.
 	r.status.ClearDegraded()
 
@@ -687,6 +691,57 @@ func (r *ReconcileGatewayAPI) maintainFinalizer(ctx context.Context, gatewayAPI
 	return utils.MaintainInstallationFinalizer(ctx, r.client, gatewayAPI, render.GatewayAPIFinalizer, &gatewayAPIDeployment)
 }
 
+// sweepOrphanedProxyPolicies deletes the proxy policy from namespaces that no longer host
+// one of our Gateways. The render owns these by the GatewayAPI CR, not by the Gateways in
+// the namespace, so nothing garbage-collects them when the last Gateway there goes away.
+// Matching is by our own policy name, so a user's own policies are never touched.
+func (r *ReconcileGatewayAPI) sweepOrphanedProxyPolicies(ctx context.Context, gatewayNamespaces []string) error {
+	policies := &v3.NetworkPolicyList{}
+	if err := r.apiReader.List(ctx, policies); err != nil {
+		return err
+	}
+	wanted := set.New(gatewayNamespaces...)
+	for i := range policies.Items {
+		policy := &policies.Items[i]
+		if policy.Name != gatewayapi.ProxyPolicyName || wanted.Has(policy.Namespace) {
+			continue
+		}
+		if err := r.client.Delete(ctx, policy); err != nil && !errors.IsNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// gatewayNamespacesAndPorts reports the namespaces holding a Gateway of one of our
+// classes, and for each the sorted listener ports its Gateways declare. The proxy policy
+// allows ingress on exactly those ports, so a user editing a listener is what widens or
+// narrows it. Both results are sorted so an unchanged cluster renders an unchanged policy.
+func gatewayNamespacesAndPorts(gateways []gapi.Gateway, ownedClass map[string]bool) ([]string, map[string][]uint16) {
+	nsSet := set.New[string]()
+	portSets := map[string]set.Set[uint16]{}
+	for i := range gateways {
+		gw := &gateways[i]
+		if !ownedClass[string(gw.Spec.GatewayClassName)] {
+			continue
+		}
+		nsSet.Insert(gw.Namespace)
+		ports, ok := portSets[gw.Namespace]
+		if !ok {
+			ports = set.New[uint16]()
+			portSets[gw.Namespace] = ports
+		}
+		for j := range gw.Spec.Listeners {
+			ports.Insert(uint16(gw.Spec.Listeners[j].Port))
+		}
+	}
+	listenerPorts := make(map[string][]uint16, len(portSets))
+	for ns, ports := range portSets {
+		listenerPorts[ns] = ports.SortedList()
+	}
+	return nsSet.SortedList(), listenerPorts
+}
+
 // reconcileGatewayNamespaceResources writes the per-namespace resources owned by the namespace's
 // Gateways, so the GC removes them once the last Gateway is gone (and the GatewayAPI CR's deletion
 // doesn't strand them). Reserved namespaces are skipped; the trust bundle and operator-secrets

pkg/controller/gatewayapi/gatewayapi_controller_test.go

@@ -117,6 +117,7 @@ var _ = Describe("Gateway API controller tests", func() {
 		fakeComponentHandlers = nil
 		r = &ReconcileGatewayAPI{
 			client:              c,
+			apiReader:           c,
 			scheme:              scheme,
 			status:              mockStatus,
 			variant:             operatorv1.CalicoEnterprise,
@@ -748,6 +749,68 @@ var _ = Describe("Gateway API controller tests", func() {
 		Expect(c.Get(ctx, client.ObjectKey{Name: gatewayapi.GatewayClassName}, &gapi.GatewayClass{})).NotTo(HaveOccurred())
 	})
 
+	It("sweeps the proxy policy out of a namespace that no longer hosts a Gateway", func() {
+		// The render owns these by the GatewayAPI CR, so nothing garbage-collects one when
+		// the last Gateway in its namespace goes away. Without the sweep a pod there could
+		// keep the allow by wearing the proxy label.
+		for _, ns := range []string{"app-ns", "gone-ns"} {
+			Expect(c.Create(ctx, gatewayapi.ProxyPolicy(ns, []uint16{80}, false))).NotTo(HaveOccurred())
+		}
+		// A policy of the user's own in a namespace we are about to sweep, to prove the
+		// sweep matches on our name and does not touch anything else.
+		theirs := gatewayapi.ProxyPolicy("gone-ns", []uint16{80}, false)
+		theirs.Name = "their-own-policy"
+		Expect(c.Create(ctx, theirs)).NotTo(HaveOccurred())
+
+		Expect(r.sweepOrphanedProxyPolicies(ctx, []string{"app-ns"})).NotTo(HaveOccurred())
+
+		Expect(c.Get(ctx, client.ObjectKey{Namespace: "app-ns", Name: gatewayapi.ProxyPolicyName}, &v3.NetworkPolicy{})).NotTo(HaveOccurred())
+		Expect(apierrors.IsNotFound(c.Get(ctx, client.ObjectKey{Namespace: "gone-ns", Name: gatewayapi.ProxyPolicyName}, &v3.NetworkPolicy{}))).To(BeTrue())
+		Expect(c.Get(ctx, client.ObjectKey{Namespace: "gone-ns", Name: "their-own-policy"}, &v3.NetworkPolicy{})).NotTo(HaveOccurred())
+	})
+
+	It("collects each Gateway namespace's listener ports, deduplicated and sorted", func() {
+		gateways := []gapi.Gateway{
+			{ObjectMeta: metav1.ObjectMeta{Namespace: "app-ns", Name: "gw1"}, Spec: gapi.GatewaySpec{
+				GatewayClassName: gatewayapi.GatewayClassName,
+				Listeners:        []gapi.Listener{{Port: 443}, {Port: 80}},
+			}},
+			// A second Gateway in the same namespace adds to the set, and the port it
+			// shares with gw1 must not appear twice.
+			{ObjectMeta: metav1.ObjectMeta{Namespace: "app-ns", Name: "gw2"}, Spec: gapi.GatewaySpec{
+				GatewayClassName: gatewayapi.GatewayClassName,
+				Listeners:        []gapi.Listener{{Port: 443}, {Port: 8080}},
+			}},
+			{ObjectMeta: metav1.ObjectMeta{Namespace: "blog-ns", Name: "gw3"}, Spec: gapi.GatewaySpec{
+				GatewayClassName: gatewayapi.GatewayClassName,
+				Listeners:        []gapi.Listener{{Port: 8443}},
+			}},
+			// Someone else's class, so neither its namespace nor its port is ours.
+			{ObjectMeta: metav1.ObjectMeta{Namespace: "other-ns", Name: "gw4"}, Spec: gapi.GatewaySpec{
+				GatewayClassName: "not-ours",
+				Listeners:        []gapi.Listener{{Port: 9999}},
+			}},
+		}
+
+		namespaces, ports := gatewayNamespacesAndPorts(gateways, map[string]bool{gatewayapi.GatewayClassName: true})
+		Expect(namespaces).To(Equal([]string{"app-ns", "blog-ns"}))
+		Expect(ports).To(Equal(map[string][]uint16{
+			"app-ns":  {80, 443, 8080},
+			"blog-ns": {8443},
+		}))
+	})
+
+	It("reports no namespaces and no ports when a Gateway has no listeners", func() {
+		gateways := []gapi.Gateway{
+			{ObjectMeta: metav1.ObjectMeta{Namespace: "app-ns", Name: "gw1"}, Spec: gapi.GatewaySpec{
+				GatewayClassName: gatewayapi.GatewayClassName,
+			}},
+		}
+		namespaces, ports := gatewayNamespacesAndPorts(gateways, map[string]bool{gatewayapi.GatewayClassName: true})
+		Expect(namespaces).To(Equal([]string{"app-ns"}))
+		Expect(ports).To(Equal(map[string][]uint16{"app-ns": {}}))
+	})
+
 	It("writes the bundle and operator-secrets RoleBinding, but not WAF resources, on Calico", func() {
 		r.newComponentHandler = utils.NewComponentHandler
 		r.ext = extensions.Extensions{}.GatewayAPI()

pkg/render/gatewayapi/gateway_api.go

@@ -71,6 +71,15 @@ const (
 
 	ControllerPolicyName       = networkpolicy.CalicoComponentPolicyPrefix + "envoy-gateway"
 	EnvoyGatewayPolicySelector = "k8s-app == '" + GatewayControllerLabel + "' || k8s-app == '" + GatewayCertgenLabel + "'"
+
+	// Selects on a label we own, rather than Envoy Gateway's
+	// gateway.envoyproxy.io/owning-gateway-name, which upstream is free to change.
+	ProxyPolicyName          = networkpolicy.CalicoComponentPolicyPrefix + "envoy-gateway-proxy"
+	EnvoyProxyPolicySelector = "k8s-app == '" + GatewayProxyLabel + "'"
+
+	// Envoy Gateway fixes the proxy's Prometheus port, and it is never a Gateway
+	// listener, so the proxy policy has to allow it by hand.
+	proxyMetricsPort = 19001
 )
 
 // Component names, which key the image overrides a variant resolves through.
@@ -109,6 +118,7 @@ const (
 	GatewayAPIName                      = "calico-gateway-api"
 	GatewayControllerLabel              = GatewayAPIName + "-controller"
 	GatewayCertgenLabel                 = GatewayAPIName + "-certgen"
+	GatewayProxyLabel                   = GatewayAPIName + "-proxy"
 	EnvoyGatewayConfigName              = "envoy-gateway-config"
 	EnvoyGatewayConfigKey               = "envoy-gateway.yaml"
 	EnvoyGatewayDeploymentContainerName = "envoy-gateway"
@@ -369,6 +379,12 @@ type GatewayAPIImplementationConfig struct {
 	// this operator.
 	GatewayNamespaces []string
 
+	// GatewayListenerPorts maps each Gateway namespace to the sorted, deduplicated
+	// listener ports the Gateways in it declare. The proxy policy allows ingress only
+	// on these, so a port the user never asked to serve on still falls through to
+	// their own tiers.
+	GatewayListenerPorts map[string][]uint16
+
 	// TrustedBundle carries the public CA bundle (extracted from the operator's UBI
 	// base image) plus Calico's internal CA. Mounted on the envoy-gateway controller
 	// and on every provisioned envoy-proxy pod so outbound TLS (OCI wasm fetch,
@@ -432,10 +448,17 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien
 	var objs, objsToDelete []client.Object
 	openShift := pr.cfg.Installation.KubernetesProvider.IsOpenShift()
 
-	// Allow policy for the controller in calico-system to punch through the core
-	// Installation's default-deny.
+	// Allow policies for the controller and the data-plane proxies, to punch through the
+	// core Installation's default-deny.
 	if pr.cfg.IncludeV3NetworkPolicy {
 		objs = append(objs, gatewayAPIControllerPolicy(common.CalicoNamespace, openShift))
+		// One per Gateway namespace. These are owned by the GatewayAPI CR rather than by
+		// the namespace's Gateways, because mergeState skips a v3.NetworkPolicy whose Spec
+		// has not changed (component.go), so a second owner reference would never merge in.
+		// A namespace that stops hosting Gateways is swept by the controller instead.
+		for _, ns := range pr.cfg.GatewayNamespaces {
+			objs = append(objs, ProxyPolicy(ns, pr.cfg.GatewayListenerPorts[ns], openShift))
+		}
 	}
 
 	// Helm-rendered envoy-gateway controller in calico-system.
@@ -696,7 +719,7 @@ func (pr *gatewayAPIImplementationComponent) controllerObjects() []client.Object
 	controllerDeployment.Spec.Template.Spec.ImagePullSecrets = append(
 		controllerDeployment.Spec.Template.Spec.ImagePullSecrets,
 		secret.GetReferenceList(pr.cfg.PullSecrets)...)
-	controllerDeployment.Spec.Template.Labels["k8s-app"] = GatewayControllerLabel
+	setGatewayComponentLabel(&controllerDeployment.Spec.Template, GatewayControllerLabel)
 
 	// Mount the trust bundle on the envoy-gateway controller. The controller pulls
 	// wasm OCI images and may call out to JWT/OIDC providers, both of which need
@@ -730,16 +753,29 @@ func (pr *gatewayAPIImplementationComponent) controllerObjects() []client.Object
 	certgenJob.Spec.Template.Spec.ImagePullSecrets = append(
 		certgenJob.Spec.Template.Spec.ImagePullSecrets,
 		secret.GetReferenceList(pr.cfg.PullSecrets)...)
-	if certgenJob.Spec.Template.Labels == nil {
-		certgenJob.Spec.Template.Labels = map[string]string{}
-	}
-	certgenJob.Spec.Template.Labels["k8s-app"] = GatewayCertgenLabel
+	setGatewayComponentLabel(&certgenJob.Spec.Template, GatewayCertgenLabel)
 	rcomp.ApplyJobOverrides(certgenJob, pr.cfg.GatewayAPI.Spec.GatewayCertgenJob)
 	objs = append(objs, certgenJob)
 
 	return objs
 }
 
+func setGatewayComponentLabel(template *corev1.PodTemplateSpec, label string) {
+	if template.Labels == nil {
+		template.Labels = map[string]string{}
+	}
+	template.Labels["k8s-app"] = label
+}
+
+// Proxy pods are created by the envoy-gateway controller at runtime, so the EnvoyProxy
+// pod spec is the only place the operator gets to label them.
+func ensureGatewayProxyLabel(pod *envoyapi.KubernetesPodSpec) {
+	if pod.Labels == nil {
+		pod.Labels = map[string]string{}
+	}
+	pod.Labels["k8s-app"] = GatewayProxyLabel
+}
+
 func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className, ns string, envoyProxy *envoyapi.EnvoyProxy, classSpec *operatorv1.GatewayClassSpec) *envoyapi.EnvoyProxy {
 	// Ensure the minimal structure that we need for basic correctness and for the following
 	// customizations.  Note, we always create the running EnvoyProxy in our own namespace, even
@@ -792,6 +828,7 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className, ns stri
 			envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Pod = &envoyapi.KubernetesPodSpec{}
 		}
 		envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Pod.ImagePullSecrets = secret.GetReferenceList(pr.cfg.PullSecrets)
+		ensureGatewayProxyLabel(envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Pod)
 		if envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Container == nil {
 			envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet.Container = &envoyapi.KubernetesContainerSpec{}
 		}
@@ -805,6 +842,7 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className, ns stri
 			envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod = &envoyapi.KubernetesPodSpec{}
 		}
 		envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.ImagePullSecrets = secret.GetReferenceList(pr.cfg.PullSecrets)
+		ensureGatewayProxyLabel(envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod)
 		if envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Container == nil {
 			envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Container = &envoyapi.KubernetesContainerSpec{}
 		}
@@ -1062,3 +1100,49 @@ func gatewayAPIControllerPolicy(namespace string, openShift bool) *v3.NetworkPol
 		},
 	}
 }
+
+// ProxyPolicy lets the data-plane envoy proxies in ns punch through any default-deny
+// there (deploy.type=GatewayNamespace). It is namespaced, not global, so a pod outside a
+// Gateway namespace cannot pick up these rules by wearing the proxy label.
+func ProxyPolicy(ns string, listenerPorts []uint16, openShift bool) *v3.NetworkPolicy {
+	egress := networkpolicy.AppendDNSEgressRules(nil, openShift)
+	egress = append(egress,
+		// xDS config (18000) and Wasm module fetch (18002) on the envoy-gateway controller.
+		// 18001 is the ratelimit->controller path, which no proxy dials.
+		v3.Rule{
+			Action:   v3.Allow,
+			Protocol: &networkpolicy.TCPProtocol,
+			Destination: v3.EntityRule{
+				NamespaceSelector: "kubernetes.io/metadata.name == '" + common.CalicoNamespace + "'",
+				Selector:          EnvoyGatewayPolicySelector,
+				Ports:             networkpolicy.Ports(18000, 18002),
+			},
+		},
+		// Pass, not an end-of-tier deny: proxy->backend egress is the user's call, so hand
+		// it down to their tier. Until they allow it the proxy answers 503.
+		v3.Rule{Action: v3.Pass},
+	)
+
+	// The metrics port is not a listener, so it never appears in a Gateway spec.
+	ports := networkpolicy.Ports(append([]uint16{proxyMetricsPort}, listenerPorts...)...)
+
+	return &v3.NetworkPolicy{
+		TypeMeta:   metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"},
+		ObjectMeta: metav1.ObjectMeta{Name: ProxyPolicyName, Namespace: ns},
+		Spec: v3.NetworkPolicySpec{
+			Order:    &networkpolicy.HighPrecedenceOrder,
+			Tier:     networkpolicy.CalicoTierName,
+			Selector: EnvoyProxyPolicySelector,
+			Types:    []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress},
+			// Allow only what a Gateway here actually listens on, then Pass, so the user
+			// keeps control of everything else. The Allow is terminal in this tier, so
+			// widening it any further would take that control away.
+			Ingress: []v3.Rule{
+				{Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"0.0.0.0/0"}}, Destination: v3.EntityRule{Ports: ports}},
+				{Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: v3.EntityRule{Nets: []string{"::/0"}}, Destination: v3.EntityRule{Ports: ports}},
+				{Action: v3.Pass},
+			},
+			Egress: egress,
+		},
+	}
+}