From 8859c7c8f0377e2a70a900f9d4b44629b8e0253c Mon Sep 17 00:00:00 2001 From: chen21019 Date: Mon, 14 Sep 2026 07:33:21 +0800 Subject: [PATCH 1/2] Fix managed bridge forwarding and flat host ports --- .github/workflows/release.yml | 12 ++- COMPATIBILITY.md | 17 +++++ README.md | 23 +++++- hostports/nft.go | 11 ++- hostports/watcher.go | 140 +++++++++++++++++++++++++--------- hostports/watcher_test.go | 105 ++++++++++++++++++++++++- 6 files changed, 267 insertions(+), 41 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 09d8a52..10798d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -177,6 +177,15 @@ jobs: digest="${reference#*@}" [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] printf 'service_digest=%s\n' "$digest" >> "$GITHUB_OUTPUT" + { + printf 'image=%s\n' "$SERVICE_IMAGE" + printf 'digest=%s\n' "$digest" + printf 'source_commit=%s\n' "$(git rev-parse HEAD)" + } > dist/release/published.txt + ( + cd dist/release + sha256sum published.txt >> SHA256SUMS + ) - name: Attest release assets uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 @@ -203,7 +212,8 @@ jobs: "dist/release/network-plugin-manager-${VERSION}-linux-amd64.tar.xz" \ "dist/release/network-plugin-manager-${VERSION}-source.cdx.json" \ "dist/release/network-plugin-manager-${VERSION}-image.cdx.json" \ - dist/release/SHA256SUMS + dist/release/SHA256SUMS \ + dist/release/published.txt - name: Clean owned runner resources if: always() diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 0bc4f71..34aab10 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -81,3 +81,20 @@ conntracked UDP flow; relying on its firewall mark alone can drop later VXLAN datagrams at Docker's bridge filter. The filter-path rule also requires DNAT state and the exact target address, protocol, and port. It does not create a host-wide accept rule or change Docker's default FORWARD policy. + +Each managed bridge subnet may initiate forwarding, while traffic returning +to it is accepted only in `ESTABLISHED` or `RELATED` conntrack states. +Unsolicited traffic whose destination is a managed subnet is not admitted by +this rule. Native nftables applies Docker's configured bridge-accept mark on +both directions; the two iptables frontends use only the selected +`CATTLE_FORWARD` chain. This bounded rule is what permits current Docker +filters to carry Metadata, DNS, and normal workload egress. + +For a CNI bridge marked `skipBridgeConfigureIP`, the workload uses an +external Layer 2 gateway. The manager therefore masquerades only its own DNAT +flows to an exact published address, protocol, and port, preventing replies +from bypassing the publishing host. Other bridge networks preserve source +addresses; only locally originated host-port traffic is masqueraded. When a +host port accepts loopback traffic, `route_localnet` is enabled only on that +specific managed bridge and is rechecked during reconciliation. No global +`route_localnet`, forwarding policy, or third-party NAT rule is changed. diff --git a/README.md b/README.md index 9c158f0..176b3cf 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ themselves establish a managed-service or multi-host rollout. The `v0.8.16` image was published with GHCR manifest digest `sha256:a042c582689561b43349fa83ed92269e849038be3b7a2342e8a9ef0149460f92`. -The current published image is `v0.8.17`, with GHCR manifest digest +The `v0.8.17` image was published with GHCR manifest digest `sha256:f13654b27b71f3fbddbcf33272c10b342d513dd402a255bdda1f341cfbe908f8`. Its signed tag resolves to verified source commit `e29dd5cefa373140d76e3a21da9bd95a3bec97e3`; the release workflow passed @@ -62,6 +62,25 @@ host ports all passed. The manager follows the Docker-selected backend; it does not change the host's firewall preference. This bounded test does not establish every existing iptables or IPsec deployment's migration safety. +The `v0.8.18` image was published with GHCR manifest digest +`sha256:1f5d44de03648a771ec9e7bc448e456ef6b21a5fcd4cc51f59f99df96a804822`. +It restores target-scoped authorization for every packet in an owned DNAT +flow, including later UDP datagrams, without accepting unrelated Docker +traffic. + +The current release is `v0.8.19`. Managed bridge subnets can initiate +outbound traffic and receive only established or related replies; this keeps +Metadata and DNS reachable behind current Docker bridge filters without +opening unsolicited inbound forwarding or changing the host's global policy. +Host ports on a flat L2 network receive target-scoped DNAT masquerading so +replies return through the publishing host even when workloads use an +external gateway. Loopback host-port access enables `route_localnet` only on +the exact managed bridge that needs it. Overlay host ports retain client +source addresses except for locally originated access. Obtain the immutable +image identity from the release's checksum-covered +[`published.txt`](https://github.com/PastureStack/network-plugin-manager/releases/latest/download/published.txt) +rather than copying an older release digest. + The current preflight inspects already loaded legacy tables using an independent iptables-legacy executable. Active old platform or Docker hooks in the other frontend block startup; an unhooked chain declaration alone does @@ -157,7 +176,7 @@ The Alpine 3.23 base image is digest-pinned. Direct runtime packages are exact-v make test make validate bash scripts/check-build-downloads -VERSION_OVERRIDE=v0.8.17 IMAGE_NAMESPACE=local/pasturestack make package +VERSION_OVERRIDE=v0.8.19 IMAGE_NAMESPACE=local/pasturestack make package ``` Pull requests and `main` run one non-publishing gate: tests, vet/format checks, govulncheck, a reproducible binary build, one runtime image build, and Trivy scans plus CycloneDX SBOMs for the source, binary, and image. All reported vulnerabilities and secrets fail the gate. Publishing remains a separate, explicitly authorized operation. diff --git a/hostports/nft.go b/hostports/nft.go index 5ef30c6..e1f8ff0 100644 --- a/hostports/nft.go +++ b/hostports/nft.go @@ -49,8 +49,9 @@ func (w *watcher) checkNFTRules() error { } ruleCount["output"]++ ruleCount["postrouting"]++ + ruleCount["postrouting"]++ } - ruleCount["forward"] = 2 + len(sortedForwardSubnets(w.applied.ForwardSubnets)) + len(sortedForwardPeers(w.applied)) + len(w.applied.Ports) + ruleCount["forward"] = 2 + 2*len(sortedForwardSubnets(w.applied.ForwardSubnets)) + len(sortedForwardPeers(w.applied)) + len(w.applied.Ports) seen := map[string]bool{} actualRules := map[string]int{} for _, item := range listing.NFTables { @@ -157,10 +158,16 @@ func nftHostportBatch(rules ruleSet, existing bool) []byte { fmt.Fprintf(buf, " iifname %q oifname %q ip daddr %s %s dport %s masquerade\n", p.Bridge, p.Bridge, p.TargetIP, p.Protocol, p.TargetPort) } fmt.Fprintf(buf, " ip saddr %s ip daddr %s %s dport %s masquerade\n", p.TargetIP, p.TargetIP, p.Protocol, p.TargetPort) + if p.MasqueradeDNAT { + fmt.Fprintf(buf, " ct status dnat ip daddr %s %s dport %s masquerade\n", p.TargetIP, p.Protocol, p.TargetPort) + } else { + fmt.Fprintf(buf, " fib saddr type local ct status dnat ip daddr %s %s dport %s masquerade\n", p.TargetIP, p.Protocol, p.TargetPort) + } } buf.WriteString(" }\n chain forward {\n type filter hook forward priority -1; policy accept;\n") for _, subnet := range sortedForwardSubnets(rules.ForwardSubnets) { - fmt.Fprintf(buf, " ip saddr %s ip daddr %s meta mark set meta mark | 0x1068 accept\n", subnet, subnet) + fmt.Fprintf(buf, " ip saddr %s ct state new,established,related meta mark set meta mark | 0x1068 accept\n", subnet) + fmt.Fprintf(buf, " ip daddr %s ct state established,related meta mark set meta mark | 0x1068 accept\n", subnet) } for _, pair := range sortedForwardPeers(rules) { fmt.Fprintf(buf, " ip saddr %s ip daddr %s meta mark set meta mark | 0x1068 accept\n", pair.Peer, pair.Local) diff --git a/hostports/watcher.go b/hostports/watcher.go index 8c20eff..26657c5 100644 --- a/hostports/watcher.go +++ b/hostports/watcher.go @@ -31,10 +31,11 @@ var ( // Watch is used to monitor metadata for changes func Watch(c metadata.Client, dc *client.Client, backend firewall.Backend, report func(error)) error { w := &watcher{ - c: c, - dc: dc, - backend: backend, - report: report, + c: c, + dc: dc, + backend: backend, + report: report, + setRouteLocalnet: setupBridgeRouteLocalnet, applied: ruleSet{ Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, @@ -58,18 +59,19 @@ func Watch(c metadata.Client, dc *client.Client, backend firewall.Backend, repor } type watcher struct { - c metadata.Client - dc *client.Client - applied ruleSet - lastApplied time.Time - reconcileMu sync.Mutex - baseRuleMu sync.Mutex - backend firewall.Backend - runCommand func(args ...string) error - output func(args ...string) ([]byte, error) - restoreRules func(name string, args []string, data []byte) error - report func(error) - localHost func(metadata.Client, *client.Client) (metadata.Host, error) + c metadata.Client + dc *client.Client + applied ruleSet + lastApplied time.Time + reconcileMu sync.Mutex + baseRuleMu sync.Mutex + backend firewall.Backend + runCommand func(args ...string) error + output func(args ...string) ([]byte, error) + restoreRules func(name string, args []string, data []byte) error + setRouteLocalnet func(bridge string) error + report func(error) + localHost func(metadata.Client, *client.Client) (metadata.Host, error) } type ruleSet struct { @@ -81,12 +83,13 @@ type ruleSet struct { // PortRule is used to store the needed information for building a // iptables rule type PortRule struct { - Bridge string - SourceIP string - SourcePort string - TargetIP string - TargetPort string - Protocol string + Bridge string + SourceIP string + SourcePort string + TargetIP string + TargetPort string + Protocol string + MasqueradeDNAT bool } func (p PortRule) prefix() []byte { @@ -144,6 +147,19 @@ func (p PortRule) iptables() []byte { buf.WriteString(fmt.Sprintf("\n-A %s -s %v -d %v -p %v -m %v --dport %v -j MASQUERADE", hostPortsPostRoutingChain, p.TargetIP, p.TargetIP, p.Protocol, p.Protocol, p.TargetPort)) + // Locally originated traffic has no ingress bridge and must not reach a + // workload with a loopback or host source address. A flat L2 workload has + // an external default gateway, so every owned DNAT flow must instead return + // through this host's conntrack entry. Keep both rules scoped to this exact + // published target; unrelated forwarding and DNAT rules remain untouched. + if p.MasqueradeDNAT { + buf.WriteString(fmt.Sprintf("\n-A %s -m conntrack --ctstate DNAT -d %v -p %v -m %v --dport %v -j MASQUERADE", + hostPortsPostRoutingChain, p.TargetIP, p.Protocol, p.Protocol, p.TargetPort)) + } else { + buf.WriteString(fmt.Sprintf("\n-A %s -m conntrack --ctstate DNAT -m addrtype --src-type LOCAL -d %v -p %v -m %v --dport %v -j MASQUERADE", + hostPortsPostRoutingChain, p.TargetIP, p.Protocol, p.Protocol, p.TargetPort)) + } + return buf.Bytes() } @@ -347,6 +363,7 @@ func (w *watcher) onChangeLocked(version string, force bool) error { for _, container := range containers { network := networks[container.NetworkUUID] bridge := "" + masqueradeDNAT := false if container.State != "running" && container.State != "starting" { continue @@ -358,16 +375,7 @@ func (w *watcher) onChangeLocked(version string, force bool) error { continue } - conf, _ := network.Metadata["cniConfig"].(map[string]interface{}) - for _, file := range conf { - props, _ := file.(map[string]interface{}) - cniType, _ := props["type"].(string) - checkBridge, _ := props["bridge"].(string) - - if isBridgeCNIType(cniType) && checkBridge != "" { - bridge = checkBridge - } - } + bridge, masqueradeDNAT = hostportBridgeForNetwork(network) for _, port := range container.Ports { rule, ok := parsePortRule(bridge, host.AgentIP, container.PrimaryIp, port) @@ -375,11 +383,15 @@ func (w *watcher) onChangeLocked(version string, force bool) error { return fmt.Errorf("invalid host port definition for container %s (%s): %q", container.Name, container.ExternalId, port) } + rule.MasqueradeDNAT = masqueradeDNAT newRules.Ports[container.ExternalId+"/"+port] = rule } } logrus.Debugf("New generated rules: %v", newRules) + if err := w.preparePortBridges(newRules); err != nil { + return err + } if force || !reflect.DeepEqual(w.applied, newRules) { logrus.Infof("Applying new port rules") return w.apply(newRules) @@ -391,6 +403,34 @@ func (w *watcher) onChangeLocked(version string, force bool) error { return nil } +func (w *watcher) preparePortBridges(rules ruleSet) error { + if w.setRouteLocalnet == nil { + return nil + } + bridges := map[string]bool{} + for _, rule := range rules.Ports { + address, err := netip.ParseAddr(rule.SourceIP) + if err != nil { + return fmt.Errorf("invalid hostport source address %q", rule.SourceIP) + } + if rule.Bridge == "" || (rule.SourceIP != "0.0.0.0" && !address.IsLoopback()) { + continue + } + bridges[rule.Bridge] = true + } + names := make([]string, 0, len(bridges)) + for bridge := range bridges { + names = append(names, bridge) + } + sort.Strings(names) + for _, bridge := range names { + if err := w.setRouteLocalnet(bridge); err != nil { + return fmt.Errorf("enable loopback host ports on bridge %s: %w", bridge, err) + } + } + return nil +} + func (w *watcher) apply(rules ruleSet) error { w.baseRuleMu.Lock() defer w.baseRuleMu.Unlock() @@ -428,7 +468,8 @@ func (w *watcher) apply(rules ruleSet) error { buf.WriteString("\nCOMMIT\n\n*filter\n:CATTLE_FORWARD -\n") buf.WriteString("-F CATTLE_FORWARD\n") for _, subnet := range sortedForwardSubnets(rules.ForwardSubnets) { - buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -s %s -d %s -j ACCEPT\n", subnet, subnet)) + buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -s %s -m conntrack --ctstate NEW,ESTABLISHED,RELATED -j ACCEPT\n", subnet)) + buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -d %s -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT\n", subnet)) } for _, pair := range sortedForwardPeers(rules) { buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -s %s -d %s -j ACCEPT\n", pair.Peer, pair.Local)) @@ -580,6 +621,21 @@ func forwardSubnetForNetwork(network metadata.Network) string { return "" } +func hostportBridgeForNetwork(network metadata.Network) (string, bool) { + conf, _ := network.Metadata["cniConfig"].(map[string]interface{}) + for _, file := range conf { + props, _ := file.(map[string]interface{}) + cniType, _ := props["type"].(string) + bridge, _ := props["bridge"].(string) + if !isBridgeCNIType(cniType) || bridge == "" { + continue + } + externalGateway, _ := props["skipBridgeConfigureIP"].(bool) + return bridge, externalGateway + } + return "", false +} + func isBridgeCNIType(cniType string) bool { return cniType == "pasture-bridge" || cniType == "rancher-bridge" } @@ -636,8 +692,22 @@ func setupKernelParameters() error { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - logrus.Errorf("error setting up kernel parameters") - return err + return fmt.Errorf("set net.bridge.bridge-nf-call-iptables: %w", err) } return nil } + +func setupBridgeRouteLocalnet(bridge string) error { + key := "net.ipv4.conf." + bridge + ".route_localnet" + out, err := exec.Command("sysctl", "-n", key).Output() + if err != nil { + return err + } + if strings.TrimSpace(string(out)) == "1" { + return nil + } + cmd := exec.Command("sysctl", "-w", key+"=1") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/hostports/watcher_test.go b/hostports/watcher_test.go index 125b7d8..fbd4484 100644 --- a/hostports/watcher_test.go +++ b/hostports/watcher_test.go @@ -32,6 +32,48 @@ func TestForwardSubnetSupportsPastureBridge(t *testing.T) { } } +func TestFlatBridgeUsesOwnedDNATMasquerade(t *testing.T) { + flat := metadata.Network{Metadata: map[string]interface{}{ + "cniConfig": map[string]interface{}{ + "10-flat.conf": map[string]interface{}{ + "type": "pasture-bridge", "bridge": "flatbr0", "skipBridgeConfigureIP": true, + }, + }, + }} + if bridge, masquerade := hostportBridgeForNetwork(flat); bridge != "flatbr0" || !masquerade { + t.Fatalf("flat bridge settings = %q,%v", bridge, masquerade) + } + overlay := metadata.Network{Metadata: map[string]interface{}{ + "cniConfig": map[string]interface{}{ + "10-overlay.conf": map[string]interface{}{ + "type": "pasture-bridge", "bridge": "cattle0", "skipBridgeConfigureIP": false, + }, + }, + }} + if bridge, masquerade := hostportBridgeForNetwork(overlay); bridge != "cattle0" || masquerade { + t.Fatalf("overlay bridge settings = %q,%v", bridge, masquerade) + } +} + +func TestPreparePortBridgesEnablesLoopbackOnlyWhereNeeded(t *testing.T) { + var bridges []string + w := &watcher{setRouteLocalnet: func(bridge string) error { + bridges = append(bridges, bridge) + return nil + }} + rules := ruleSet{Ports: map[string]PortRule{ + "all": {Bridge: "flatbr0", SourceIP: "0.0.0.0"}, + "loopback": {Bridge: "cattle0", SourceIP: "127.0.0.1"}, + "bound": {Bridge: "private0", SourceIP: "192.0.2.10"}, + }, ForwardSubnets: map[string]string{}} + if err := w.preparePortBridges(rules); err != nil { + t.Fatal(err) + } + if want := []string{"cattle0", "flatbr0"}; !reflect.DeepEqual(bridges, want) { + t.Fatalf("prepared bridges = %v, want %v", bridges, want) + } +} + func TestPerHostPeerMarksOnlyManagedSubnets(t *testing.T) { rules := ruleSet{ Ports: map[string]PortRule{}, @@ -233,6 +275,66 @@ func TestPublishedUDPDNATIsAcceptedForWholeFlow(t *testing.T) { } } +func TestManagedSubnetAllowsOutboundAndEstablishedReturnOnly(t *testing.T) { + rules := testRuleSet() + var iptablesRules string + w := &watcher{ + backend: firewall.Backend{Mode: firewall.IptablesNFT, Command: "iptables-nft", Restore: "iptables-nft-restore"}, + restoreRules: func(_ string, _ []string, data []byte) error { + iptablesRules = string(data) + return nil + }, + runCommand: func(...string) error { return nil }, + output: func(...string) ([]byte, error) { return []byte("-A FORWARD -j CATTLE_FORWARD\n"), nil }, + } + if err := w.apply(rules); err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "-A CATTLE_FORWARD -s 10.42.0.0/16 -m conntrack --ctstate NEW,ESTABLISHED,RELATED -j ACCEPT", + "-A CATTLE_FORWARD -d 10.42.0.0/16 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT", + } { + if !strings.Contains(iptablesRules, want) { + t.Fatalf("iptables rules missing %q", want) + } + } + if strings.Contains(iptablesRules, "-d 10.42.0.0/16 -m conntrack --ctstate NEW") { + t.Fatal("unsolicited inbound traffic was allowed") + } + nftRules := string(nftHostportBatch(rules, false)) + for _, want := range []string{ + "ip saddr 10.42.0.0/16 ct state new,established,related meta mark set meta mark | 0x1068 accept", + "ip daddr 10.42.0.0/16 ct state established,related meta mark set meta mark | 0x1068 accept", + } { + if !strings.Contains(nftRules, want) { + t.Fatalf("nft rules missing %q", want) + } + } +} + +func TestHostportMasqueradeIsScopedToOwnedDNATTarget(t *testing.T) { + flat := PortRule{Bridge: "flatbr0", SourceIP: "0.0.0.0", SourcePort: "18045", TargetIP: "192.168.204.18", TargetPort: "42", Protocol: "tcp", MasqueradeDNAT: true} + flatRules := string(flat.iptables()) + if !strings.Contains(flatRules, "-m conntrack --ctstate DNAT -d 192.168.204.18 -p tcp -m tcp --dport 42 -j MASQUERADE") { + t.Fatal("flat iptables rules lack target-scoped DNAT masquerade") + } + nftFlat := string(nftHostportBatch(ruleSet{Ports: map[string]PortRule{"flat": flat}, ForwardSubnets: map[string]string{}}, false)) + if !strings.Contains(nftFlat, "ct status dnat ip daddr 192.168.204.18 tcp dport 42 masquerade") { + t.Fatal("flat nft rules lack target-scoped DNAT masquerade") + } + overlay := flat + overlay.Bridge = "cattle0" + overlay.MasqueradeDNAT = false + overlayRules := string(overlay.iptables()) + if !strings.Contains(overlayRules, "-m conntrack --ctstate DNAT -m addrtype --src-type LOCAL -d 192.168.204.18") { + t.Fatal("overlay iptables rules do not limit extra masquerade to local traffic") + } + nftOverlay := string(nftHostportBatch(ruleSet{Ports: map[string]PortRule{"overlay": overlay}, ForwardSubnets: map[string]string{}}, false)) + if !strings.Contains(nftOverlay, "fib saddr type local ct status dnat ip daddr 192.168.204.18 tcp dport 42 masquerade") { + t.Fatal("overlay nft rules do not limit extra masquerade to local traffic") + } +} + func TestApplyIptablesValidatesBeforeUpdatingOrRepairingHooks(t *testing.T) { var calls []string rules := testRuleSet() @@ -358,7 +460,8 @@ func TestNativeNFTUsesOwnedTableAndSingleCheckedBatch(t *testing.T) { "type filter hook forward priority -1", "meta mark & 0x1068 == 0x1068 accept", "ct status dnat ip daddr 10.42.1.2 tcp dport 8080 meta mark set meta mark | 0x1068 accept", - "ip saddr 10.42.0.0/16 ip daddr 10.42.0.0/16 meta mark set meta mark | 0x1068 accept", + "ip saddr 10.42.0.0/16 ct state new,established,related meta mark set meta mark | 0x1068 accept", + "ip daddr 10.42.0.0/16 ct state established,related meta mark set meta mark | 0x1068 accept", } { if !strings.Contains(string(checks), expected) { t.Fatalf("batch missing %q", expected) From fee04e59bd36dcf355e700e55c32d722f62a3205 Mon Sep 17 00:00:00 2001 From: chen21019 Date: Mon, 14 Sep 2026 09:18:13 +0800 Subject: [PATCH 2/2] Harden managed forwarding and loopback transitions --- COMPATIBILITY.md | 18 +- README.md | 11 +- hostports/iptables_vm_test.go | 8 +- hostports/lifecycle_test.go | 5 +- hostports/nft.go | 31 +++- hostports/nft_lifecycle_vm_test.go | 4 +- hostports/readiness_test.go | 4 +- hostports/route_localnet.go | 181 ++++++++++++++++++ hostports/route_localnet_test.go | 168 +++++++++++++++++ hostports/watcher.go | 289 ++++++++++++++++++----------- hostports/watcher_test.go | 44 ++--- 11 files changed, 600 insertions(+), 163 deletions(-) create mode 100644 hostports/route_localnet.go create mode 100644 hostports/route_localnet_test.go diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 34aab10..ad75e26 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -85,10 +85,13 @@ host-wide accept rule or change Docker's default FORWARD policy. Each managed bridge subnet may initiate forwarding, while traffic returning to it is accepted only in `ESTABLISHED` or `RELATED` conntrack states. Unsolicited traffic whose destination is a managed subnet is not admitted by -this rule. Native nftables applies Docker's configured bridge-accept mark on -both directions; the two iptables frontends use only the selected +this rule. Both directions are matched against the exact validated CNI bridge +and subnet pair; missing or conflicting bridge metadata fails before any +firewall change. Native nftables applies Docker's configured bridge-accept +mark on both directions; the two iptables frontends use only the selected `CATTLE_FORWARD` chain. This bounded rule is what permits current Docker -filters to carry Metadata, DNS, and normal workload egress. +filters to carry Metadata, DNS, and normal workload egress without trusting a +spoofed managed prefix arriving on another host interface. For a CNI bridge marked `skipBridgeConfigureIP`, the workload uses an external Layer 2 gateway. The manager therefore masquerades only its own DNAT @@ -96,5 +99,10 @@ flows to an exact published address, protocol, and port, preventing replies from bypassing the publishing host. Other bridge networks preserve source addresses; only locally originated host-port traffic is masqueraded. When a host port accepts loopback traffic, `route_localnet` is enabled only on that -specific managed bridge and is rechecked during reconciliation. No global -`route_localnet`, forwarding policy, or third-party NAT rule is changed. +specific managed bridge and is rechecked during reconciliation. Before that +sysctl is enabled, the selected backend must validate and install a +bridge-scoped raw-prerouting drop for `127.0.0.0/8`. The manager records the +original per-bridge value in host-mounted runtime state and restores it before +removing the last guard. Failed preflight or firewall application never opens +the sysctl. No global `route_localnet`, forwarding policy, or third-party NAT +rule is changed. diff --git a/README.md b/README.md index 176b3cf..97c4908 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,18 @@ The current release is `v0.8.19`. Managed bridge subnets can initiate outbound traffic and receive only established or related replies; this keeps Metadata and DNS reachable behind current Docker bridge filters without opening unsolicited inbound forwarding or changing the host's global policy. +Every forwarding exception is bound to the exact validated CNI bridge and +subnet pair; missing or conflicting bridge metadata fails before any firewall +change. This prevents traffic arriving on an unrelated host interface from +claiming a managed source prefix. Host ports on a flat L2 network receive target-scoped DNAT masquerading so replies return through the publishing host even when workloads use an external gateway. Loopback host-port access enables `route_localnet` only on -the exact managed bridge that needs it. Overlay host ports retain client -source addresses except for locally originated access. Obtain the immutable +the exact managed bridge that needs it, after a bridge-scoped raw-prerouting +drop for `127.0.0.0/8` is live. The original per-bridge value is recorded on +the host-mounted runtime state and restored before the final guard is removed. +Overlay host ports retain client source addresses except for locally +originated access. Obtain the immutable image identity from the release's checksum-covered [`published.txt`](https://github.com/PastureStack/network-plugin-manager/releases/latest/download/published.txt) rather than copying an older release digest. diff --git a/hostports/iptables_vm_test.go b/hostports/iptables_vm_test.go index 40d90d2..a789d8d 100644 --- a/hostports/iptables_vm_test.go +++ b/hostports/iptables_vm_test.go @@ -53,7 +53,7 @@ func testIptablesOnDisposableVM(t *testing.T, mode firewall.Mode) { if out, err := exec.Command(command, "-t", "nat", "-S", "DOCKER").CombinedOutput(); err != nil { t.Fatalf("requires Docker-owned NAT chain in %s: %v: %s", mode, err, out) } - for _, table := range []string{"nat", "filter"} { + for _, table := range []string{"raw", "nat", "filter"} { out, err := xtVMCommand(command, "-t", table, "-S") if err != nil { t.Fatalf("inspect existing %s rules: %v: %s", table, err, out) @@ -68,6 +68,7 @@ func testIptablesOnDisposableVM(t *testing.T, mode firewall.Mode) { "isolated": {Bridge: "pstest0", SourceIP: "198.51.100.2", SourcePort: "55555", TargetIP: "10.254.250.2", TargetPort: "55556", Protocol: "tcp"}, }, ForwardSubnets: map[string]string{"isolated": "10.254.250.0/24"}, + ForwardBridges: map[string]string{"isolated": "pstest0"}, } w := &watcher{backend: firewall.Backend{Mode: mode, Command: command, Restore: restore}} for attempt := 1; attempt <= 2; attempt++ { @@ -75,6 +76,7 @@ func testIptablesOnDisposableVM(t *testing.T, mode firewall.Mode) { t.Fatalf("iptables-nft apply %d (includes --test -n): %v", attempt, err) } for _, hook := range []struct{ table, chain, target string }{ + {"raw", "PREROUTING", hostPortsRawChain}, {"nat", "PREROUTING", "CATTLE_PREROUTING"}, {"nat", "OUTPUT", "CATTLE_OUTPUT"}, {"nat", "POSTROUTING", hostPortsPostRoutingChain}, @@ -100,6 +102,7 @@ func cleanupXTTestRules(t *testing.T, command string) { table, chain string spec []string }{ + {"raw", "PREROUTING", []string{"-j", hostPortsRawChain}}, {"nat", "PREROUTING", []string{"-m", "addrtype", "--dst-type", "LOCAL", "-j", "CATTLE_PREROUTING"}}, {"nat", "OUTPUT", []string{"-m", "addrtype", "--dst-type", "LOCAL", "-j", "CATTLE_OUTPUT"}}, {"nat", "POSTROUTING", []string{"-j", hostPortsPostRoutingChain}}, @@ -118,6 +121,7 @@ func cleanupXTTestRules(t *testing.T, command string) { } } for _, entry := range []struct{ table, chain string }{ + {"raw", hostPortsRawChain}, {"nat", "CATTLE_PREROUTING"}, {"nat", "CATTLE_POSTROUTING"}, {"nat", "CATTLE_OUTPUT"}, @@ -135,7 +139,7 @@ func cleanupXTTestRules(t *testing.T, command string) { t.Errorf("delete own chain %s/%s: %v: %s", entry.table, entry.chain, err, out) } } - for _, table := range []string{"nat", "filter"} { + for _, table := range []string{"raw", "nat", "filter"} { out, err := xtVMCommand(command, "-t", table, "-S") if err != nil { t.Errorf("verify %s cleanup: %v: %s", table, err, out) diff --git a/hostports/lifecycle_test.go b/hostports/lifecycle_test.go index 3d5ca65..b23c369 100644 --- a/hostports/lifecycle_test.go +++ b/hostports/lifecycle_test.go @@ -25,7 +25,9 @@ func emptyHostportWatcher() *watcher { localHost: func(metadata.Client, *client.Client) (metadata.Host, error) { return metadata.Host{UUID: "host-1"}, nil }, - applied: ruleSet{Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}}, + applied: ruleSet{ + Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, RouteLocalnetBridges: map[string]bool{}, + }, lastApplied: time.Now(), } } @@ -44,6 +46,7 @@ func TestNoPerHostNetworkDoesNotReapplyUnchangedFirewall(t *testing.T) { const intactEmptyNFT = `{"nftables":[ {"table":{"family":"ip","name":"pasturestack_hostports"}}, + {"chain":{"family":"ip","table":"pasturestack_hostports","name":"raw_prerouting","type":"filter","hook":"prerouting","prio":-300,"policy":"accept"}}, {"chain":{"family":"ip","table":"pasturestack_hostports","name":"prerouting","type":"nat","hook":"prerouting","prio":-101,"policy":"accept"}}, {"chain":{"family":"ip","table":"pasturestack_hostports","name":"output","type":"nat","hook":"output","prio":-101,"policy":"accept"}}, {"chain":{"family":"ip","table":"pasturestack_hostports","name":"postrouting","type":"nat","hook":"postrouting","prio":99,"policy":"accept"}}, diff --git a/hostports/nft.go b/hostports/nft.go index e1f8ff0..503c7f0 100644 --- a/hostports/nft.go +++ b/hostports/nft.go @@ -35,12 +35,14 @@ func (w *watcher) checkNFTRules() error { Policy string `json:"policy"` } expected := map[string]chain{ - "prerouting": {Type: "nat", Hook: "prerouting", Prio: -101, Policy: "accept"}, - "output": {Type: "nat", Hook: "output", Prio: -101, Policy: "accept"}, - "postrouting": {Type: "nat", Hook: "postrouting", Prio: 99, Policy: "accept"}, - "forward": {Type: "filter", Hook: "forward", Prio: -1, Policy: "accept"}, + "raw_prerouting": {Type: "filter", Hook: "prerouting", Prio: -300, Policy: "accept"}, + "prerouting": {Type: "nat", Hook: "prerouting", Prio: -101, Policy: "accept"}, + "output": {Type: "nat", Hook: "output", Prio: -101, Policy: "accept"}, + "postrouting": {Type: "nat", Hook: "postrouting", Prio: 99, Policy: "accept"}, + "forward": {Type: "filter", Hook: "forward", Prio: -1, Policy: "accept"}, } ruleCount := map[string]int{} + ruleCount["raw_prerouting"] = len(sortedEnabledBridges(w.applied.RouteLocalnetBridges)) for _, p := range w.applied.Ports { ruleCount["prerouting"]++ if p.Bridge != "" { @@ -51,7 +53,7 @@ func (w *watcher) checkNFTRules() error { ruleCount["postrouting"]++ ruleCount["postrouting"]++ } - ruleCount["forward"] = 2 + 2*len(sortedForwardSubnets(w.applied.ForwardSubnets)) + len(sortedForwardPeers(w.applied)) + len(w.applied.Ports) + ruleCount["forward"] = 2 + 2*len(sortedForwardNetworks(w.applied)) + len(sortedForwardPeers(w.applied)) + len(w.applied.Ports) seen := map[string]bool{} actualRules := map[string]int{} for _, item := range listing.NFTables { @@ -110,9 +112,15 @@ func (w *watcher) applyNFT(rules ruleSet) error { if err := w.restore("nft", []string{"-c", "-f", "-"}, batch); err != nil { return fmt.Errorf("validate native nft hostport batch: %w", err) } + if err := w.restoreRemovedRouteLocalnet(rules); err != nil { + return err + } if err := w.restore("nft", []string{"-f", "-"}, batch); err != nil { return fmt.Errorf("apply native nft hostport batch: %w", err) } + if err := w.activateRouteLocalnet(rules); err != nil { + return err + } w.applied = rules w.lastApplied = time.Now() return nil @@ -124,6 +132,11 @@ func nftHostportBatch(rules ruleSet, existing bool) []byte { fmt.Fprintf(buf, "delete table ip %s\n", nftHostportsTable) } fmt.Fprintf(buf, "table ip %s {\n", nftHostportsTable) + buf.WriteString(" chain raw_prerouting {\n type filter hook prerouting priority -300; policy accept;\n") + for _, bridge := range sortedEnabledBridges(rules.RouteLocalnetBridges) { + fmt.Fprintf(buf, " iifname %q ip daddr 127.0.0.0/8 drop\n", bridge) + } + buf.WriteString(" }\n") buf.WriteString(" chain prerouting {\n type nat hook prerouting priority -101; policy accept;\n") keys := make([]string, 0, len(rules.Ports)) for key := range rules.Ports { @@ -165,12 +178,12 @@ func nftHostportBatch(rules ruleSet, existing bool) []byte { } } buf.WriteString(" }\n chain forward {\n type filter hook forward priority -1; policy accept;\n") - for _, subnet := range sortedForwardSubnets(rules.ForwardSubnets) { - fmt.Fprintf(buf, " ip saddr %s ct state new,established,related meta mark set meta mark | 0x1068 accept\n", subnet) - fmt.Fprintf(buf, " ip daddr %s ct state established,related meta mark set meta mark | 0x1068 accept\n", subnet) + for _, network := range sortedForwardNetworks(rules) { + fmt.Fprintf(buf, " iifname %q ip saddr %s ct state new,established,related meta mark set meta mark | 0x1068 accept\n", network.Bridge, network.Subnet) + fmt.Fprintf(buf, " oifname %q ip daddr %s ct state established,related meta mark set meta mark | 0x1068 accept\n", network.Bridge, network.Subnet) } for _, pair := range sortedForwardPeers(rules) { - fmt.Fprintf(buf, " ip saddr %s ip daddr %s meta mark set meta mark | 0x1068 accept\n", pair.Peer, pair.Local) + fmt.Fprintf(buf, " oifname %q ip saddr %s ip daddr %s meta mark set meta mark | 0x1068 accept\n", pair.Bridge, pair.Peer, pair.Local) } // NAT chains see only the first packet of a conntracked flow. Restore the // Docker bridge-accept mark on every forwarded datagram, scoped to our diff --git a/hostports/nft_lifecycle_vm_test.go b/hostports/nft_lifecycle_vm_test.go index 1e40d04..d5ed8ae 100644 --- a/hostports/nft_lifecycle_vm_test.go +++ b/hostports/nft_lifecycle_vm_test.go @@ -74,7 +74,9 @@ func TestNativeNFTLifecycleInUnsharedVMNamespace(t *testing.T) { localHost: func(metadata.Client, *client.Client) (metadata.Host, error) { return metadata.Host{UUID: "test-host", AgentIP: "198.51.100.2"}, nil }, - applied: ruleSet{Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}}, + applied: ruleSet{ + Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, RouteLocalnetBridges: map[string]bool{}, + }, } if err := w.onChange("initial"); err != nil { t.Fatalf("initial production nft apply: %v", err) diff --git a/hostports/readiness_test.go b/hostports/readiness_test.go index 8964259..8ba4856 100644 --- a/hostports/readiness_test.go +++ b/hostports/readiness_test.go @@ -27,7 +27,9 @@ func TestMalformedEligibleHostPortFailsReconcileWithoutApplying(t *testing.T) { localHost: func(metadata.Client, *client.Client) (metadata.Host, error) { return metadata.Host{UUID: "host-1", AgentIP: "192.0.2.10"}, nil }, - applied: ruleSet{Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}}, + applied: ruleSet{ + Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, RouteLocalnetBridges: map[string]bool{}, + }, } err := w.onChange("test") if err == nil || !strings.Contains(err.Error(), "invalid host port definition") { diff --git a/hostports/route_localnet.go b/hostports/route_localnet.go new file mode 100644 index 0000000..cef3201 --- /dev/null +++ b/hostports/route_localnet.go @@ -0,0 +1,181 @@ +package hostports + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +const routeLocalnetStatePath = "/run/pasturestack/network-plugin-manager/route-localnet.json" + +type persistedRouteLocalnetState struct { + Version int `json:"version"` + Original map[string]bool `json:"original"` +} + +func loadRouteLocalnetState(path string) (map[string]bool, error) { + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return map[string]bool{}, nil + } + if err != nil { + return nil, err + } + var state persistedRouteLocalnetState + if err := json.Unmarshal(data, &state); err != nil { + return nil, fmt.Errorf("decode %s: %w", path, err) + } + if state.Version != 1 || state.Original == nil { + return nil, fmt.Errorf("unsupported or incomplete state in %s", path) + } + for bridge := range state.Original { + if err := validateBridgeName(bridge); err != nil { + return nil, fmt.Errorf("invalid persisted bridge: %w", err) + } + } + return state.Original, nil +} + +func saveRouteLocalnetState(path string, original map[string]bool) error { + if path == "" { + return nil + } + directory := filepath.Dir(path) + if err := os.MkdirAll(directory, 0700); err != nil { + return err + } + data, err := json.Marshal(persistedRouteLocalnetState{Version: 1, Original: original}) + if err != nil { + return err + } + data = append(data, '\n') + temporary, err := os.CreateTemp(directory, ".route-localnet-*") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0600); err != nil { + temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + return os.Rename(temporaryPath, path) +} + +func readBridgeRouteLocalnet(bridge string) (bool, error) { + if err := validateBridgeName(bridge); err != nil { + return false, err + } + path := "/proc/sys/net/ipv4/conf/" + bridge + "/route_localnet" + value, err := os.ReadFile(path) + if err != nil { + return false, err + } + switch strings.TrimSpace(string(value)) { + case "0": + return false, nil + case "1": + return true, nil + default: + return false, fmt.Errorf("unexpected route_localnet value for bridge %s", bridge) + } +} + +func setBridgeRouteLocalnet(bridge string, enabled bool) error { + if err := validateBridgeName(bridge); err != nil { + return err + } + path := "/proc/sys/net/ipv4/conf/" + bridge + "/route_localnet" + value := []byte("0\n") + if enabled { + value = []byte("1\n") + } + if err := os.WriteFile(path, value, 0644); err != nil { + if !enabled && os.IsNotExist(err) { + return nil + } + return err + } + return nil +} + +func sortedEnabledBridges(bridges map[string]bool) []string { + names := []string{} + for bridge, enabled := range bridges { + if enabled { + names = append(names, bridge) + } + } + sort.Strings(names) + return names +} + +// restoreRemovedRouteLocalnet runs after a complete firewall precheck and +// before a guard is removed. It restores only values previously changed by +// this process family, as recorded on the host-mounted /run filesystem. +func (w *watcher) restoreRemovedRouteLocalnet(rules ruleSet) error { + if w.setRouteLocalnet == nil { + return nil + } + bridges := make([]string, 0, len(w.routeLocalnetOriginal)) + for bridge := range w.routeLocalnetOriginal { + if !rules.RouteLocalnetBridges[bridge] { + bridges = append(bridges, bridge) + } + } + sort.Strings(bridges) + for _, bridge := range bridges { + if err := w.setRouteLocalnet(bridge, w.routeLocalnetOriginal[bridge]); err != nil { + return fmt.Errorf("restore route_localnet on bridge %s: %w", bridge, err) + } + } + return nil +} + +// activateRouteLocalnet is called only after the bridge-scoped 127/8 guard is +// live. Ownership is persisted before enabling so a process restart can still +// restore the original per-bridge value. +func (w *watcher) activateRouteLocalnet(rules ruleSet) error { + if w.setRouteLocalnet == nil { + return nil + } + if w.getRouteLocalnet == nil { + return fmt.Errorf("route_localnet reader is not configured") + } + next := map[string]bool{} + for _, bridge := range sortedEnabledBridges(rules.RouteLocalnetBridges) { + original, known := w.routeLocalnetOriginal[bridge] + if !known { + var err error + original, err = w.getRouteLocalnet(bridge) + if err != nil { + return fmt.Errorf("read route_localnet on bridge %s: %w", bridge, err) + } + } + next[bridge] = original + } + if err := saveRouteLocalnetState(w.routeLocalnetStatePath, next); err != nil { + return fmt.Errorf("persist route_localnet ownership: %w", err) + } + w.routeLocalnetOriginal = next + for _, bridge := range sortedEnabledBridges(rules.RouteLocalnetBridges) { + if err := w.setRouteLocalnet(bridge, true); err != nil { + return fmt.Errorf("enable guarded route_localnet on bridge %s: %w", bridge, err) + } + } + return nil +} diff --git a/hostports/route_localnet_test.go b/hostports/route_localnet_test.go new file mode 100644 index 0000000..563ddd9 --- /dev/null +++ b/hostports/route_localnet_test.go @@ -0,0 +1,168 @@ +package hostports + +import ( + "errors" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/PastureStack/network-plugin-manager/internal/firewall" + "github.com/PastureStack/network-plugin-manager/internal/metadata" +) + +func routeLocalnetTestWatcher(t *testing.T, calls *[]string) *watcher { + t.Helper() + return &watcher{ + backend: firewall.Backend{Mode: firewall.IptablesNFT, Command: "iptables-nft", Restore: "iptables-nft-restore"}, + restoreRules: func(_ string, args []string, _ []byte) error { + if reflect.DeepEqual(args, []string{"--test", "-n"}) { + *calls = append(*calls, "precheck") + } else if reflect.DeepEqual(args, []string{"-n"}) { + *calls = append(*calls, "apply") + } + return nil + }, + runCommand: func(...string) error { return nil }, + output: func(...string) ([]byte, error) { + return []byte("-A FORWARD -j CATTLE_FORWARD\n"), nil + }, + getRouteLocalnet: func(string) (bool, error) { + *calls = append(*calls, "read-original") + return false, nil + }, + setRouteLocalnet: func(_ string, enabled bool) error { + *calls = append(*calls, "set:"+map[bool]string{false: "0", true: "1"}[enabled]) + return nil + }, + routeLocalnetOriginal: map[string]bool{}, + routeLocalnetStatePath: filepath.Join(t.TempDir(), "route-localnet.json"), + applied: ruleSet{Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, RouteLocalnetBridges: map[string]bool{}}, + } +} + +func TestRouteLocalnetGuardAndRestoreOrdering(t *testing.T) { + var calls []string + w := routeLocalnetTestWatcher(t, &calls) + rules := ruleSet{ + Ports: map[string]PortRule{"port": {Bridge: "flatbr0", SourceIP: "0.0.0.0", SourcePort: "18045", TargetIP: "192.0.2.20", TargetPort: "42", Protocol: "tcp"}}, + ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, + RouteLocalnetBridges: map[string]bool{"flatbr0": true}, + } + if err := w.apply(rules); err != nil { + t.Fatal(err) + } + if got, want := calls, []string{"precheck", "apply", "read-original", "set:1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("enable order = %v, want %v", got, want) + } + state, err := loadRouteLocalnetState(w.routeLocalnetStatePath) + if err != nil || len(state) != 1 || state["flatbr0"] { + t.Fatalf("persisted original = %v, err=%v", state, err) + } + + calls = nil + empty := ruleSet{Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, RouteLocalnetBridges: map[string]bool{}} + if err := w.apply(empty); err != nil { + t.Fatal(err) + } + if got, want := calls, []string{"precheck", "set:0", "apply"}; !reflect.DeepEqual(got, want) { + t.Fatalf("disable order = %v, want %v", got, want) + } + state, err = loadRouteLocalnetState(w.routeLocalnetStatePath) + if err != nil || len(state) != 0 { + t.Fatalf("state after restore = %v, err=%v", state, err) + } +} + +func TestRouteLocalnetPrecheckFailureDoesNotMutateSysctl(t *testing.T) { + var calls []string + w := routeLocalnetTestWatcher(t, &calls) + w.restoreRules = func(_ string, args []string, _ []byte) error { + calls = append(calls, strings.Join(args, " ")) + return errors.New("invalid batch") + } + rules := ruleSet{Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, RouteLocalnetBridges: map[string]bool{"flatbr0": true}} + if err := w.apply(rules); err == nil { + t.Fatal("expected precheck failure") + } + if got, want := calls, []string{"--test -n"}; !reflect.DeepEqual(got, want) { + t.Fatalf("calls after failed precheck = %v, want %v", got, want) + } +} + +func TestForwardSubnetWithoutBridgeFailsBeforeFirewallMutation(t *testing.T) { + var calls []string + w := routeLocalnetTestWatcher(t, &calls) + rules := ruleSet{ + Ports: map[string]PortRule{}, + ForwardSubnets: map[string]string{"network": "10.42.0.0/16"}, + ForwardBridges: map[string]string{}, RouteLocalnetBridges: map[string]bool{}, + } + if err := w.apply(rules); err == nil || !strings.Contains(err.Error(), "invalid bridge name") { + t.Fatalf("missing bridge error = %v", err) + } + if len(calls) != 0 { + t.Fatalf("firewall or sysctl mutated before bridge validation: %v", calls) + } +} + +func TestNativeNFTRouteLocalnetTransitionOrdering(t *testing.T) { + var calls []string + w := routeLocalnetTestWatcher(t, &calls) + w.backend = firewall.Backend{Mode: firewall.NFTables} + w.output = func(...string) ([]byte, error) { return []byte(""), nil } + w.restoreRules = func(_ string, args []string, _ []byte) error { + calls = append(calls, "nft:"+strings.Join(args, " ")) + return nil + } + rules := ruleSet{ + Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, + RouteLocalnetBridges: map[string]bool{"flatbr0": true}, + } + if err := w.apply(rules); err != nil { + t.Fatal(err) + } + if got, want := calls, []string{"nft:-c -f -", "nft:-f -", "read-original", "set:1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("native nft enable order = %v, want %v", got, want) + } + + calls = nil + empty := ruleSet{Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, RouteLocalnetBridges: map[string]bool{}} + if err := w.apply(empty); err != nil { + t.Fatal(err) + } + if got, want := calls, []string{"nft:-c -f -", "set:0", "nft:-f -"}; !reflect.DeepEqual(got, want) { + t.Fatalf("native nft disable order = %v, want %v", got, want) + } +} + +func TestRouteLocalnetGuardIsBridgeScopedAcrossBackends(t *testing.T) { + rules := ruleSet{Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{}, ForwardBridges: map[string]string{}, RouteLocalnetBridges: map[string]bool{"flatbr0": true}} + var xtables string + w := &watcher{ + backend: firewall.Backend{Mode: firewall.IptablesLegacy, Command: "iptables-legacy", Restore: "iptables-legacy-restore"}, + restoreRules: func(_ string, _ []string, data []byte) error { xtables = string(data); return nil }, + runCommand: func(...string) error { return nil }, + output: func(...string) ([]byte, error) { return []byte("-A FORWARD -j CATTLE_FORWARD\n"), nil }, + } + if err := w.apply(rules); err != nil { + t.Fatal(err) + } + if !strings.Contains(xtables, "-A CATTLE_HOSTPORTS_RAW -i flatbr0 -d 127.0.0.0/8 -j DROP") { + t.Fatal("xtables guard is missing or not bridge-scoped") + } + native := string(nftHostportBatch(rules, false)) + if !strings.Contains(native, "iifname \"flatbr0\" ip daddr 127.0.0.0/8 drop") { + t.Fatal("native nft guard is missing or not bridge-scoped") + } +} + +func TestConflictingManagedBridgeMetadataFailsClosed(t *testing.T) { + network := metadata.Network{Metadata: map[string]interface{}{"cniConfig": map[string]interface{}{ + "10-a.conf": map[string]interface{}{"type": "pasture-bridge", "bridge": "flatbr0", "bridgeSubnet": "192.0.2.0/24"}, + "20-b.conf": map[string]interface{}{"type": "pasture-bridge", "bridge": "flatbr1", "bridgeSubnet": "198.51.100.0/24"}, + }}} + if _, err := bridgeConfigForNetwork(network); err == nil { + t.Fatal("conflicting bridge metadata was accepted") + } +} diff --git a/hostports/watcher.go b/hostports/watcher.go index 26657c5..3d3c1eb 100644 --- a/hostports/watcher.go +++ b/hostports/watcher.go @@ -26,19 +26,33 @@ var ( baseRuleRepairEvery = 30 * time.Second hostPortsLabel = "io.rancher.network.host_ports" hostPortsPostRoutingChain = "CATTLE_HOSTPORTS_POSTROUTING" + hostPortsRawChain = "CATTLE_HOSTPORTS_RAW" ) // Watch is used to monitor metadata for changes func Watch(c metadata.Client, dc *client.Client, backend firewall.Backend, report func(error)) error { + routeLocalnetOriginal, err := loadRouteLocalnetState(routeLocalnetStatePath) + if err != nil { + return fmt.Errorf("load route_localnet ownership: %w", err) + } + previousRouteLocalnet := map[string]bool{} + for bridge := range routeLocalnetOriginal { + previousRouteLocalnet[bridge] = true + } w := &watcher{ - c: c, - dc: dc, - backend: backend, - report: report, - setRouteLocalnet: setupBridgeRouteLocalnet, + c: c, + dc: dc, + backend: backend, + report: report, + getRouteLocalnet: readBridgeRouteLocalnet, + setRouteLocalnet: setBridgeRouteLocalnet, + routeLocalnetOriginal: routeLocalnetOriginal, + routeLocalnetStatePath: routeLocalnetStatePath, applied: ruleSet{ - Ports: map[string]PortRule{}, - ForwardSubnets: map[string]string{}, + Ports: map[string]PortRule{}, + ForwardSubnets: map[string]string{}, + ForwardBridges: map[string]string{}, + RouteLocalnetBridges: previousRouteLocalnet, }, } @@ -59,25 +73,30 @@ func Watch(c metadata.Client, dc *client.Client, backend firewall.Backend, repor } type watcher struct { - c metadata.Client - dc *client.Client - applied ruleSet - lastApplied time.Time - reconcileMu sync.Mutex - baseRuleMu sync.Mutex - backend firewall.Backend - runCommand func(args ...string) error - output func(args ...string) ([]byte, error) - restoreRules func(name string, args []string, data []byte) error - setRouteLocalnet func(bridge string) error - report func(error) - localHost func(metadata.Client, *client.Client) (metadata.Host, error) + c metadata.Client + dc *client.Client + applied ruleSet + lastApplied time.Time + reconcileMu sync.Mutex + baseRuleMu sync.Mutex + backend firewall.Backend + runCommand func(args ...string) error + output func(args ...string) ([]byte, error) + restoreRules func(name string, args []string, data []byte) error + getRouteLocalnet func(bridge string) (bool, error) + setRouteLocalnet func(bridge string, enabled bool) error + routeLocalnetOriginal map[string]bool + routeLocalnetStatePath string + report func(error) + localHost func(metadata.Client, *client.Client) (metadata.Host, error) } type ruleSet struct { - Ports map[string]PortRule - ForwardSubnets map[string]string - ForwardPeers map[string][]string + Ports map[string]PortRule + ForwardSubnets map[string]string + ForwardBridges map[string]string + ForwardPeers map[string][]string + RouteLocalnetBridges map[string]bool } // PortRule is used to store the needed information for building a @@ -172,6 +191,11 @@ func (w *watcher) insertBaseRules() error { return fmt.Errorf("hostports iptables command is not configured") } var errs []string + if w.run(iptables, "-w", "-t", "raw", "-C", "PREROUTING", "-j", hostPortsRawChain) != nil { + if err := w.run(iptables, "-w", "-t", "raw", "-I", "PREROUTING", "1", "-j", hostPortsRawChain); err != nil { + errs = append(errs, err.Error()) + } + } if w.run(iptables, "-w", "-t", "nat", "-C", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "CATTLE_PREROUTING") != nil { if err := w.run(iptables, "-w", "-t", "nat", "-I", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "CATTLE_PREROUTING"); err != nil { errs = append(errs, err.Error()) @@ -310,8 +334,10 @@ func (w *watcher) reportLocked(err error) { func (w *watcher) onChangeLocked(version string, force bool) error { logrus.Debug("Creating rule set") newRules := ruleSet{ - Ports: map[string]PortRule{}, - ForwardSubnets: map[string]string{}, + Ports: map[string]PortRule{}, + ForwardSubnets: map[string]string{}, + ForwardBridges: map[string]string{}, + RouteLocalnetBridges: map[string]bool{}, } resolveHost := w.localHost @@ -330,12 +356,20 @@ func (w *watcher) onChangeLocked(version string, force bool) error { var peerHosts []metadata.Host peersLoaded := false for uuid, network := range networks { - if subnet := forwardSubnetForNetwork(network); subnet != "" { + bridgeConfig, err := bridgeConfigForNetwork(network) + if err != nil { + return fmt.Errorf("network %s bridge configuration: %w", uuid, err) + } + if subnet := bridgeConfig.Subnet; subnet != "" { + if bridgeConfig.Bridge == "" { + return fmt.Errorf("network %s has a forward subnet but no managed bridge", uuid) + } resolved, err := hostlabel.Resolve(subnet, host.Labels) if err != nil { return fmt.Errorf("network %s hostport subnet: %w", uuid, err) } newRules.ForwardSubnets[uuid] = resolved + newRules.ForwardBridges[uuid] = bridgeConfig.Bridge if hostlabel.IsReference(subnet) { if newRules.ForwardPeers == nil { newRules.ForwardPeers = map[string][]string{} @@ -375,7 +409,11 @@ func (w *watcher) onChangeLocked(version string, force bool) error { continue } - bridge, masqueradeDNAT = hostportBridgeForNetwork(network) + bridgeConfig, err := bridgeConfigForNetwork(network) + if err != nil { + return fmt.Errorf("network %s bridge configuration: %w", container.NetworkUUID, err) + } + bridge, masqueradeDNAT = bridgeConfig.Bridge, bridgeConfig.ExternalGateway for _, port := range container.Ports { rule, ok := parsePortRule(bridge, host.AgentIP, container.PrimaryIp, port) @@ -385,13 +423,14 @@ func (w *watcher) onChangeLocked(version string, force bool) error { rule.MasqueradeDNAT = masqueradeDNAT newRules.Ports[container.ExternalId+"/"+port] = rule + address, _ := netip.ParseAddr(rule.SourceIP) + if rule.Bridge != "" && (rule.SourceIP == "0.0.0.0" || address.IsLoopback()) { + newRules.RouteLocalnetBridges[rule.Bridge] = true + } } } logrus.Debugf("New generated rules: %v", newRules) - if err := w.preparePortBridges(newRules); err != nil { - return err - } if force || !reflect.DeepEqual(w.applied, newRules) { logrus.Infof("Applying new port rules") return w.apply(newRules) @@ -403,34 +442,6 @@ func (w *watcher) onChangeLocked(version string, force bool) error { return nil } -func (w *watcher) preparePortBridges(rules ruleSet) error { - if w.setRouteLocalnet == nil { - return nil - } - bridges := map[string]bool{} - for _, rule := range rules.Ports { - address, err := netip.ParseAddr(rule.SourceIP) - if err != nil { - return fmt.Errorf("invalid hostport source address %q", rule.SourceIP) - } - if rule.Bridge == "" || (rule.SourceIP != "0.0.0.0" && !address.IsLoopback()) { - continue - } - bridges[rule.Bridge] = true - } - names := make([]string, 0, len(bridges)) - for bridge := range bridges { - names = append(names, bridge) - } - sort.Strings(names) - for _, bridge := range names { - if err := w.setRouteLocalnet(bridge); err != nil { - return fmt.Errorf("enable loopback host ports on bridge %s: %w", bridge, err) - } - } - return nil -} - func (w *watcher) apply(rules ruleSet) error { w.baseRuleMu.Lock() defer w.baseRuleMu.Unlock() @@ -445,6 +456,14 @@ func (w *watcher) apply(rules ruleSet) error { } buf := &bytes.Buffer{} + buf.WriteString("*raw\n") + buf.WriteString(fmt.Sprintf(":%s -\n", hostPortsRawChain)) + buf.WriteString(fmt.Sprintf("-F %s\n", hostPortsRawChain)) + for _, bridge := range sortedEnabledBridges(rules.RouteLocalnetBridges) { + buf.WriteString(fmt.Sprintf("-A %s -i %s -d 127.0.0.0/8 -j DROP\n", hostPortsRawChain, bridge)) + } + buf.WriteString("COMMIT\n\n") + // NOTE: We don't use CATTLE_POSTROUTING, but for migration we just wipe it out buf.WriteString("*nat\n") buf.WriteString(":CATTLE_PREROUTING -\n") @@ -467,12 +486,12 @@ func (w *watcher) apply(rules ruleSet) error { buf.WriteString("\nCOMMIT\n\n*filter\n:CATTLE_FORWARD -\n") buf.WriteString("-F CATTLE_FORWARD\n") - for _, subnet := range sortedForwardSubnets(rules.ForwardSubnets) { - buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -s %s -m conntrack --ctstate NEW,ESTABLISHED,RELATED -j ACCEPT\n", subnet)) - buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -d %s -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT\n", subnet)) + for _, network := range sortedForwardNetworks(rules) { + buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -i %s -s %s -m conntrack --ctstate NEW,ESTABLISHED,RELATED -j ACCEPT\n", network.Bridge, network.Subnet)) + buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -o %s -d %s -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT\n", network.Bridge, network.Subnet)) } for _, pair := range sortedForwardPeers(rules) { - buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -s %s -d %s -j ACCEPT\n", pair.Peer, pair.Local)) + buf.WriteString(fmt.Sprintf("-A CATTLE_FORWARD -o %s -s %s -d %s -j ACCEPT\n", pair.Bridge, pair.Peer, pair.Local)) } // A nat-table MARK is evaluated only for the first packet of a conntracked // flow. In particular, subsequent VXLAN UDP datagrams can reach Docker's @@ -496,6 +515,9 @@ func (w *watcher) apply(rules ruleSet) error { if err := w.restore(w.backend.Restore, []string{"--test", "-n"}, buf.Bytes()); err != nil { return fmt.Errorf("validate hostport rules: %w", err) } + if err := w.restoreRemovedRouteLocalnet(rules); err != nil { + return err + } if err := w.restore(w.backend.Restore, []string{"-n"}, buf.Bytes()); err != nil { return fmt.Errorf("apply hostport rules: %w", err) } @@ -503,6 +525,9 @@ func (w *watcher) apply(rules ruleSet) error { if err := w.insertBaseRules(); err != nil { return fmt.Errorf("apply port base iptables rules: %w", err) } + if err := w.activateRouteLocalnet(rules); err != nil { + return err + } w.applied = rules w.lastApplied = time.Now() @@ -526,16 +551,28 @@ func validateRuleSet(rules ruleSet) error { return err } } - for _, subnet := range rules.ForwardSubnets { + for uuid, subnet := range rules.ForwardSubnets { prefix, err := netip.ParsePrefix(subnet) if err != nil || !prefix.Addr().Is4() || prefix.Bits() == 0 { return fmt.Errorf("invalid IPv4 forward subnet %q", subnet) } + bridge := rules.ForwardBridges[uuid] + if err := validateBridgeName(bridge); err != nil { + return fmt.Errorf("forward network %s: %w", uuid, err) + } + } + for uuid := range rules.ForwardBridges { + if rules.ForwardSubnets[uuid] == "" { + return fmt.Errorf("forward bridge network %s has no local subnet", uuid) + } } for uuid, peers := range rules.ForwardPeers { if rules.ForwardSubnets[uuid] == "" { return fmt.Errorf("forward peer network %s has no local subnet", uuid) } + if rules.ForwardBridges[uuid] == "" { + return fmt.Errorf("forward peer network %s has no managed bridge", uuid) + } for _, peer := range peers { prefix, err := netip.ParsePrefix(peer) if err != nil || !prefix.Addr().Is4() || prefix.Bits() == 0 { @@ -543,18 +580,18 @@ func validateRuleSet(rules ruleSet) error { } } } + for bridge := range rules.RouteLocalnetBridges { + if err := validateBridgeName(bridge); err != nil { + return fmt.Errorf("route_localnet: %w", err) + } + } return nil } func validatePortRule(rule PortRule) error { if rule.Bridge != "" { - if len(rule.Bridge) > 15 { - return fmt.Errorf("invalid bridge name %q", rule.Bridge) - } - for _, char := range rule.Bridge { - if !((char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '_' || char == '-' || char == '.') { - return fmt.Errorf("invalid bridge name %q", rule.Bridge) - } + if err := validateBridgeName(rule.Bridge); err != nil { + return err } } for _, address := range []string{rule.SourceIP, rule.TargetIP} { @@ -569,6 +606,18 @@ func validatePortRule(rule PortRule) error { return nil } +func validateBridgeName(bridge string) error { + if bridge == "" || len(bridge) > 15 { + return fmt.Errorf("invalid bridge name %q", bridge) + } + for _, char := range bridge { + if !((char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '_' || char == '-' || char == '.') { + return fmt.Errorf("invalid bridge name %q", bridge) + } + } + return nil +} + func parsePortRule(bridge, hostIP, targetIP, portDef string) (PortRule, bool) { proto := "tcp" parts := strings.Split(portDef, ":") @@ -606,55 +655,84 @@ func validPort(port string) bool { return err == nil && n > 0 && n <= 65535 } -func forwardSubnetForNetwork(network metadata.Network) string { +type bridgeNetworkConfig struct { + Bridge string + Subnet string + ExternalGateway bool +} + +func bridgeConfigForNetwork(network metadata.Network) (bridgeNetworkConfig, error) { conf, _ := network.Metadata["cniConfig"].(map[string]interface{}) - for _, file := range conf { + keys := make([]string, 0, len(conf)) + for key := range conf { + keys = append(keys, key) + } + sort.Strings(keys) + + var result bridgeNetworkConfig + resultKey := "" + found := false + for _, key := range keys { + file := conf[key] props, _ := file.(map[string]interface{}) cniType, _ := props["type"].(string) + if !isBridgeCNIType(cniType) { + continue + } + bridge, _ := props["bridge"].(string) bridgeSubnet, _ := props["bridgeSubnet"].(string) - - if isBridgeCNIType(cniType) && bridgeSubnet != "" { - return bridgeSubnet + externalGateway, _ := props["skipBridgeConfigureIP"].(bool) + candidate := bridgeNetworkConfig{Bridge: bridge, Subnet: bridgeSubnet, ExternalGateway: externalGateway} + if !found { + result = candidate + resultKey = key + found = true + continue + } + if candidate != result { + return bridgeNetworkConfig{}, fmt.Errorf("conflicting managed bridge entries %q and %q", resultKey, key) } } + return result, nil +} - return "" +func forwardSubnetForNetwork(network metadata.Network) string { + config, _ := bridgeConfigForNetwork(network) + return config.Subnet } func hostportBridgeForNetwork(network metadata.Network) (string, bool) { - conf, _ := network.Metadata["cniConfig"].(map[string]interface{}) - for _, file := range conf { - props, _ := file.(map[string]interface{}) - cniType, _ := props["type"].(string) - bridge, _ := props["bridge"].(string) - if !isBridgeCNIType(cniType) || bridge == "" { - continue - } - externalGateway, _ := props["skipBridgeConfigureIP"].(bool) - return bridge, externalGateway - } - return "", false + config, _ := bridgeConfigForNetwork(network) + return config.Bridge, config.ExternalGateway } func isBridgeCNIType(cniType string) bool { return cniType == "pasture-bridge" || cniType == "rancher-bridge" } -func sortedForwardSubnets(subnetsByNetwork map[string]string) []string { +type forwardNetwork struct{ UUID, Bridge, Subnet string } + +func sortedForwardNetworks(rules ruleSet) []forwardNetwork { + keys := make([]string, 0, len(rules.ForwardSubnets)) + for key := range rules.ForwardSubnets { + keys = append(keys, key) + } + sort.Strings(keys) seen := map[string]bool{} - subnets := []string{} - for _, subnet := range subnetsByNetwork { - if subnet == "" || seen[subnet] { + result := []forwardNetwork{} + for _, key := range keys { + network := forwardNetwork{UUID: key, Bridge: rules.ForwardBridges[key], Subnet: rules.ForwardSubnets[key]} + identity := network.Bridge + "\x00" + network.Subnet + if network.Bridge == "" || network.Subnet == "" || seen[identity] { continue } - seen[subnet] = true - subnets = append(subnets, subnet) + seen[identity] = true + result = append(result, network) } - sort.Strings(subnets) - return subnets + return result } -type forwardPair struct{ Peer, Local string } +type forwardPair struct{ Peer, Local, Bridge string } func sortedForwardPeers(rules ruleSet) []forwardPair { keys := make([]string, 0, len(rules.ForwardPeers)) @@ -667,7 +745,7 @@ func sortedForwardPeers(rules ruleSet) []forwardPair { peers := append([]string(nil), rules.ForwardPeers[key]...) sort.Strings(peers) for _, peer := range peers { - pairs = append(pairs, forwardPair{Peer: peer, Local: rules.ForwardSubnets[key]}) + pairs = append(pairs, forwardPair{Peer: peer, Local: rules.ForwardSubnets[key], Bridge: rules.ForwardBridges[key]}) } } return pairs @@ -696,18 +774,3 @@ func setupKernelParameters() error { } return nil } - -func setupBridgeRouteLocalnet(bridge string) error { - key := "net.ipv4.conf." + bridge + ".route_localnet" - out, err := exec.Command("sysctl", "-n", key).Output() - if err != nil { - return err - } - if strings.TrimSpace(string(out)) == "1" { - return nil - } - cmd := exec.Command("sysctl", "-w", key+"=1") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} diff --git a/hostports/watcher_test.go b/hostports/watcher_test.go index fbd4484..5085799 100644 --- a/hostports/watcher_test.go +++ b/hostports/watcher_test.go @@ -55,36 +55,18 @@ func TestFlatBridgeUsesOwnedDNATMasquerade(t *testing.T) { } } -func TestPreparePortBridgesEnablesLoopbackOnlyWhereNeeded(t *testing.T) { - var bridges []string - w := &watcher{setRouteLocalnet: func(bridge string) error { - bridges = append(bridges, bridge) - return nil - }} - rules := ruleSet{Ports: map[string]PortRule{ - "all": {Bridge: "flatbr0", SourceIP: "0.0.0.0"}, - "loopback": {Bridge: "cattle0", SourceIP: "127.0.0.1"}, - "bound": {Bridge: "private0", SourceIP: "192.0.2.10"}, - }, ForwardSubnets: map[string]string{}} - if err := w.preparePortBridges(rules); err != nil { - t.Fatal(err) - } - if want := []string{"cattle0", "flatbr0"}; !reflect.DeepEqual(bridges, want) { - t.Fatalf("prepared bridges = %v, want %v", bridges, want) - } -} - func TestPerHostPeerMarksOnlyManagedSubnets(t *testing.T) { rules := ruleSet{ Ports: map[string]PortRule{}, ForwardSubnets: map[string]string{"net": "10.51.1.0/24"}, + ForwardBridges: map[string]string{"net": "cattle0"}, ForwardPeers: map[string][]string{"net": {"10.51.2.0/24"}}, } if err := validateRuleSet(rules); err != nil { t.Fatal(err) } batch := string(nftHostportBatch(rules, false)) - if !strings.Contains(batch, "ip saddr 10.51.2.0/24 ip daddr 10.51.1.0/24 meta mark set meta mark | 0x1068 accept") { + if !strings.Contains(batch, "oifname \"cattle0\" ip saddr 10.51.2.0/24 ip daddr 10.51.1.0/24 meta mark set meta mark | 0x1068 accept") { t.Fatal("native nft did not mark an active peer's routed traffic") } if strings.Contains(batch, "ip saddr 0.0.0.0/0") || strings.Contains(batch, "flush ruleset") { @@ -235,7 +217,9 @@ func testRuleSet() ruleSet { Ports: map[string]PortRule{ "container/80:80:8080/tcp": {Bridge: "cattle0", SourceIP: "0.0.0.0", SourcePort: "80", TargetIP: "10.42.1.2", TargetPort: "8080", Protocol: "tcp"}, }, - ForwardSubnets: map[string]string{"network": "10.42.0.0/16"}, + ForwardSubnets: map[string]string{"network": "10.42.0.0/16"}, + ForwardBridges: map[string]string{"network": "cattle0"}, + RouteLocalnetBridges: map[string]bool{"cattle0": true}, } } @@ -291,8 +275,8 @@ func TestManagedSubnetAllowsOutboundAndEstablishedReturnOnly(t *testing.T) { t.Fatal(err) } for _, want := range []string{ - "-A CATTLE_FORWARD -s 10.42.0.0/16 -m conntrack --ctstate NEW,ESTABLISHED,RELATED -j ACCEPT", - "-A CATTLE_FORWARD -d 10.42.0.0/16 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT", + "-A CATTLE_FORWARD -i cattle0 -s 10.42.0.0/16 -m conntrack --ctstate NEW,ESTABLISHED,RELATED -j ACCEPT", + "-A CATTLE_FORWARD -o cattle0 -d 10.42.0.0/16 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT", } { if !strings.Contains(iptablesRules, want) { t.Fatalf("iptables rules missing %q", want) @@ -303,8 +287,8 @@ func TestManagedSubnetAllowsOutboundAndEstablishedReturnOnly(t *testing.T) { } nftRules := string(nftHostportBatch(rules, false)) for _, want := range []string{ - "ip saddr 10.42.0.0/16 ct state new,established,related meta mark set meta mark | 0x1068 accept", - "ip daddr 10.42.0.0/16 ct state established,related meta mark set meta mark | 0x1068 accept", + "iifname \"cattle0\" ip saddr 10.42.0.0/16 ct state new,established,related meta mark set meta mark | 0x1068 accept", + "oifname \"cattle0\" ip daddr 10.42.0.0/16 ct state established,related meta mark set meta mark | 0x1068 accept", } { if !strings.Contains(nftRules, want) { t.Fatalf("nft rules missing %q", want) @@ -343,10 +327,10 @@ func TestApplyIptablesValidatesBeforeUpdatingOrRepairingHooks(t *testing.T) { backend: firewall.Backend{Mode: firewall.IptablesNFT, Command: "iptables-nft", Restore: "iptables-nft-restore"}, restoreRules: func(name string, args []string, data []byte) error { calls = append(calls, name+" "+strings.Join(args, " ")) - if !strings.Contains(string(data), "-A CATTLE_FORWARD -s 10.42.0.0/16") { + if !strings.Contains(string(data), "-A CATTLE_FORWARD -i cattle0 -s 10.42.0.0/16") { t.Fatal("missing forward rule") } - if !strings.Contains(string(data), "-A CATTLE_FORWARD -s 10.51.2.0/24 -d 10.42.0.0/16 -j ACCEPT") { + if !strings.Contains(string(data), "-A CATTLE_FORWARD -o cattle0 -s 10.51.2.0/24 -d 10.42.0.0/16 -j ACCEPT") { t.Fatal("missing bounded peer forward rule") } if !strings.Contains(string(data), "-A CATTLE_FORWARD -m conntrack --ctstate DNAT -d 10.42.1.2 -p tcp -m tcp --dport 8080 -j ACCEPT") { @@ -458,10 +442,12 @@ func TestNativeNFTUsesOwnedTableAndSingleCheckedBatch(t *testing.T) { "type nat hook output priority -101", "type nat hook postrouting priority 99", "type filter hook forward priority -1", + "type filter hook prerouting priority -300", + "iifname \"cattle0\" ip daddr 127.0.0.0/8 drop", "meta mark & 0x1068 == 0x1068 accept", "ct status dnat ip daddr 10.42.1.2 tcp dport 8080 meta mark set meta mark | 0x1068 accept", - "ip saddr 10.42.0.0/16 ct state new,established,related meta mark set meta mark | 0x1068 accept", - "ip daddr 10.42.0.0/16 ct state established,related meta mark set meta mark | 0x1068 accept", + "iifname \"cattle0\" ip saddr 10.42.0.0/16 ct state new,established,related meta mark set meta mark | 0x1068 accept", + "oifname \"cattle0\" ip daddr 10.42.0.0/16 ct state established,related meta mark set meta mark | 0x1068 accept", } { if !strings.Contains(string(checks), expected) { t.Fatalf("batch missing %q", expected)