From 82a61e3ef37e95d676e4c75291fa5fd5dc3b0336 Mon Sep 17 00:00:00 2001
From: Jory Irving <jory@jory.dev>
Date: Mon, 12 Oct 2026 13:54:33 -0600
Subject: [PATCH] fix(helm): decouple liveness from database availability
 (#1171)

---
 .github/workflows/helm-lint.yaml |  8 +++-
 charts/dispatch/values.yaml      |  9 ++--
 scripts/check-helm-probes.py     | 81 ++++++++++++++++++++++++++++++++
 3 files changed, 92 insertions(+), 6 deletions(-)
 create mode 100644 scripts/check-helm-probes.py

diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml
index c8d1e404..ac49d76f 100644
--- a/.github/workflows/helm-lint.yaml
+++ b/.github/workflows/helm-lint.yaml
@@ -4,6 +4,7 @@ on:
   pull_request:
     paths:
       - charts/dispatch/**
+      - scripts/check-helm-probes.py
       - .github/workflows/helm-lint.yaml
 
 jobs:
@@ -27,7 +28,12 @@ jobs:
         run: helm lint charts/dispatch
 
       - name: Template chart (default values)
-        run: helm template dispatch charts/dispatch
+        run: helm template dispatch charts/dispatch > /tmp/dispatch-rendered.yaml
+
+      - name: Check rendered probes
+        run: |
+          python3 -m pip install --disable-pip-version-check PyYAML==6.0.2
+          python3 scripts/check-helm-probes.py /tmp/dispatch-rendered.yaml
 
       - name: Template chart (custom values)
         run: |
diff --git a/charts/dispatch/values.yaml b/charts/dispatch/values.yaml
index c0dca9fa..977de0c4 100644
--- a/charts/dispatch/values.yaml
+++ b/charts/dispatch/values.yaml
@@ -42,15 +42,14 @@ controllers:
           - secretRef:
               name: dispatch
         probes:
-          # Probes target /api/health (public, excluded from the auth
-          # middleware) so basic-auth deployments pass them and readiness
-          # reflects real DB health (503 when the database is unreachable).
+          # TCP liveness checks that the server process is accepting connections
+          # without coupling restarts to database availability. Readiness targets
+          # /api/health (public, excluded from auth middleware) and reflects DB health.
           liveness:
             enabled: true
             custom: true
             spec:
-              httpGet:
-                path: /api/health
+              tcpSocket:
                 port: 3000
               initialDelaySeconds: 10
               periodSeconds: 10
diff --git a/scripts/check-helm-probes.py b/scripts/check-helm-probes.py
new file mode 100644
index 00000000..ef94f707
--- /dev/null
+++ b/scripts/check-helm-probes.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Assert chart probe configuration and demonstrate TCP vs HTTP health semantics."""
+import socket
+import sys
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+import yaml
+
+
+def fail(message):
+    raise SystemExit(message)
+
+
+if len(sys.argv) != 2:
+    fail(f"usage: {sys.argv[0]} rendered.yaml")
+
+with open(sys.argv[1], encoding="utf-8") as rendered_file:
+    documents = yaml.safe_load_all(rendered_file)
+    deployment = next(
+        (doc for doc in documents if isinstance(doc, dict)
+         and doc.get("kind") == "Deployment" and doc.get("metadata", {}).get("labels", {}).get("app.kubernetes.io/controller") == "main"),
+        None,
+    )
+
+if deployment is None:
+    fail("rendered YAML does not contain the main Deployment")
+
+containers = deployment["spec"]["template"]["spec"]["containers"]
+container = next((item for item in containers if item.get("name") == "main"), None)
+if container is None:
+    fail("main Deployment has no main container")
+
+liveness = container.get("livenessProbe", {})
+if liveness.get("tcpSocket", {}).get("port") != 3000 or "httpGet" in liveness:
+    fail("main liveness probe must use TCP port 3000, not HTTP")
+readiness = container.get("readinessProbe", {})
+http_get = readiness.get("httpGet", {})
+if http_get.get("path") != "/api/health" or http_get.get("port") != 3000:
+    fail("main readiness probe must GET /api/health on port 3000")
+
+# Model a live process with an unhealthy dependency: HTTP readiness fails while
+# the listening socket remains alive. A stopped server then fails the TCP check.
+class UnhealthyHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        self.send_response(503)
+        self.end_headers()
+
+    def log_message(self, *_args):
+        pass
+
+server = HTTPServer(("127.0.0.1", 0), UnhealthyHandler)
+thread = threading.Thread(target=server.serve_forever, daemon=True)
+thread.start()
+port = server.server_address[1]
+try:
+    with socket.create_connection(("127.0.0.1", port), timeout=1):
+        pass
+    try:
+        from urllib.request import urlopen
+        urlopen(f"http://127.0.0.1:{port}/api/health", timeout=1)
+        fail("simulated unhealthy dependency unexpectedly passed readiness")
+    except Exception as error:
+        if getattr(error, "code", None) != 503:
+            raise
+finally:
+    server.shutdown()
+    server.server_close()
+    thread.join(timeout=2)
+
+for _ in range(20):
+    try:
+        socket.create_connection(("127.0.0.1", port), timeout=0.1)
+    except OSError:
+        break
+    time.sleep(0.05)
+else:
+    fail("stopped server unexpectedly passed TCP liveness")
+
+print("Helm probe configuration and simulated dependency-health behavior passed")