Support PostgreSQL connection secrets

.github/workflows/helm-e2e.yaml

@@ -36,6 +36,10 @@ jobs:
         go-version-file: go.mod
     - name: Setup Helm
       uses: azure/setup-helm@v4
+    - name: Test Helm chart
+      run: |
+        helm plugin install https://github.com/helm-unittest/helm-unittest.git --version 1.0.3 --verify=false
+        make helm-test
     - name: Cache micro-VM assets
       uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
       with:

Makefile

@@ -146,6 +146,10 @@ clean:
 helm-template:
 	@./hack/render-manifests.sh
 
+.PHONY: helm-test
+helm-test:
+	@helm unittest charts/substrate
+
 # Verify that manifests/ate-install/ matches the chart output. Used in CI.
 .PHONY: verify-helm-template
 verify-helm-template:

charts/substrate/README.md

@@ -41,8 +41,14 @@ See `values.yaml` for the full set; the important keys:
 | Key | Default | Notes |
 |-----|---------|-------|
 | `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance |
-| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL |
-| `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema |
+| `postgres.readWriteConnectionStringSecretRef` | managed Secret | Read the read/write connection from a Secret |
+| `postgres.ownerConnectionStringSecretRef` | managed Secret | Read the owner connection from a Secret |
+| `postgres.bootstrap` | `true` | Create missing fixed users, roles, schema, and grants on startup; the direct `ateapi` binary defaults to `false` |
+| `postgres.readWriteRole` | `substrate_readwrite` | Role assumed by read/write connections; configurable when bootstrap is disabled |
+| `postgres.ownerRole` | `substrate_owner` | Role assumed by owner connections; configurable when bootstrap is disabled |
+| `postgres.adminSecretRef` | `postgres-admin` | Select the administrator Secret |
+| `postgres.pool.maxConnLifetime` | `""` (pgx default) | Maximum physical connection lifetime; bounds Secret credential turnover |
+| `postgres.schema` | `substrate` | Store the Substrate tables in this PostgreSQL schema |
 | `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size |
 | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots |
 | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` |
@@ -57,3 +63,45 @@ See `values.yaml` for the full set; the important keys:
 | `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` |
 | `otel.logs.enabled` | `true` | Set to `false` to export no logs. Gates both OTLP log sources: ateapi's actor lifecycle events and the router access log |
 | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` |
+
+## PostgreSQL credential rotation
+
+The bundled PostgreSQL pod creates `postgres.database` (`atepg` by default). The chart uses the same database name in both managed connection Secrets. Under Kagent, the umbrella chart supplies its bundled PostgreSQL Service address and `kagent` database name to Substrate's fixed credential templates.
+
+The Substrate binary also has these fixed development usernames and passwords. During bootstrap it checks both connection Secrets against those constants, then uses the constants to create missing users. Bootstrap runs on every `ateapi` pod start while `postgres.bootstrap=true`. It never changes an existing user's password. To use different credentials, create the users yourself and disable bootstrap.
+
+The chart's default administrator and application passwords are fixed, published values. This bundled bootstrap setup is for development and evaluation, not production. For production, provision unique users and permissions outside Substrate, supply connection Secrets, and set `postgres.bootstrap=false`.
+
+The Substrate binary executes `cmd/ateapi/internal/store/atepg/identity.sql` from the Substrate repository during bootstrap. Operators can run that same file after supplying its transaction-local settings; it is separate from table migrations.
+For manual provisioning with custom chart role names, set the optional
+`substrate.bootstrap_owner_role` and `substrate.bootstrap_readwrite_role`
+transaction-local settings before running the file. They default to the fixed
+development names; bundled binary bootstrap passes those names explicitly.
+
+Substrate mounts connection Secrets as projected files. Kubernetes updates
+these files when the Secret changes. Substrate reads the current value for
+each new physical connection.
+
+`postgres.pool.maxConnLifetime` bounds how long established connections may
+continue using an old credential; rotation is not immediate. Keep old and new
+credentials valid long enough for Kubernetes projection and connection
+turnover.
+
+When rotation changes a login username, grant the applicable configured group
+role. Bootstrap uses the fixed `substrate_readwrite` and `substrate_owner` roles.
+
+For bundled PostgreSQL, first create replacement logins and connection Secrets.
+Then set `postgres.bootstrap=false` and set both connection Secret references
+in the same Helm upgrade. PostgreSQL remains bundled, but Substrate stops
+creating or verifying the fixed login users. The bundled PostgreSQL pod still
+uses its administrator Secret; the API server no longer mounts it. Bootstrap
+never changes the password of an existing login.
+
+Substrate runs `SET ROLE` for each new connection. It rejects a login without
+the required membership.
+
+Set `postgres.bootstrap=false` for a BYO database. Create both group roles and
+all grants before installation, then set `postgres.readWriteRole` and
+`postgres.ownerRole` to those names. Use distinct roles and table schemas for
+separate installs sharing one database. Give each install separate logins and
+grant each login membership only in its install's roles.

charts/substrate/templates/_helpers.tpl

@@ -78,6 +78,27 @@ Plaintext HTTP URL that clients use to reach atenet-router.
 {{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}}
 {{- end -}}
 
+{{- define "substrate.postgres.adminSecretName" -}}
+{{- .Values.postgres.adminSecretRef.name | default "postgres-admin" -}}
+{{- end -}}
+
+{{- define "substrate.postgres.readWriteSecretName" -}}
+{{- .Values.postgres.readWriteConnectionStringSecretRef.name | default (include "substrate.fullname" (list "postgres-readwrite" .)) -}}
+{{- end -}}
+
+{{- define "substrate.postgres.ownerSecretName" -}}
+{{- .Values.postgres.ownerConnectionStringSecretRef.name | default (include "substrate.fullname" (list "postgres-owner" .)) -}}
+{{- end -}}
+
+{{/* Fixed bundled identities. Callers supply only the database endpoint. */}}
+{{- define "substrate.postgres.readWriteConnectionString" -}}
+{{- printf "postgresql://substrate_readwrite_user:substrate-readwrite@%s:5432/%s?%s" .host .database .params -}}
+{{- end -}}
+
+{{- define "substrate.postgres.ownerConnectionString" -}}
+{{- printf "postgresql://substrate_admin_user:substrate-admin@%s:5432/%s?%s" .host .database .params -}}
+{{- end -}}
+
 {{/*
 OTLP endpoint a signal exports to, or empty when the signal is disabled or no
 endpoint resolves. The per-signal endpoint wins over the generic one, matching

charts/substrate/templates/ate-api-server-envvars.yaml

@@ -14,14 +14,25 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) }}
-{{- fail "postgres.connectionString is required when postgres.enabled=false" }}
+{{- if not (kindIs "bool" .Values.postgres.bootstrap) }}
+{{- fail "postgres.bootstrap must be true or false" }}
+{{- end }}
+{{- if and .Values.postgres.enabled .Values.postgres.bootstrap (or .Values.postgres.readWriteConnectionStringSecretRef.name .Values.postgres.ownerConnectionStringSecretRef.name) }}
+{{- fail "postgres.bootstrap requires the chart-managed application Secrets; disable bootstrap for operator-managed Secrets" }}
+{{- end }}
+{{- if and (or (not .Values.postgres.enabled) (not .Values.postgres.bootstrap)) (not .Values.postgres.readWriteConnectionStringSecretRef.name) }}
+{{- fail "postgres.readWriteConnectionStringSecretRef.name is required when bootstrap is disabled or PostgreSQL is external" }}
+{{- end }}
+{{- if and (or (not .Values.postgres.enabled) (not .Values.postgres.bootstrap)) (not .Values.postgres.ownerConnectionStringSecretRef.name) }}
+{{- fail "postgres.ownerConnectionStringSecretRef.name is required when bootstrap is disabled or PostgreSQL is external" }}
+{{- end }}
+{{- if and .Values.postgres.bootstrap (not .Values.postgres.enabled) (not .Values.postgres.adminSecretRef.name) }}
+{{- fail "postgres.adminSecretRef.name is required when bootstrap is enabled without bundled PostgreSQL" }}
 {{- end }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ .Values.ateApiServerEnvVarsConfigMap }}
   namespace: {{ .Release.Namespace }}
 data:
-  ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }}
   ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }}

charts/substrate/templates/ate-api-server.yaml

@@ -14,6 +14,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- $readWriteConnectionStringSecretRef := .Values.postgres.readWriteConnectionStringSecretRef -}}
+{{- $ownerConnectionStringSecretRef := .Values.postgres.ownerConnectionStringSecretRef -}}
+{{- $bootstrapEnabled := .Values.postgres.bootstrap -}}
+{{- if not (kindIs "bool" $bootstrapEnabled) -}}
+{{- fail "postgres.bootstrap must be true or false" -}}
+{{- end -}}
+{{- if and $bootstrapEnabled (or (ne .Values.postgres.readWriteRole "substrate_readwrite") (ne .Values.postgres.ownerRole "substrate_owner")) -}}
+{{- fail "postgres bootstrap requires readWriteRole=substrate_readwrite and ownerRole=substrate_owner" -}}
+{{- end -}}
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -85,8 +95,19 @@ spec:
         - "--grpc-listen-addr=0.0.0.0:443"
         - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem"
         - "--authentication-config=/etc/ateapi/authentication/authentication.yaml"
-        - "--postgres-connection-string=@env"
+        - "--postgres-read-write-connection-string=@file:/etc/ateapi/postgres/read-write/connection-string"
+        - "--postgres-owner-connection-string=@file:/etc/ateapi/postgres/owner/connection-string"
+        - {{ printf "--postgres-read-write-role=%s" .Values.postgres.readWriteRole | quote }}
+        - {{ printf "--postgres-owner-role=%s" .Values.postgres.ownerRole | quote }}
         - "--postgres-schema=@env"
+        - {{ printf "--postgres-bootstrap=%t" $bootstrapEnabled | quote }}
+{{- if $bootstrapEnabled }}
+        - "--postgres-admin-username-file=/etc/ateapi/postgres/admin/username"
+        - "--postgres-admin-password-file=/etc/ateapi/postgres/admin/password"
+{{- end }}
+{{- with .Values.postgres.pool.maxConnLifetime }}
+        - {{ printf "--postgres-max-conn-lifetime=%s" . | quote }}
+{{- end }}
         - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json"
         - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json"
         - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443"
@@ -140,6 +161,17 @@ spec:
         - { name: actor-id-ca-pool,    mountPath: /run/actor-id-ca-pool, readOnly: true }
         - { name: podidentity,         mountPath: /run/podidentity.podcert.ate.dev, readOnly: true }
         - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true }
+        - name: postgres-read-write-connection
+          mountPath: /etc/ateapi/postgres/read-write
+          readOnly: true
+        - name: postgres-owner-connection
+          mountPath: /etc/ateapi/postgres/owner
+          readOnly: true
+{{- if $bootstrapEnabled }}
+        - name: postgres-admin
+          mountPath: /etc/ateapi/postgres/admin
+          readOnly: true
+{{- end }}
         ports:
         - containerPort: 443
         - name: prometheus
@@ -201,6 +233,34 @@ spec:
                 matchLabels:
                   podcert.ate.dev/canarying: live
               path: trust-bundle.pem
+      - name: postgres-read-write-connection
+        projected:
+          sources:
+          - secret:
+              name: {{ include "substrate.postgres.readWriteSecretName" . | quote }}
+              items:
+              - key: {{ get $readWriteConnectionStringSecretRef "key" | default "readWriteConnectionString" | quote }}
+                path: connection-string
+      - name: postgres-owner-connection
+        projected:
+          sources:
+          - secret:
+              name: {{ include "substrate.postgres.ownerSecretName" . | quote }}
+              items:
+              - key: {{ get $ownerConnectionStringSecretRef "key" | default "ownerConnectionString" | quote }}
+                path: connection-string
+{{- if $bootstrapEnabled }}
+      - name: postgres-admin
+        projected:
+          sources:
+          - secret:
+              name: {{ include "substrate.postgres.adminSecretName" . | quote }}
+              items:
+              - key: {{ .Values.postgres.adminSecretRef.usernameKey | quote }}
+                path: username
+              - key: {{ .Values.postgres.adminSecretRef.passwordKey | quote }}
+                path: password
+{{- end }}
 ---
 apiVersion: policy/v1
 kind: PodDisruptionBudget

charts/substrate/templates/postgres-secrets.yaml

@@ -0,0 +1,54 @@
+{{/*
+Copyright 2026 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.postgres.enabled }}
+{{- $host := printf "%s.%s.svc" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace -}}
+{{- /* The default service-DNS CA signs with Ed25519, which pgx cannot hash for SCRAM channel binding. TLS verification and client certificates remain required. */ -}}
+{{- $tls := "sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem&channel_binding=disable" -}}
+{{- if not .Values.postgres.adminSecretRef.name }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "substrate.postgres.adminSecretName" . }}
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+stringData:
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: postgres
+---
+{{- end }}
+{{- if not .Values.postgres.readWriteConnectionStringSecretRef.name }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "substrate.postgres.readWriteSecretName" . }}
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+stringData:
+  readWriteConnectionString: {{ include "substrate.postgres.readWriteConnectionString" (dict "host" $host "database" .Values.postgres.database "params" $tls) | quote }}
+---
+{{- end }}
+{{- if not .Values.postgres.ownerConnectionStringSecretRef.name }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "substrate.postgres.ownerSecretName" . }}
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+stringData:
+  ownerConnectionString: {{ include "substrate.postgres.ownerConnectionString" (dict "host" $host "database" .Values.postgres.database "params" $tls) | quote }}
+{{- end }}
+{{- end }}

charts/substrate/templates/postgres.yaml

@@ -34,9 +34,8 @@ data:
     # health checks, the workload's idempotent database bootstrap, and the
     # tls-reloader sidecar's configuration reloads.
     local all all trust
-    # PostgreSQL verifies client certificates against the pod-identity CA. It
-    # does not need its own serving CA because it never verifies its server certificate.
-    hostssl all all all trust clientcert=verify-ca
+    # PostgreSQL requires a password and a pod-identity client certificate.
+    hostssl all all all scram-sha-256 clientcert=verify-ca
   reload-tls.sh: |
     # PostgreSQL opens ssl_cert_file, ssl_key_file and ssl_ca_file at startup
     # and on SIGHUP, and nowhere else. The kubelet replaces the projected pod
@@ -67,7 +66,7 @@ data:
       # Starting empty also means a restart of this container costs one
       # redundant reload rather than a missed one.
       if [ "${current}" != "${reloaded}" ] \
-        && psql -U postgres -d postgres -Atc 'SELECT pg_reload_conf()' >/dev/null 2>&1; then
+        && psql -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -Atc 'SELECT pg_reload_conf()' >/dev/null 2>&1; then
         reloaded="${current}"
         echo "$(date -u +%FT%TZ) reloaded TLS configuration"
       fi
@@ -130,6 +129,14 @@ spec:
         command:
         - /bin/sh
         - /etc/postgresql/reload-tls.sh
+        env:
+        - name: POSTGRES_DB
+          value: {{ .Values.postgres.database | quote }}
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "substrate.postgres.adminSecretName" . }}
+              key: {{ .Values.postgres.adminSecretRef.usernameKey }}
         volumeMounts:
         - name: config
           mountPath: /etc/postgresql
@@ -149,38 +156,32 @@ spec:
       containers:
       - name: postgres
         image: {{ include "substrate.thirdPartyImage" (list .Values.images.postgres .) }}
-        lifecycle:
-          postStart:
-            exec:
-              command:
-              - /bin/sh
-              - -ec
-              - |
-                until psql -U postgres -d postgres -Atc 'SELECT 1' >/dev/null 2>&1; do
-                  sleep 1
-                done
-                if ! psql -U postgres -d postgres -Atc \
-                  "SELECT 1 FROM pg_database WHERE datname = 'atepg'" | grep -qx 1; then
-                  createdb -U postgres atepg
-                fi
         env:
         - name: POSTGRES_DB
-          value: atepg
-        - name: POSTGRES_HOST_AUTH_METHOD
-          value: trust
+          value: {{ .Values.postgres.database | quote }}
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "substrate.postgres.adminSecretName" . }}
+              key: {{ .Values.postgres.adminSecretRef.usernameKey }}
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "substrate.postgres.adminSecretName" . }}
+              key: {{ .Values.postgres.adminSecretRef.passwordKey }}
         - name: PGDATA
           value: /var/lib/postgresql/data/pgdata
         ports:
         - name: postgres
           containerPort: 5432
         readinessProbe:
           exec:
-            command: ["/bin/sh", "-ec", "psql -U postgres -d atepg -Atc 'SELECT 1' >/dev/null"]
+            command: ["/bin/sh", "-ec", "pg_isready -U \"$POSTGRES_USER\" -d \"$POSTGRES_DB\""]
           initialDelaySeconds: 2
           periodSeconds: 2
         livenessProbe:
           exec:
-            command: ["pg_isready", "-U", "postgres", "-d", "postgres"]
+            command: ["/bin/sh", "-ec", "pg_isready -U \"$POSTGRES_USER\" -d \"$POSTGRES_DB\""]
           initialDelaySeconds: 10
           periodSeconds: 10
         args: ["-c", "config_file=/etc/postgresql/postgresql.conf"]