test(tmachine): add a K3s HA conformance lane with PostgreSQL and mTLS

.github/workflows/branch-e2e.yml

@@ -256,6 +256,7 @@ jobs:
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
           {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+          {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
           {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
         ]

.github/workflows/integration-runner.yml

@@ -26,6 +26,7 @@ on:
           [
             {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
             {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+            {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
             {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
             {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
           ]

.github/workflows/integration-test.yml

@@ -29,6 +29,7 @@ on:
           [
             {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
             {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+            {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
             {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
             {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
           ]

.github/workflows/release-dev.yml

@@ -127,6 +127,7 @@ jobs:
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
           {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+          {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
           {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
         ]

.github/workflows/release-tag.yml

@@ -178,6 +178,7 @@ jobs:
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
           {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+          {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
           {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
         ]

CI.md

@@ -144,16 +144,18 @@ push-triggered workflow jobs themselves.
 
 ### K3s conformance version baseline
 
-The tmachine `ubuntu-k3s` conformance lane pins Agent Sandbox v0.5.0 as the
-compatibility baseline for the v1beta1 Sandbox API. It does not track the local
+The tmachine `ubuntu-k3s` conformance lanes pin Agent Sandbox v0.5.0 as the
+compatibility baseline for the v1beta1 Sandbox API. They do not track the local
 K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through
 its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version.
 
-The `ubuntu-k3s` lane registers the gateway's ClusterIP Service directly, so
-recreating the Service requires reprovisioning the fixture. Before conformance,
-every installer waits up to five minutes for a connected gateway; interactive
-shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*`
-artifacts.
+The `ubuntu-k3s` lanes register the gateway's ClusterIP Service directly, so
+recreating the Service requires reprovisioning the fixture. The `k3s` lane runs
+one plaintext gateway. The `k3s-ha` lane runs the supported HA configuration:
+three gateway replicas sharing PostgreSQL, serving TLS, with the CLI connecting
+over mTLS. Before conformance, every installer waits up to five minutes for a
+connected gateway; interactive shells skip the wait. K3s failures upload
+diagnostics as `tmachine-diagnostics-*` artifacts.
 
 ### Run only the policy advisor conformance tests
 

tests/ansible/playbooks/openshell-k3s-ha.yaml

@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Stage PostgreSQL for the HA gateway
+  hosts: all
+  become: true
+  gather_facts: false
+  tasks:
+    - name: Wait for SSH
+      ansible.builtin.wait_for_connection:
+
+    # The install layer boots fresh, so the K3s API may still be starting.
+    - name: Apply OpenShell namespace
+      ansible.builtin.command:
+        argv: [/usr/local/bin/k3s, kubectl, apply, --filename, "-"]
+        stdin: |
+          apiVersion: v1
+          kind: Namespace
+          metadata:
+            name: openshell
+      register: k3s_namespace
+      until: k3s_namespace.rc == 0
+      retries: 30
+      delay: 2
+
+    - name: Apply PostgreSQL fixture
+      ansible.builtin.command:
+        argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, "-"]
+        stdin: "{{ lookup('ansible.builtin.file', openshell_postgres_fixture) }}"
+
+    # Gateway Pods that start before the database crash-loop with growing backoff.
+    - name: Wait for PostgreSQL
+      ansible.builtin.command:
+        argv:
+          - /usr/local/bin/k3s
+          - kubectl
+          - --namespace
+          - openshell
+          - rollout
+          - status
+          - deployment/openshell-e2e-postgres
+          - --timeout=300s
+      changed_when: false
+
+- import_playbook: openshell-k3s.yaml
+  vars:
+    openshell_k3s_helm_values_overlay: |
+      replicaCount: 3
+      workload:
+        kind: deployment
+      server:
+        externalDbSecret: openshell-e2e-postgres-credentials
+        disableTls: false
+
+- name: Register the HA gateway over mTLS
+  hosts: all
+  become: true
+  gather_facts: false
+  tasks:
+    # The chart's server certificate covers the Service DNS name but not its
+    # ClusterIP, and the node resolver cannot reach cluster DNS.
+    - name: Resolve the gateway Service name to its ClusterIP
+      ansible.builtin.lineinfile:
+        path: /etc/hosts
+        regexp: '\sopenshell\.openshell\.svc\.cluster\.local$'
+        line: "{{ k3s_gateway_address.stdout.split(':')[0] }} openshell.openshell.svc.cluster.local"
+
+    - name: Read chart client TLS Secret
+      ansible.builtin.command:
+        argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, get, secret, openshell-client-tls, --output, json]
+      register: k3s_client_tls
+      changed_when: false
+      no_log: true
+
+    - name: Create guest mTLS directory
+      ansible.builtin.file:
+        path: /home/tmachine/.config/openshell/gateways/tmachine/mtls
+        state: directory
+        owner: tmachine
+        group: tmachine
+        mode: "0700"
+
+    - name: Write guest client TLS bundle
+      ansible.builtin.copy:
+        content: "{{ (k3s_client_tls.stdout | from_json).data[item] | b64decode }}"
+        dest: "/home/tmachine/.config/openshell/gateways/tmachine/mtls/{{ item }}"
+        owner: tmachine
+        group: tmachine
+        mode: "0600"
+      loop: [ca.crt, tls.crt, tls.key]
+      no_log: true
+
+    # The base playbook registered a plaintext endpoint; replace it.
+    - name: Remove baseline guest gateway registration
+      become: false
+      ansible.builtin.command:
+        argv: [/usr/local/bin/openshell, gateway, remove, tmachine]
+
+    - name: Register guest gateway
+      ansible.builtin.include_role:
+        name: openshell_client
+        apply:
+          become: false
+      vars:
+        openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}"

tests/ansible/playbooks/openshell-k3s.yaml

@@ -87,6 +87,12 @@
             disableTls: true
             telemetryEnabled: false
 
+    - name: Write OpenShell Helm values overlay
+      ansible.builtin.copy:
+        dest: /var/lib/openshell/artifacts/values-overlay.yaml
+        mode: "0600"
+        content: "{{ openshell_k3s_helm_values_overlay | default('{}') }}"
+
     - name: Install Agent Sandbox
       ansible.builtin.command:
         argv:
@@ -132,6 +138,8 @@
           - --create-namespace
           - --values
           - /var/lib/openshell/artifacts/values.yaml
+          - --values
+          - /var/lib/openshell/artifacts/values-overlay.yaml
           - --wait
           - --timeout=5m
       environment:

tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh

@@ -43,8 +43,13 @@ collect() {
   collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \
     -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions'
   collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp
-  collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --tail=300 --timestamps
-  collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --previous --tail=300 --timestamps
+  # Gateway Pods are named by a StatefulSet or a Deployment, depending on the installer.
+  mapfile -t gateway_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
+    --selector app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell --output name 2>/dev/null)
+  for pod in "${gateway_pods[@]}"; do
+    collect "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --tail=300 --timestamps
+    collect "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous --tail=300 --timestamps
+  done
   collect 'Memory' free -m
   collect 'Disk' df -h / /var/lib/rancher/k3s
 } | sed -E \

tests/config.nix

@@ -19,6 +19,14 @@ let
     OVMF = firmwarePkgs.OVMF;
   };
   qemu = qemuPkgs.qemu.override { hostCpuOnly = true; };
+  k3sInputs = {
+    agent_sandbox_version = "0.5.0";
+    openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell";
+    openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar";
+    openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz";
+    openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar";
+    openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar";
+  };
   config = (pkgs.formats.yaml { }).generate "tmachine-config.yaml" {
     machines = [
       {
@@ -85,13 +93,14 @@ let
         name = "k3s";
         use_galaxy = false;
         playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ];
-        inputs = {
-          agent_sandbox_version = "0.5.0";
-          openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell";
-          openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar";
-          openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz";
-          openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar";
-          openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar";
+        inputs = k3sInputs;
+      }
+      {
+        name = "k3s-ha";
+        use_galaxy = false;
+        playbooks = [ "ansible/playbooks/openshell-k3s-ha.yaml" ];
+        inputs = k3sInputs // {
+          openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml";
         };
       }
       {