From 19671550b8ac5311417dab215958c0b5dd9224bf Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 9 Oct 2026 13:19:02 -0700 Subject: [PATCH 1/6] test(tmachine): add K3s PostgreSQL and mTLS conformance installer Signed-off-by: Matthew Grossman --- .../playbooks/openshell-k3s-ha-tls.yaml | 112 ++++++++++++++++++ tests/config.nix | 14 +++ 2 files changed, 126 insertions(+) create mode 100644 tests/ansible/playbooks/openshell-k3s-ha-tls.yaml diff --git a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml new file mode 100644 index 0000000000..51df48660c --- /dev/null +++ b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- import_playbook: openshell-k3s.yaml + +- name: Add PostgreSQL and TLS to the K3s test installation + hosts: all + become: true + gather_facts: false + tasks: + - name: Copy PostgreSQL fixture + ansible.builtin.copy: + src: "{{ openshell_postgres_fixture }}" + dest: /var/lib/openshell/artifacts/postgres.yaml + mode: "0600" + + - name: Apply PostgreSQL fixture + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, /var/lib/openshell/artifacts/postgres.yaml] + + - name: Wait for PostgreSQL + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s] + changed_when: false + + - name: Write PostgreSQL and TLS values overlay + ansible.builtin.copy: + dest: /var/lib/openshell/artifacts/values-ha-tls.yaml + mode: "0600" + content: | + replicaCount: 3 + workload: + kind: deployment + server: + externalDbSecret: openshell-e2e-postgres-credentials + disableTls: false + auth: + # Kubernetes has no mTLS user-identity mode. Require the client + # certificate at the TLS layer and use the fixture's dev user. + allowUnauthenticatedUsers: true + + - name: Apply PostgreSQL and TLS overlay to OpenShell + ansible.builtin.command: + argv: + - /usr/local/bin/helm + - upgrade + - openshell + - /var/lib/openshell/artifacts/helm-chart.tgz + - --namespace + - openshell + - --values + - /var/lib/openshell/artifacts/values.yaml + - --values + - /var/lib/openshell/artifacts/values-ha-tls.yaml + - --wait + - --timeout=5m + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + + - name: Restart gateway forwarder after TLS upgrade + ansible.builtin.systemd_service: + name: openshell-k3s-port-forward.service + state: restarted + + - name: Read chart client TLS Secret + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, get, secret, openshell-client-tls, --output, json] + register: k3s_client_tls + changed_when: false + no_log: true + + - name: Create guest mTLS directory + ansible.builtin.file: + path: /home/tmachine/.config/openshell/gateways/tmachine/mtls + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Write guest client TLS bundle + ansible.builtin.copy: + content: "{{ (k3s_client_tls.stdout | from_json).data[item] | b64decode }}" + dest: "/home/tmachine/.config/openshell/gateways/tmachine/mtls/{{ item }}" + owner: tmachine + group: tmachine + mode: "0600" + loop: [ca.crt, tls.crt, tls.key] + no_log: true + + - name: Remove baseline guest gateway registration + become: false + ansible.builtin.command: + argv: [/usr/local/bin/openshell, gateway, remove, tmachine] + + - name: Register guest gateway + ansible.builtin.include_role: + name: openshell_client + apply: + become: false + vars: + openshell_client_gateway_endpoint: https://localhost:17670 + + - name: Wait for authenticated gateway API + become: false + ansible.builtin.command: + argv: [/usr/local/bin/openshell, sandbox, list, --output, json] + register: k3s_gateway_api + until: k3s_gateway_api.rc == 0 + retries: 24 + delay: 5 + changed_when: false diff --git a/tests/config.nix b/tests/config.nix index 26d27b712b..f5e414fb80 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -94,6 +94,20 @@ let openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar"; }; } + { + name = "k3s-ha-tls"; + use_galaxy = false; + playbooks = [ "ansible/playbooks/openshell-k3s-ha-tls.yaml" ]; + inputs = { + openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml"; + agent_sandbox_version = "0.5.0"; + openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; + openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar"; + openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz"; + openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar"; + openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar"; + }; + } { name = "none"; use_galaxy = false; From e772f8999eebe14114bf445a88e8ee3cb789b3bb Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 9 Oct 2026 13:34:31 -0700 Subject: [PATCH 2/6] fix(tmachine): reach the HA TLS gateway through its ClusterIP Drop the port-forward restart removed by the ClusterIP change, resolve the Service DNS name covered by the server certificate to the ClusterIP, bound each authenticated API wait attempt, and collect diagnostics from every gateway Pod. Signed-off-by: Matthew Grossman --- tests/ansible/playbooks/openshell-k3s-ha-tls.yaml | 15 +++++++++------ .../openshell_diagnostics/files/collect-k3s.sh | 9 +++++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml index 51df48660c..52465a3e2a 100644 --- a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml +++ b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml @@ -58,10 +58,13 @@ environment: KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - name: Restart gateway forwarder after TLS upgrade - ansible.builtin.systemd_service: - name: openshell-k3s-port-forward.service - state: restarted + # The chart's server certificate covers the Service DNS name but not its + # ClusterIP, and the node resolver cannot reach cluster DNS. + - name: Resolve the gateway Service name to its ClusterIP + ansible.builtin.lineinfile: + path: /etc/hosts + regexp: '\sopenshell\.openshell\.svc\.cluster\.local$' + line: "{{ k3s_gateway_address.stdout.split(':')[0] }} openshell.openshell.svc.cluster.local" - name: Read chart client TLS Secret ansible.builtin.command: @@ -99,12 +102,12 @@ apply: become: false vars: - openshell_client_gateway_endpoint: https://localhost:17670 + openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}" - name: Wait for authenticated gateway API become: false ansible.builtin.command: - argv: [/usr/local/bin/openshell, sandbox, list, --output, json] + argv: [timeout, 10s, /usr/local/bin/openshell, sandbox, list, --output, json] register: k3s_gateway_api until: k3s_gateway_api.rc == 0 retries: 24 diff --git a/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh index c20334bb13..4ec06f130b 100644 --- a/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh +++ b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh @@ -43,8 +43,13 @@ collect() { collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \ -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions' collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp - collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --tail=300 --timestamps - collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --previous --tail=300 --timestamps + # Gateway Pods are named by a StatefulSet or a Deployment, depending on the installer. + mapfile -t gateway_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ + --selector app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell --output name 2>/dev/null) + for pod in "${gateway_pods[@]}"; do + collect "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --tail=300 --timestamps + collect "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous --tail=300 --timestamps + done collect 'Memory' free -m collect 'Disk' df -h / /var/lib/rancher/k3s } | sed -E \ From 387026536c126df72764b82551435b96867aa249 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 9 Oct 2026 14:42:11 -0700 Subject: [PATCH 3/6] ci(tmachine): run K3s PostgreSQL and mTLS conformance Add the k3s-ha-tls installer to every core conformance matrix so Branch E2E, Release Dev, Release Tag, and manual Integration Tests cover three gateway replicas sharing PostgreSQL behind mTLS. Signed-off-by: Matthew Grossman --- .github/workflows/branch-e2e.yml | 1 + .github/workflows/integration-runner.yml | 1 + .github/workflows/integration-test.yml | 1 + .github/workflows/release-dev.yml | 1 + .github/workflows/release-tag.yml | 1 + CI.md | 6 ++++-- 6 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index d46f3fb31f..1e392e6e95 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -256,6 +256,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index 4be954eced..feee44f0e1 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -26,6 +26,7 @@ on: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index fe2428a0df..9265447987 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -29,6 +29,7 @@ on: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 0da5d1e186..e1f95f627f 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -127,6 +127,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 7a225270a6..bf6bf3d53e 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -178,6 +178,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/CI.md b/CI.md index de6c53511b..9673d3423a 100644 --- a/CI.md +++ b/CI.md @@ -149,8 +149,10 @@ compatibility baseline for the v1beta1 Sandbox API. It does not track the local K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version. -The `ubuntu-k3s` lane registers the gateway's ClusterIP Service directly, so -recreating the Service requires reprovisioning the fixture. Before conformance, +The `ubuntu-k3s` lanes register the gateway's ClusterIP Service directly, so +recreating the Service requires reprovisioning the fixture. The `k3s-ha-tls` +lane runs the same conformance suite against three gateway replicas sharing +PostgreSQL, with the CLI connecting over mTLS. Before conformance, every installer waits up to five minutes for a connected gateway; interactive shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*` artifacts. From 861d41e9c0c4448edde5cf13f290f4f73976edc9 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 12 Oct 2026 08:49:47 -0700 Subject: [PATCH 4/6] refactor(tmachine): rename k3s-ha-tls installer to k3s-ha TLS is part of the supported HA configuration once gateway replicas share PostgreSQL, so the installer name only needs to distinguish HA from the single-replica k3s lane. Signed-off-by: Matthew Grossman --- .github/workflows/branch-e2e.yml | 2 +- .github/workflows/integration-runner.yml | 2 +- .github/workflows/integration-test.yml | 2 +- .github/workflows/release-dev.yml | 2 +- .github/workflows/release-tag.yml | 2 +- CI.md | 7 ++++--- .../{openshell-k3s-ha-tls.yaml => openshell-k3s-ha.yaml} | 4 ++-- tests/config.nix | 4 ++-- 8 files changed, 13 insertions(+), 12 deletions(-) rename tests/ansible/playbooks/{openshell-k3s-ha-tls.yaml => openshell-k3s-ha.yaml} (97%) diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 1e392e6e95..503cde6458 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -256,7 +256,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index feee44f0e1..98d8feb301 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -26,7 +26,7 @@ on: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 9265447987..9c46b27410 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -29,7 +29,7 @@ on: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index e1f95f627f..bede6f6c2f 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -127,7 +127,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index bf6bf3d53e..ac7eeea1df 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -178,7 +178,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/CI.md b/CI.md index 9673d3423a..7ab6eb6300 100644 --- a/CI.md +++ b/CI.md @@ -150,9 +150,10 @@ K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 throug its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version. The `ubuntu-k3s` lanes register the gateway's ClusterIP Service directly, so -recreating the Service requires reprovisioning the fixture. The `k3s-ha-tls` -lane runs the same conformance suite against three gateway replicas sharing -PostgreSQL, with the CLI connecting over mTLS. Before conformance, +recreating the Service requires reprovisioning the fixture. The `k3s` lane runs +one plaintext gateway. The `k3s-ha` lane runs the supported HA configuration: +three gateway replicas sharing PostgreSQL, serving TLS, with the CLI connecting +over mTLS. Before conformance, every installer waits up to five minutes for a connected gateway; interactive shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*` artifacts. diff --git a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml b/tests/ansible/playbooks/openshell-k3s-ha.yaml similarity index 97% rename from tests/ansible/playbooks/openshell-k3s-ha-tls.yaml rename to tests/ansible/playbooks/openshell-k3s-ha.yaml index 52465a3e2a..ff211ea831 100644 --- a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml +++ b/tests/ansible/playbooks/openshell-k3s-ha.yaml @@ -26,7 +26,7 @@ - name: Write PostgreSQL and TLS values overlay ansible.builtin.copy: - dest: /var/lib/openshell/artifacts/values-ha-tls.yaml + dest: /var/lib/openshell/artifacts/values-ha.yaml mode: "0600" content: | replicaCount: 3 @@ -52,7 +52,7 @@ - --values - /var/lib/openshell/artifacts/values.yaml - --values - - /var/lib/openshell/artifacts/values-ha-tls.yaml + - /var/lib/openshell/artifacts/values-ha.yaml - --wait - --timeout=5m environment: diff --git a/tests/config.nix b/tests/config.nix index f5e414fb80..e9e7e37479 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -95,9 +95,9 @@ let }; } { - name = "k3s-ha-tls"; + name = "k3s-ha"; use_galaxy = false; - playbooks = [ "ansible/playbooks/openshell-k3s-ha-tls.yaml" ]; + playbooks = [ "ansible/playbooks/openshell-k3s-ha.yaml" ]; inputs = { openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml"; agent_sandbox_version = "0.5.0"; From fe6c9545e91489d8ed1114da0875252ac8e149ec Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 12 Oct 2026 10:53:04 -0700 Subject: [PATCH 5/6] refactor(tmachine): install K3s HA in a single Helm rollout Add an optional Helm values overlay to the K3s installer so the k3s-ha installer stages PostgreSQL first and installs the three-replica TLS gateway directly, instead of installing a plaintext gateway and upgrading. Drop the redundant unauthenticated-users override and the install-time API wait, which the conformance playbook already performs. Share K3s installer inputs in the tmachine configuration. Signed-off-by: Matthew Grossman --- tests/ansible/playbooks/openshell-k3s-ha.yaml | 84 +++++++------------ tests/ansible/playbooks/openshell-k3s.yaml | 8 ++ tests/config.nix | 25 +++--- 3 files changed, 46 insertions(+), 71 deletions(-) diff --git a/tests/ansible/playbooks/openshell-k3s-ha.yaml b/tests/ansible/playbooks/openshell-k3s-ha.yaml index ff211ea831..fe9ceacd45 100644 --- a/tests/ansible/playbooks/openshell-k3s-ha.yaml +++ b/tests/ansible/playbooks/openshell-k3s-ha.yaml @@ -2,62 +2,43 @@ # SPDX-License-Identifier: Apache-2.0 --- -- import_playbook: openshell-k3s.yaml - -- name: Add PostgreSQL and TLS to the K3s test installation +- name: Stage PostgreSQL for the HA gateway hosts: all become: true gather_facts: false tasks: - - name: Copy PostgreSQL fixture - ansible.builtin.copy: - src: "{{ openshell_postgres_fixture }}" - dest: /var/lib/openshell/artifacts/postgres.yaml - mode: "0600" + - name: Wait for SSH + ansible.builtin.wait_for_connection: - - name: Apply PostgreSQL fixture + # The install layer boots fresh, so the K3s API may still be starting. + - name: Create OpenShell namespace ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, /var/lib/openshell/artifacts/postgres.yaml] + argv: [/usr/local/bin/k3s, kubectl, create, namespace, openshell] + register: k3s_namespace + until: k3s_namespace.rc == 0 + retries: 30 + delay: 2 - - name: Wait for PostgreSQL + - name: Apply PostgreSQL fixture ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s] - changed_when: false - - - name: Write PostgreSQL and TLS values overlay - ansible.builtin.copy: - dest: /var/lib/openshell/artifacts/values-ha.yaml - mode: "0600" - content: | - replicaCount: 3 - workload: - kind: deployment - server: - externalDbSecret: openshell-e2e-postgres-credentials - disableTls: false - auth: - # Kubernetes has no mTLS user-identity mode. Require the client - # certificate at the TLS layer and use the fixture's dev user. - allowUnauthenticatedUsers: true + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, "-"] + stdin: "{{ lookup('ansible.builtin.file', openshell_postgres_fixture) }}" - - name: Apply PostgreSQL and TLS overlay to OpenShell - ansible.builtin.command: - argv: - - /usr/local/bin/helm - - upgrade - - openshell - - /var/lib/openshell/artifacts/helm-chart.tgz - - --namespace - - openshell - - --values - - /var/lib/openshell/artifacts/values.yaml - - --values - - /var/lib/openshell/artifacts/values-ha.yaml - - --wait - - --timeout=5m - environment: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml +- import_playbook: openshell-k3s.yaml + vars: + openshell_k3s_helm_values_overlay: | + replicaCount: 3 + workload: + kind: deployment + server: + externalDbSecret: openshell-e2e-postgres-credentials + disableTls: false +- name: Register the HA gateway over mTLS + hosts: all + become: true + gather_facts: false + tasks: # The chart's server certificate covers the Service DNS name but not its # ClusterIP, and the node resolver cannot reach cluster DNS. - name: Resolve the gateway Service name to its ClusterIP @@ -91,6 +72,7 @@ loop: [ca.crt, tls.crt, tls.key] no_log: true + # The base playbook registered a plaintext endpoint; replace it. - name: Remove baseline guest gateway registration become: false ansible.builtin.command: @@ -103,13 +85,3 @@ become: false vars: openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}" - - - name: Wait for authenticated gateway API - become: false - ansible.builtin.command: - argv: [timeout, 10s, /usr/local/bin/openshell, sandbox, list, --output, json] - register: k3s_gateway_api - until: k3s_gateway_api.rc == 0 - retries: 24 - delay: 5 - changed_when: false diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index 591262a933..e39a9f3523 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -87,6 +87,12 @@ disableTls: true telemetryEnabled: false + - name: Write OpenShell Helm values overlay + ansible.builtin.copy: + dest: /var/lib/openshell/artifacts/values-overlay.yaml + mode: "0600" + content: "{{ openshell_k3s_helm_values_overlay | default('{}') }}" + - name: Install Agent Sandbox ansible.builtin.command: argv: @@ -132,6 +138,8 @@ - --create-namespace - --values - /var/lib/openshell/artifacts/values.yaml + - --values + - /var/lib/openshell/artifacts/values-overlay.yaml - --wait - --timeout=5m environment: diff --git a/tests/config.nix b/tests/config.nix index e9e7e37479..0ede4802fa 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -19,6 +19,14 @@ let OVMF = firmwarePkgs.OVMF; }; qemu = qemuPkgs.qemu.override { hostCpuOnly = true; }; + k3sInputs = { + agent_sandbox_version = "0.5.0"; + openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; + openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar"; + openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz"; + openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar"; + openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar"; + }; config = (pkgs.formats.yaml { }).generate "tmachine-config.yaml" { machines = [ { @@ -85,27 +93,14 @@ let name = "k3s"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; - inputs = { - agent_sandbox_version = "0.5.0"; - openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; - openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar"; - openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz"; - openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar"; - openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar"; - }; + inputs = k3sInputs; } { name = "k3s-ha"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s-ha.yaml" ]; - inputs = { + inputs = k3sInputs // { openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml"; - agent_sandbox_version = "0.5.0"; - openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; - openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar"; - openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz"; - openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar"; - openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar"; }; } { From 8a8d6b14289c2911b1541bbdf3a0f8dde23a51fa Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 12 Oct 2026 13:21:06 -0700 Subject: [PATCH 6/6] fix(tmachine): make K3s HA staging rerunnable and wait for PostgreSQL Apply the OpenShell namespace instead of creating it so the staging play tolerates an existing namespace, and wait for the PostgreSQL rollout so gateway Pods do not start crash-looping against a missing database. Reflow and pluralize the K3s lane notes in CI.md. Signed-off-by: Matthew Grossman --- CI.md | 11 ++++----- tests/ansible/playbooks/openshell-k3s-ha.yaml | 23 +++++++++++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/CI.md b/CI.md index 7ab6eb6300..e8e66af5af 100644 --- a/CI.md +++ b/CI.md @@ -144,8 +144,8 @@ push-triggered workflow jobs themselves. ### K3s conformance version baseline -The tmachine `ubuntu-k3s` conformance lane pins Agent Sandbox v0.5.0 as the -compatibility baseline for the v1beta1 Sandbox API. It does not track the local +The tmachine `ubuntu-k3s` conformance lanes pin Agent Sandbox v0.5.0 as the +compatibility baseline for the v1beta1 Sandbox API. They do not track the local K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version. @@ -153,10 +153,9 @@ The `ubuntu-k3s` lanes register the gateway's ClusterIP Service directly, so recreating the Service requires reprovisioning the fixture. The `k3s` lane runs one plaintext gateway. The `k3s-ha` lane runs the supported HA configuration: three gateway replicas sharing PostgreSQL, serving TLS, with the CLI connecting -over mTLS. Before conformance, -every installer waits up to five minutes for a connected gateway; interactive -shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*` -artifacts. +over mTLS. Before conformance, every installer waits up to five minutes for a +connected gateway; interactive shells skip the wait. K3s failures upload +diagnostics as `tmachine-diagnostics-*` artifacts. ### Run only the policy advisor conformance tests diff --git a/tests/ansible/playbooks/openshell-k3s-ha.yaml b/tests/ansible/playbooks/openshell-k3s-ha.yaml index fe9ceacd45..efcc736d49 100644 --- a/tests/ansible/playbooks/openshell-k3s-ha.yaml +++ b/tests/ansible/playbooks/openshell-k3s-ha.yaml @@ -11,9 +11,14 @@ ansible.builtin.wait_for_connection: # The install layer boots fresh, so the K3s API may still be starting. - - name: Create OpenShell namespace + - name: Apply OpenShell namespace ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, create, namespace, openshell] + argv: [/usr/local/bin/k3s, kubectl, apply, --filename, "-"] + stdin: | + apiVersion: v1 + kind: Namespace + metadata: + name: openshell register: k3s_namespace until: k3s_namespace.rc == 0 retries: 30 @@ -24,6 +29,20 @@ argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, "-"] stdin: "{{ lookup('ansible.builtin.file', openshell_postgres_fixture) }}" + # Gateway Pods that start before the database crash-loop with growing backoff. + - name: Wait for PostgreSQL + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --namespace + - openshell + - rollout + - status + - deployment/openshell-e2e-postgres + - --timeout=300s + changed_when: false + - import_playbook: openshell-k3s.yaml vars: openshell_k3s_helm_values_overlay: |