diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 43ad23a628..6874f6aa3c 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -256,6 +256,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index 4be954eced..98d8feb301 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -26,6 +26,7 @@ on: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index fe2428a0df..9c46b27410 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -29,6 +29,7 @@ on: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 4e129cf200..1305dff193 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -127,6 +127,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 817b34a0c4..63e009da2b 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -178,6 +178,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] diff --git a/CI.md b/CI.md index de6c53511b..e8e66af5af 100644 --- a/CI.md +++ b/CI.md @@ -144,16 +144,18 @@ push-triggered workflow jobs themselves. ### K3s conformance version baseline -The tmachine `ubuntu-k3s` conformance lane pins Agent Sandbox v0.5.0 as the -compatibility baseline for the v1beta1 Sandbox API. It does not track the local +The tmachine `ubuntu-k3s` conformance lanes pin Agent Sandbox v0.5.0 as the +compatibility baseline for the v1beta1 Sandbox API. They do not track the local K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version. -The `ubuntu-k3s` lane registers the gateway's ClusterIP Service directly, so -recreating the Service requires reprovisioning the fixture. Before conformance, -every installer waits up to five minutes for a connected gateway; interactive -shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*` -artifacts. +The `ubuntu-k3s` lanes register the gateway's ClusterIP Service directly, so +recreating the Service requires reprovisioning the fixture. The `k3s` lane runs +one plaintext gateway. The `k3s-ha` lane runs the supported HA configuration: +three gateway replicas sharing PostgreSQL, serving TLS, with the CLI connecting +over mTLS. Before conformance, every installer waits up to five minutes for a +connected gateway; interactive shells skip the wait. K3s failures upload +diagnostics as `tmachine-diagnostics-*` artifacts. ### Run only the policy advisor conformance tests diff --git a/tests/ansible/playbooks/openshell-k3s-ha.yaml b/tests/ansible/playbooks/openshell-k3s-ha.yaml new file mode 100644 index 0000000000..efcc736d49 --- /dev/null +++ b/tests/ansible/playbooks/openshell-k3s-ha.yaml @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Stage PostgreSQL for the HA gateway + hosts: all + become: true + gather_facts: false + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + # The install layer boots fresh, so the K3s API may still be starting. + - name: Apply OpenShell namespace + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, apply, --filename, "-"] + stdin: | + apiVersion: v1 + kind: Namespace + metadata: + name: openshell + register: k3s_namespace + until: k3s_namespace.rc == 0 + retries: 30 + delay: 2 + + - name: Apply PostgreSQL fixture + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, "-"] + stdin: "{{ lookup('ansible.builtin.file', openshell_postgres_fixture) }}" + + # Gateway Pods that start before the database crash-loop with growing backoff. + - name: Wait for PostgreSQL + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --namespace + - openshell + - rollout + - status + - deployment/openshell-e2e-postgres + - --timeout=300s + changed_when: false + +- import_playbook: openshell-k3s.yaml + vars: + openshell_k3s_helm_values_overlay: | + replicaCount: 3 + workload: + kind: deployment + server: + externalDbSecret: openshell-e2e-postgres-credentials + disableTls: false + +- name: Register the HA gateway over mTLS + hosts: all + become: true + gather_facts: false + tasks: + # The chart's server certificate covers the Service DNS name but not its + # ClusterIP, and the node resolver cannot reach cluster DNS. + - name: Resolve the gateway Service name to its ClusterIP + ansible.builtin.lineinfile: + path: /etc/hosts + regexp: '\sopenshell\.openshell\.svc\.cluster\.local$' + line: "{{ k3s_gateway_address.stdout.split(':')[0] }} openshell.openshell.svc.cluster.local" + + - name: Read chart client TLS Secret + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, get, secret, openshell-client-tls, --output, json] + register: k3s_client_tls + changed_when: false + no_log: true + + - name: Create guest mTLS directory + ansible.builtin.file: + path: /home/tmachine/.config/openshell/gateways/tmachine/mtls + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Write guest client TLS bundle + ansible.builtin.copy: + content: "{{ (k3s_client_tls.stdout | from_json).data[item] | b64decode }}" + dest: "/home/tmachine/.config/openshell/gateways/tmachine/mtls/{{ item }}" + owner: tmachine + group: tmachine + mode: "0600" + loop: [ca.crt, tls.crt, tls.key] + no_log: true + + # The base playbook registered a plaintext endpoint; replace it. + - name: Remove baseline guest gateway registration + become: false + ansible.builtin.command: + argv: [/usr/local/bin/openshell, gateway, remove, tmachine] + + - name: Register guest gateway + ansible.builtin.include_role: + name: openshell_client + apply: + become: false + vars: + openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}" diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index 591262a933..e39a9f3523 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -87,6 +87,12 @@ disableTls: true telemetryEnabled: false + - name: Write OpenShell Helm values overlay + ansible.builtin.copy: + dest: /var/lib/openshell/artifacts/values-overlay.yaml + mode: "0600" + content: "{{ openshell_k3s_helm_values_overlay | default('{}') }}" + - name: Install Agent Sandbox ansible.builtin.command: argv: @@ -132,6 +138,8 @@ - --create-namespace - --values - /var/lib/openshell/artifacts/values.yaml + - --values + - /var/lib/openshell/artifacts/values-overlay.yaml - --wait - --timeout=5m environment: diff --git a/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh index c20334bb13..4ec06f130b 100644 --- a/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh +++ b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh @@ -43,8 +43,13 @@ collect() { collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \ -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions' collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp - collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --tail=300 --timestamps - collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --previous --tail=300 --timestamps + # Gateway Pods are named by a StatefulSet or a Deployment, depending on the installer. + mapfile -t gateway_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ + --selector app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell --output name 2>/dev/null) + for pod in "${gateway_pods[@]}"; do + collect "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --tail=300 --timestamps + collect "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous --tail=300 --timestamps + done collect 'Memory' free -m collect 'Disk' df -h / /var/lib/rancher/k3s } | sed -E \ diff --git a/tests/config.nix b/tests/config.nix index 26d27b712b..0ede4802fa 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -19,6 +19,14 @@ let OVMF = firmwarePkgs.OVMF; }; qemu = qemuPkgs.qemu.override { hostCpuOnly = true; }; + k3sInputs = { + agent_sandbox_version = "0.5.0"; + openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; + openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar"; + openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz"; + openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar"; + openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar"; + }; config = (pkgs.formats.yaml { }).generate "tmachine-config.yaml" { machines = [ { @@ -85,13 +93,14 @@ let name = "k3s"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; - inputs = { - agent_sandbox_version = "0.5.0"; - openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; - openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar"; - openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz"; - openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar"; - openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar"; + inputs = k3sInputs; + } + { + name = "k3s-ha"; + use_galaxy = false; + playbooks = [ "ansible/playbooks/openshell-k3s-ha.yaml" ]; + inputs = k3sInputs // { + openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml"; }; } {