fix(ci): wait for controller reconciliation before Kubernetes conformance

.agents/skills/watch-github-actions/SKILL.md

@@ -125,6 +125,8 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId
 
 ## View Job Logs
 
+For tmachine Kubernetes conformance failures, inspect the `tmachine-diagnostics-*` artifact. The K3s report includes Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The conformance playbook checks reconciliation after guest boot with a temporary scheduling-gated Sandbox probe before running tests. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works.
+
 `Release Tag` temporarily omits Codex Security from its `Security Scan` job.
 Other scanners and qualification gates still run. Use the standalone
 `Codex Security Release Qualification` workflow for manual investigation.

TESTING.md

@@ -297,6 +297,8 @@ The gateway must already be installed, reachable, and selected before the tests
 start. The test suite does not provision a gateway or select a compute driver.
 Set `OPENSHELL_BIN` to another executable to test a different prebuilt CLI.
 
+The installed-artifact conformance playbook waits for gateway connectivity after each guest boot. On K3s, it also creates a temporary Agent Sandbox resource and waits for its controller-owned Pod. A scheduling gate prevents the probe Pod from running or pulling an image. The playbook removes the probe and its dependent resources before running conformance. This checks reconciliation because the upstream controller Deployment can become available before its controller starts processing resources. Failure diagnostics include controller state, leader leases, events, and current and previous controller logs from the last 30 minutes, with each command and output bounded. Controller logs retain both the beginning and end of that window so later scenarios do not discard startup evidence.
+
 The phase-1 scenario verifies the complete CLI-to-gateway-to-driver path without
 depending on how the gateway was installed or which driver is configured. It
 requires machine-readable gRPC status, creates a uniquely named detached

tests/ansible/playbooks/conformance/cli.yaml

@@ -6,8 +6,10 @@
   hosts: all
   gather_facts: false
   vars:
-    # Conformance is skipped when the gateway never connects.
-    conformance_failed: "{{ gateway_status is failed or conformance_result.rc != 0 }}"
+    conformance_failed: >-
+      {{ gateway_status is failed or
+         (controller_readiness_failed | default(false)) or
+         conformance_result.rc != 0 }}
   tasks:
     - name: Wait for SSH
       ansible.builtin.wait_for_connection:
@@ -72,6 +74,21 @@
       delay: 5
       ignore_errors: true
 
+    - name: Check for K3s
+      ansible.builtin.stat:
+        path: /usr/local/bin/k3s
+      register: conformance_k3s
+
+    - name: Prepare the Kubernetes controller
+      when: gateway_status is succeeded and conformance_k3s.stat.exists
+      block:
+        - name: Verify Agent Sandbox reconciliation
+          ansible.builtin.include_tasks: k3s-ready.yaml
+      rescue:
+        - name: Record controller readiness failure
+          ansible.builtin.set_fact:
+            controller_readiness_failed: true
+
     - name: Run OpenShell conformance archive
       ansible.builtin.command:
         argv:
@@ -94,7 +111,7 @@
       register: conformance_result
       changed_when: false
       failed_when: false
-      when: gateway_status is succeeded
+      when: gateway_status is succeeded and not (controller_readiness_failed | default(false))
 
     # Preserve both streams for failures without adding passing-test output to
     # every tmachine run.
@@ -114,6 +131,12 @@
           - gateway_status is succeeded
         fail_msg: OpenShell gateway did not connect before conformance; see gateway diagnostics
 
+    - name: Require a ready Kubernetes controller
+      ansible.builtin.assert:
+        that:
+          - not (controller_readiness_failed | default(false))
+        fail_msg: Agent Sandbox reconciliation did not become ready before conformance; see controller diagnostics
+
     - name: Require OpenShell conformance success
       ansible.builtin.assert:
         that:

tests/ansible/playbooks/conformance/k3s-ready.yaml

@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Check Agent Sandbox reconciliation after boot
+  become: true
+  block:
+    - name: Choose a unique controller probe name
+      ansible.builtin.set_fact:
+        controller_probe_name: "conformance-startup-{{ 9999999999999999 | random }}"
+
+    - name: Apply a scheduling-gated controller probe
+      ansible.builtin.command:
+        argv:
+          - timeout
+          - 15s
+          - /usr/local/bin/k3s
+          - kubectl
+          - --request-timeout=10s
+          - apply
+          - --filename=-
+          - --output=json
+        stdin: |
+          apiVersion: agents.x-k8s.io/v1beta1
+          kind: Sandbox
+          metadata:
+            name: "{{ controller_probe_name }}"
+            namespace: openshell
+          spec:
+            podTemplate:
+              spec:
+                automountServiceAccountToken: false
+                schedulingGates:
+                  - name: openshell.ai/conformance-startup
+                containers:
+                  - name: probe
+                    image: registry.k8s.io/pause:3.9
+      register: controller_probe
+      until: controller_probe.rc == 0
+      retries: 60
+      delay: 5
+
+    # Deployment readiness does not establish leadership or reconciliation.
+    # The gate keeps this probe from running or pulling its image.
+    - name: Wait for the controller-owned gated Pod
+      ansible.builtin.command:
+        argv:
+          - timeout
+          - 10s
+          - /usr/local/bin/k3s
+          - kubectl
+          - --request-timeout=5s
+          - --namespace=openshell
+          - get
+          - pod
+          - "{{ controller_probe_name }}"
+          - --output=json
+      register: controller_probe_pod
+      changed_when: false
+      until: >-
+        controller_probe_pod.rc == 0 and
+        ((controller_probe_pod.stdout | from_json).metadata.ownerReferences | default([]) |
+         selectattr('controller', 'defined') | selectattr('controller', 'equalto', true) |
+         selectattr('kind', 'equalto', 'Sandbox') |
+         selectattr('uid', 'equalto', (controller_probe.stdout | from_json).metadata.uid) | list | length) == 1 and
+        ((controller_probe_pod.stdout | from_json).spec.schedulingGates | default([]) |
+         selectattr('name', 'equalto', 'openshell.ai/conformance-startup') | list | length) == 1 and
+        not ((controller_probe_pod.stdout | from_json).spec.nodeName | default(''))
+      retries: 60
+      delay: 5
+
+  always:
+    - name: Remove the controller probe and its dependent resources
+      ansible.builtin.command:
+        argv:
+          - timeout
+          - 30s
+          - /usr/local/bin/k3s
+          - kubectl
+          - --request-timeout=5s
+          - --namespace=openshell
+          - delete
+          - sandboxes.agents.x-k8s.io
+          - "{{ controller_probe_name }}"
+          - --ignore-not-found=true
+          - --cascade=foreground
+          - --wait=true
+          - --timeout=20s
+      when: controller_probe_name is defined

tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh

@@ -16,6 +16,18 @@ collect() {
   printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
 }
 
+collect_log() {
+  printf '\n### %s\n' "$1"
+  shift
+  # Retain startup and recent evidence when later scenarios fill the log.
+  timeout 10s "$@" 2>&1 | {
+    head -c 131072
+    printf '\n### Remaining log tail\n'
+    tail -c 131072
+  }
+  printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
+}
+
 {
   date --utc --iso-8601=seconds
   collect 'K3s service state' systemctl show k3s.service \
@@ -34,6 +46,21 @@ collect() {
   collect 'K3s version' /usr/local/bin/k3s --version
   collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz
   collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide
+  collect 'Agent Sandbox controller deployment' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get deployment agent-sandbox-controller \
+    -o 'custom-columns=NAME:.metadata.name,GENERATION:.metadata.generation,STATUS:.status'
+  collect 'Agent Sandbox controller Pods' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \
+    -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses'
+  collect 'Agent Sandbox leader leases' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get leases \
+    -o 'custom-columns=NAME:.metadata.name,HOLDER:.spec.holderIdentity,ACQUIRED:.spec.acquireTime,RENEWED:.spec.renewTime,DURATION:.spec.leaseDurationSeconds,TRANSITIONS:.spec.leaseTransitions'
+  collect 'Agent Sandbox controller events' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get events --sort-by=.lastTimestamp
+  collect 'Sandbox reconciliation status' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get sandboxes.agents.x-k8s.io \
+    -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,CREATED:.metadata.creationTimestamp,GENERATION:.metadata.generation,STATUS:.status'
+  mapfile -t controller_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \
+    --selector app=agent-sandbox-controller --output name 2>/dev/null)
+  for pod in "${controller_pods[@]}"; do
+    collect_log "Agent Sandbox current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --since=30m --timestamps
+    collect_log "Agent Sandbox previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --previous --since=30m --timestamps
+  done
   collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
     -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses'
   collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \