diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index d829837ba0..39868baff1 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -125,6 +125,8 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId ## View Job Logs +For tmachine Kubernetes conformance failures, inspect the `tmachine-diagnostics-*` artifact. The K3s report includes Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The conformance playbook checks reconciliation after guest boot with a temporary scheduling-gated Sandbox probe before running tests. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works. + `Pre-release` temporarily omits Codex Security from its `Security Scan` job. `Release` uses the passing prerelease qualification result. Other scanners and qualification gates still run. Use the standalone diff --git a/tests/ansible/playbooks/conformance/cli.yaml b/tests/ansible/playbooks/conformance/cli.yaml index 11eb07e36d..e841a11442 100644 --- a/tests/ansible/playbooks/conformance/cli.yaml +++ b/tests/ansible/playbooks/conformance/cli.yaml @@ -6,8 +6,10 @@ hosts: all gather_facts: false vars: - # Conformance is skipped when the gateway never connects. - conformance_failed: "{{ gateway_status is failed or conformance_result.rc != 0 }}" + conformance_failed: >- + {{ gateway_status is failed or + (controller_readiness_failed | default(false)) or + conformance_result.rc != 0 }} tasks: - name: Wait for SSH ansible.builtin.wait_for_connection: @@ -72,6 +74,21 @@ delay: 5 ignore_errors: true + - name: Check for K3s + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: conformance_k3s + + - name: Prepare the Kubernetes controller + when: gateway_status is succeeded and conformance_k3s.stat.exists + block: + - name: Verify Agent Sandbox reconciliation + ansible.builtin.include_tasks: k3s-ready.yaml + rescue: + - name: Record controller readiness failure + ansible.builtin.set_fact: + controller_readiness_failed: true + - name: Run OpenShell conformance archive ansible.builtin.command: argv: @@ -94,7 +111,7 @@ register: conformance_result changed_when: false failed_when: false - when: gateway_status is succeeded + when: gateway_status is succeeded and not (controller_readiness_failed | default(false)) # Preserve both streams for failures without adding passing-test output to # every tmachine run. @@ -114,6 +131,12 @@ - gateway_status is succeeded fail_msg: OpenShell gateway did not connect before conformance; see gateway diagnostics + - name: Require a ready Kubernetes controller + ansible.builtin.assert: + that: + - not (controller_readiness_failed | default(false)) + fail_msg: Agent Sandbox reconciliation did not become ready before conformance; see controller diagnostics + - name: Require OpenShell conformance success ansible.builtin.assert: that: diff --git a/tests/ansible/playbooks/conformance/k3s-ready.yaml b/tests/ansible/playbooks/conformance/k3s-ready.yaml new file mode 100644 index 0000000000..c958d45026 --- /dev/null +++ b/tests/ansible/playbooks/conformance/k3s-ready.yaml @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Check Agent Sandbox reconciliation after boot + become: true + block: + - name: Choose a unique controller probe name + ansible.builtin.set_fact: + controller_probe_name: "conformance-startup-{{ 9999999999999999 | random }}" + + - name: Apply a scheduling-gated controller probe + ansible.builtin.command: + argv: + - timeout + - 15s + - /usr/local/bin/k3s + - kubectl + - --request-timeout=10s + - apply + - --filename=- + - --output=json + stdin: | + apiVersion: agents.x-k8s.io/v1beta1 + kind: Sandbox + metadata: + name: "{{ controller_probe_name }}" + namespace: openshell + spec: + podTemplate: + spec: + automountServiceAccountToken: false + schedulingGates: + - name: openshell.ai/conformance-startup + containers: + - name: probe + image: registry.k8s.io/pause:3.9 + register: controller_probe + until: controller_probe.rc == 0 + retries: 60 + delay: 5 + + # Deployment readiness does not establish leadership or reconciliation. + # The gate keeps this probe from running or pulling its image. + - name: Wait for the controller-owned gated Pod + ansible.builtin.command: + argv: + - timeout + - 10s + - /usr/local/bin/k3s + - kubectl + - --request-timeout=5s + - --namespace=openshell + - get + - pod + - "{{ controller_probe_name }}" + - --output=json + register: controller_probe_pod + changed_when: false + until: >- + controller_probe_pod.rc == 0 and + ((controller_probe_pod.stdout | from_json).metadata.ownerReferences | default([]) | + selectattr('controller', 'defined') | selectattr('controller', 'equalto', true) | + selectattr('kind', 'equalto', 'Sandbox') | + selectattr('uid', 'equalto', (controller_probe.stdout | from_json).metadata.uid) | list | length) == 1 and + ((controller_probe_pod.stdout | from_json).spec.schedulingGates | default([]) | + selectattr('name', 'equalto', 'openshell.ai/conformance-startup') | list | length) == 1 and + not ((controller_probe_pod.stdout | from_json).spec.nodeName | default('')) + retries: 60 + delay: 5 + + always: + - name: Remove the controller probe and its dependent resources + ansible.builtin.command: + argv: + - timeout + - 30s + - /usr/local/bin/k3s + - kubectl + - --request-timeout=5s + - --namespace=openshell + - delete + - sandboxes.agents.x-k8s.io + - "{{ controller_probe_name }}" + - --ignore-not-found=true + - --cascade=foreground + - --wait=true + - --timeout=20s + when: controller_probe_name is defined diff --git a/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh index 4ec06f130b..63e3193b21 100644 --- a/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh +++ b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh @@ -16,6 +16,18 @@ collect() { printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" } +collect_log() { + printf '\n### %s\n' "$1" + shift + # Retain startup and recent evidence when later scenarios fill the log. + timeout 10s "$@" 2>&1 | { + head -c 131072 + printf '\n### Remaining log tail\n' + tail -c 131072 + } + printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" +} + { date --utc --iso-8601=seconds collect 'K3s service state' systemctl show k3s.service \ @@ -34,6 +46,21 @@ collect() { collect 'K3s version' /usr/local/bin/k3s --version collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide + collect 'Agent Sandbox controller deployment' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get deployment agent-sandbox-controller \ + -o 'custom-columns=NAME:.metadata.name,GENERATION:.metadata.generation,STATUS:.status' + collect 'Agent Sandbox controller Pods' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \ + -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses' + collect 'Agent Sandbox leader leases' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get leases \ + -o 'custom-columns=NAME:.metadata.name,HOLDER:.spec.holderIdentity,ACQUIRED:.spec.acquireTime,RENEWED:.spec.renewTime,DURATION:.spec.leaseDurationSeconds,TRANSITIONS:.spec.leaseTransitions' + collect 'Agent Sandbox controller events' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get events --sort-by=.lastTimestamp + collect 'Sandbox reconciliation status' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get sandboxes.agents.x-k8s.io \ + -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,CREATED:.metadata.creationTimestamp,GENERATION:.metadata.generation,STATUS:.status' + mapfile -t controller_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \ + --selector app=agent-sandbox-controller --output name 2>/dev/null) + for pod in "${controller_pods[@]}"; do + collect_log "Agent Sandbox current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --since=30m --timestamps + collect_log "Agent Sandbox previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --previous --since=30m --timestamps + done collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses' collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \