Skip to content

Commit 7417f75

Browse files
pimlockpimlock
authored
fix(ci): wait for controller reconciliation before Kubernetes conformance (#4676)
Signed-off-by: Piotr Mlocek <pmlocek@nvidia.com>
1 parent d4ad1bb commit 7417f75

4 files changed

Lines changed: 144 additions & 3 deletions

File tree

‎.agents/skills/watch-github-actions/SKILL.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,8 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId
125125

126126
## View Job Logs
127127

128+
For tmachine Kubernetes conformance failures, inspect the `tmachine-diagnostics-*` artifact. The K3s report includes Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The conformance playbook checks reconciliation after guest boot with a temporary scheduling-gated Sandbox probe before running tests. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works.
129+
128130
`Pre-release` temporarily omits Codex Security from its `Security Scan` job.
129131
`Release` uses the passing prerelease qualification result. Other scanners and
130132
qualification gates still run. Use the standalone

‎tests/ansible/playbooks/conformance/cli.yaml‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@
66
hosts: all
77
gather_facts: false
88
vars:
9-
# Conformance is skipped when the gateway never connects.
10-
conformance_failed: "{{ gateway_status is failed or conformance_result.rc != 0 }}"
9+
conformance_failed: >-
10+
{{ gateway_status is failed or
11+
(controller_readiness_failed | default(false)) or
12+
conformance_result.rc != 0 }}
1113
tasks:
1214
- name: Wait for SSH
1315
ansible.builtin.wait_for_connection:
@@ -72,6 +74,21 @@
7274
delay: 5
7375
ignore_errors: true
7476

77+
- name: Check for K3s
78+
ansible.builtin.stat:
79+
path: /usr/local/bin/k3s
80+
register: conformance_k3s
81+
82+
- name: Prepare the Kubernetes controller
83+
when: gateway_status is succeeded and conformance_k3s.stat.exists
84+
block:
85+
- name: Verify Agent Sandbox reconciliation
86+
ansible.builtin.include_tasks: k3s-ready.yaml
87+
rescue:
88+
- name: Record controller readiness failure
89+
ansible.builtin.set_fact:
90+
controller_readiness_failed: true
91+
7592
- name: Run OpenShell conformance archive
7693
ansible.builtin.command:
7794
argv:
@@ -94,7 +111,7 @@
94111
register: conformance_result
95112
changed_when: false
96113
failed_when: false
97-
when: gateway_status is succeeded
114+
when: gateway_status is succeeded and not (controller_readiness_failed | default(false))
98115

99116
# Preserve both streams for failures without adding passing-test output to
100117
# every tmachine run.
@@ -114,6 +131,12 @@
114131
- gateway_status is succeeded
115132
fail_msg: OpenShell gateway did not connect before conformance; see gateway diagnostics
116133

134+
- name: Require a ready Kubernetes controller
135+
ansible.builtin.assert:
136+
that:
137+
- not (controller_readiness_failed | default(false))
138+
fail_msg: Agent Sandbox reconciliation did not become ready before conformance; see controller diagnostics
139+
117140
- name: Require OpenShell conformance success
118141
ansible.builtin.assert:
119142
that:

‎tests/ansible/playbooks/conformance/k3s-ready.yaml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
---
5+
- name: Check Agent Sandbox reconciliation after boot
6+
become: true
7+
block:
8+
- name: Choose a unique controller probe name
9+
ansible.builtin.set_fact:
10+
controller_probe_name: "conformance-startup-{{ 9999999999999999 | random }}"
11+
12+
- name: Apply a scheduling-gated controller probe
13+
ansible.builtin.command:
14+
argv:
15+
- timeout
16+
- 15s
17+
- /usr/local/bin/k3s
18+
- kubectl
19+
- --request-timeout=10s
20+
- apply
21+
- --filename=-
22+
- --output=json
23+
stdin: |
24+
apiVersion: agents.x-k8s.io/v1beta1
25+
kind: Sandbox
26+
metadata:
27+
name: "{{ controller_probe_name }}"
28+
namespace: openshell
29+
spec:
30+
podTemplate:
31+
spec:
32+
automountServiceAccountToken: false
33+
schedulingGates:
34+
- name: openshell.ai/conformance-startup
35+
containers:
36+
- name: probe
37+
image: registry.k8s.io/pause:3.9
38+
register: controller_probe
39+
until: controller_probe.rc == 0
40+
retries: 60
41+
delay: 5
42+
43+
# Deployment readiness does not establish leadership or reconciliation.
44+
# The gate keeps this probe from running or pulling its image.
45+
- name: Wait for the controller-owned gated Pod
46+
ansible.builtin.command:
47+
argv:
48+
- timeout
49+
- 10s
50+
- /usr/local/bin/k3s
51+
- kubectl
52+
- --request-timeout=5s
53+
- --namespace=openshell
54+
- get
55+
- pod
56+
- "{{ controller_probe_name }}"
57+
- --output=json
58+
register: controller_probe_pod
59+
changed_when: false
60+
until: >-
61+
controller_probe_pod.rc == 0 and
62+
((controller_probe_pod.stdout | from_json).metadata.ownerReferences | default([]) |
63+
selectattr('controller', 'defined') | selectattr('controller', 'equalto', true) |
64+
selectattr('kind', 'equalto', 'Sandbox') |
65+
selectattr('uid', 'equalto', (controller_probe.stdout | from_json).metadata.uid) | list | length) == 1 and
66+
((controller_probe_pod.stdout | from_json).spec.schedulingGates | default([]) |
67+
selectattr('name', 'equalto', 'openshell.ai/conformance-startup') | list | length) == 1 and
68+
not ((controller_probe_pod.stdout | from_json).spec.nodeName | default(''))
69+
retries: 60
70+
delay: 5
71+
72+
always:
73+
- name: Remove the controller probe and its dependent resources
74+
ansible.builtin.command:
75+
argv:
76+
- timeout
77+
- 30s
78+
- /usr/local/bin/k3s
79+
- kubectl
80+
- --request-timeout=5s
81+
- --namespace=openshell
82+
- delete
83+
- sandboxes.agents.x-k8s.io
84+
- "{{ controller_probe_name }}"
85+
- --ignore-not-found=true
86+
- --cascade=foreground
87+
- --wait=true
88+
- --timeout=20s
89+
when: controller_probe_name is defined

‎tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,18 @@ collect() {
1616
printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
1717
}
1818

19+
collect_log() {
20+
printf '\n### %s\n' "$1"
21+
shift
22+
# Retain startup and recent evidence when later scenarios fill the log.
23+
timeout 10s "$@" 2>&1 | {
24+
head -c 131072
25+
printf '\n### Remaining log tail\n'
26+
tail -c 131072
27+
}
28+
printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
29+
}
30+
1931
{
2032
date --utc --iso-8601=seconds
2133
collect 'K3s service state' systemctl show k3s.service \
@@ -34,6 +46,21 @@ collect() {
3446
collect 'K3s version' /usr/local/bin/k3s --version
3547
collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz
3648
collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide
49+
collect 'Agent Sandbox controller deployment' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get deployment agent-sandbox-controller \
50+
-o 'custom-columns=NAME:.metadata.name,GENERATION:.metadata.generation,STATUS:.status'
51+
collect 'Agent Sandbox controller Pods' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \
52+
-o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses'
53+
collect 'Agent Sandbox leader leases' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get leases \
54+
-o 'custom-columns=NAME:.metadata.name,HOLDER:.spec.holderIdentity,ACQUIRED:.spec.acquireTime,RENEWED:.spec.renewTime,DURATION:.spec.leaseDurationSeconds,TRANSITIONS:.spec.leaseTransitions'
55+
collect 'Agent Sandbox controller events' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get events --sort-by=.lastTimestamp
56+
collect 'Sandbox reconciliation status' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get sandboxes.agents.x-k8s.io \
57+
-o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,CREATED:.metadata.creationTimestamp,GENERATION:.metadata.generation,STATUS:.status'
58+
mapfile -t controller_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \
59+
--selector app=agent-sandbox-controller --output name 2>/dev/null)
60+
for pod in "${controller_pods[@]}"; do
61+
collect_log "Agent Sandbox current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --since=30m --timestamps
62+
collect_log "Agent Sandbox previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --previous --since=30m --timestamps
63+
done
3764
collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
3865
-o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses'
3966
collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \

0 commit comments

Comments
 (0)