diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 9fdf315c3c..e58a3c39da 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -271,6 +271,13 @@ above) instead of relying only on `OPENSHELL_GATEWAY_ENDPOINT`; some tests call ### Kubernetes E2E Notes Use `mise run e2e:kubernetes` for the standard Helm-backed Kubernetes suite. +Set `KATA_CONTAINERS_RUNTIME_CLASS` to the desired RuntimeClass name (for example, +`kata-qemu`) to pass it as `server.defaultRuntimeClassName` to Helm. The wrapper +labels that RuntimeClass `openshell.ai/sandbox-attachable=true` before installation; +the class must exist and the caller must have permission to label it. Cleanup +restores the original label value, or removes the label if it was absent, even +when the test run fails. + The kube e2e wrapper creates only one port-forward, to `svc/openshell`; it no longer forwards the unauthenticated health listener or runs a `/readyz` e2e target. `/readyz` remains covered by server unit/integration tests. diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index ee755b8760..5aff8761f3 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -94,6 +94,8 @@ PORTFORWARD_LOG="${WORKDIR}/portforward.log" PORTFORWARD_HEALTH_PID="" PORTFORWARD_HEALTH_LOG="${WORKDIR}/portforward-health.log" HELM_INSTALLED=0 +KATA_RUNTIME_CLASS_LABEL_RESTORE=0 +KATA_RUNTIME_CLASS_PREVIOUS_LABEL="" EXTERNAL_PG_FIXTURE_DEPLOYED=0 EXTERNAL_PG_FIXTURE_SECRET="" EXTERNAL_PG_FIXTURE_MANIFEST="${ROOT}/e2e/kubernetes/postgres-fixture.yaml" @@ -508,6 +510,16 @@ cleanup() { stop_gateway_portforward + if [ "${KATA_RUNTIME_CLASS_LABEL_RESTORE}" = "1" ]; then + if ! kctl label runtimeclass "${KATA_CONTAINERS_RUNTIME_CLASS}" \ + "${KATA_RUNTIME_CLASS_PREVIOUS_LABEL}" --overwrite; then + echo "ERROR: failed to restore admission label on RuntimeClass ${KATA_CONTAINERS_RUNTIME_CLASS}" >&2 + if [ "${exit_code}" -eq 0 ]; then + exit_code=1 + fi + fi + fi + if [ "${exit_code}" -ne 0 ] && [ -n "${KUBE_CONTEXT}" ] && [ -n "${NAMESPACE}" ]; then if command -v kubectl >/dev/null 2>&1 \ && kctl get namespace "${NAMESPACE}" >/dev/null 2>&1; then @@ -653,6 +665,7 @@ cleanup() { fi rm -rf "${WORKDIR}" 2>/dev/null || true + exit "${exit_code}" } trap cleanup EXIT @@ -1234,6 +1247,18 @@ helm_extra_args=() helm_post_renderer_args=() helm_extra_args+=(--set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}") helm_extra_args+=(--set supervisor.sandboxRuntime.networkPolicyEnforced=true) +if [ -n "${KATA_CONTAINERS_RUNTIME_CLASS:-}" ]; then + # Admission approval is cluster-wide; preserve even an explicitly empty label. + # shellcheck disable=SC2016 # These variables belong to kubectl's Go template. + KATA_RUNTIME_CLASS_PREVIOUS_LABEL="$(kctl get runtimeclass "${KATA_CONTAINERS_RUNTIME_CLASS}" \ + -o 'go-template={{range $key, $value := .metadata.labels}}{{if eq $key "openshell.ai/sandbox-attachable"}}{{$key}}={{$value}}{{end}}{{end}}')" + KATA_RUNTIME_CLASS_PREVIOUS_LABEL="${KATA_RUNTIME_CLASS_PREVIOUS_LABEL:-openshell.ai/sandbox-attachable-}" + # Arm cleanup before mutation so a failed or interrupted label request is covered. + KATA_RUNTIME_CLASS_LABEL_RESTORE=1 + kctl label runtimeclass "${KATA_CONTAINERS_RUNTIME_CLASS}" \ + openshell.ai/sandbox-attachable=true --overwrite + helm_extra_args+=(--set-string "server.defaultRuntimeClassName=${KATA_CONTAINERS_RUNTIME_CLASS}") +fi if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" != "1" ]; then echo "ERROR: external Kubernetes driver e2e requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=1." >&2 diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ad331a2bda..7edab2229c 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -103,8 +103,11 @@ For resource-admission failures, distinguish disabled caller driver config from missing resource approval. Helm defaults `server.drivers.kubernetes.allowDriverConfig` to false and `resourceAdmission.enabled` to true. Existing PVCs, RuntimeClasses, and PriorityClasses need matching administrator-owned labels; namespace -membership and read-only access do not grant approval. GPU devices and -operator-selected image-pull Secrets do not need admission labels. In managed +membership and read-only access do not grant approval. For Kata deployments, +confirm the RuntimeClass selected by `server.defaultRuntimeClassName` exists +and has the administrator-owned `openshell.ai/sandbox-attachable=true` label, +including when it is the default. GPU devices and operator-selected image-pull +Secrets do not need admission labels. In managed mode, inspect the configured source image-pull Secret in the gateway namespace and the generation copies (`openshell.ai/component=image-pull`) in the workspace namespace. Legacy workloads without