diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index dd2aee44..cfb2875b 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -171,5 +171,17 @@ Options: * `-w`: wide option * `-o yaml/json`: output format (default: json) * `-k kubeconfig`: path to kubeconfig (default: ~/.kube/config) +* `AMD_TEST_RUNNER_LOG_DIR`: host directory containing test runner output + (default: `/var/log/amd-test-runner`). The script copies this directory from + each node it collects into the techsupport bundle, preserving generated gzip + files. Set this environment variable when the test runner uses a different + `hostPath`, for example: + + ```bash + AMD_TEST_RUNNER_LOG_DIR=/mnt/test-runner-logs ./tools/techsupport_dump.sh all + ``` + + The path must be absolute and below the host root. Use the same path as the + test runner's configured log `hostPath`. Please file an issue with collected techsupport bundle on our [GitHub Issues](https://github.com/ROCm/gpu-operator/issues) page diff --git a/tools/techsupport_dump.sh b/tools/techsupport_dump.sh index 9c747725..adbbd8c3 100755 --- a/tools/techsupport_dump.sh +++ b/tools/techsupport_dump.sh @@ -35,6 +35,7 @@ WORKFLOW_RESOURCES="workflows" OUTPUT_FORMAT="json" WIDE="" +AMD_TEST_RUNNER_LOG_DIR="${AMD_TEST_RUNNER_LOG_DIR:-/var/log/amd-test-runner}" red='\033[0;31m' green='\033[0;32m' clr='\033[0m' @@ -44,6 +45,7 @@ usage() { echo -e " [-w] wide option " echo -e " [-o yaml/json] output format, yaml/json(default)" echo -e " [-k kubeconfig] path to kubeconfig(default ~/.kube/config)" + echo -e " [AMD_TEST_RUNNER_LOG_DIR=/path] runner host log directory (default /var/log/amd-test-runner)" exit 0 } @@ -55,6 +57,25 @@ die() { echo -e "${red}$* ${clr}" && exit 1 } +case "${AMD_TEST_RUNNER_LOG_DIR}" in + /*) ;; + *) die "AMD_TEST_RUNNER_LOG_DIR must be an absolute host path" ;; +esac +# Normalize harmless separators and dot components, while rejecting parent +# traversal and paths that resolve lexically to the host root (for example +# /, //, /./, or /foo/..). +RUNNER_LOG_DIR_NORMALIZED="" +IFS='/' read -r -a RUNNER_LOG_DIR_PARTS <<< "${AMD_TEST_RUNNER_LOG_DIR}" +for runner_log_dir_part in "${RUNNER_LOG_DIR_PARTS[@]}"; do + case "${runner_log_dir_part}" in + ""|.) continue ;; + ..) die "AMD_TEST_RUNNER_LOG_DIR must not contain parent traversal" ;; + *) RUNNER_LOG_DIR_NORMALIZED="${RUNNER_LOG_DIR_NORMALIZED}/${runner_log_dir_part}" ;; + esac +done +[ -n "${RUNNER_LOG_DIR_NORMALIZED}" ] || die "AMD_TEST_RUNNER_LOG_DIR must name a directory below the host root" +AMD_TEST_RUNNER_LOG_DIR="${RUNNER_LOG_DIR_NORMALIZED}" + pod_logs() { NS=$1 FEATURE=$2 @@ -69,8 +90,17 @@ pod_logs() { log " ${NS}/${pod}" ${KNS} logs "${pod}" --all-containers >${TECH_SUPPORT_FILE}/${NODE}/${FEATURE}/${NS}_${pod}.txt 2>&1 || true ${KNS} logs -p "${pod}" --all-containers --tail 1 >/dev/null 2>&1 && ${KNS} logs -p "${pod}" >${TECH_SUPPORT_FILE}/${NODE}/${FEATURE}/${NS}_${pod}_previous.txt 2>&1 || true + # Capture init-container output explicitly: --all-containers may fail + # while the regular workload container is waiting on init completion. + INIT_CONTAINERS=$(${KNS} get pod "${pod}" -o 'jsonpath={.spec.initContainers[*].name}' 2>/dev/null || true) + for init_container in ${INIT_CONTAINERS}; do + ${KNS} logs "${pod}" -c "${init_container}" >${TECH_SUPPORT_FILE}/${NODE}/${FEATURE}/${NS}_${pod}_${init_container}.txt 2>&1 || true + if ${KNS} logs -p "${pod}" -c "${init_container}" --tail 1 >/dev/null 2>&1; then + ${KNS} logs -p "${pod}" -c "${init_container}" >${TECH_SUPPORT_FILE}/${NODE}/${FEATURE}/${NS}_${pod}_${init_container}_previous.txt 2>&1 || true + fi + done done - echo "${PODS}" >${TECH_SUPPORT_FILE}/${node}/${FEATURE}/pods.txt + echo "${PODS}" >>${TECH_SUPPORT_FILE}/${node}/${FEATURE}/pods.txt } while getopts who:k: opt; do @@ -511,12 +541,20 @@ REASON fi fi - # test runner pod logs - TEST_RUNNER_PODS=$(${KNS} get pods -o name --field-selector spec.nodeName=${node} 2>/dev/null | grep -i test-runner- || true) - if [ -n "$TEST_RUNNER_PODS" ]; then - if ! pod_logs $GPUOPER_NS "test-runner" $node "$TEST_RUNNER_PODS"; then - log "Failed to collect logs for test-runner on node ${node}" - fi + # Test runner Jobs may run in any namespace and their generated pod names + # are recipe/job names (for example, gst-single-manual-...), not necessarily + # names containing "test-runner". Select pods by runner container/init + # container name instead. + TEST_RUNNER_ROWS=$(${KUBECTL} get pods -A --field-selector spec.nodeName=${node} \ + -o 'custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,CONTAINERS:.spec.containers[*].name,INIT_CONTAINERS:.spec.initContainers[*].name' \ + --no-headers 2>/dev/null | awk '$0 ~ /test[-_]runner/ { print $1, $2 }' || true) + if [ -n "$TEST_RUNNER_ROWS" ]; then + while read -r runner_ns runner_pod; do + [ -z "$runner_pod" ] && continue + if ! ( pod_logs "$runner_ns" "test-runner" "$node" "pod/${runner_pod}" ); then + log "Failed to collect logs for test-runner pod ${runner_ns}/${runner_pod}" + fi + done <<< "$TEST_RUNNER_ROWS" fi # workflow controller pod logs @@ -553,6 +591,33 @@ REASON # wait for the debug pod for dbgpod in ${dbgpods}; do ${KUBECTL} wait --for=condition=Ready=true ${dbgpod} >/dev/null + + # Test runner writes compressed recipe logs to this hostPath. Copy the + # configured directory from each selected node so logs survive Job expiry. + # Set AMD_TEST_RUNNER_LOG_DIR to override the documented default. + RUNNER_BUNDLE_DIR="${TECH_SUPPORT_FILE}/${node}/test-runner" + RUNNER_HOST_LOG_DIR="/host${AMD_TEST_RUNNER_LOG_DIR}" + LOG_PATH_KEY=$(printf '%s' "${AMD_TEST_RUNNER_LOG_DIR}" | sed -e 's#^/##' -e 's#[^[:alnum:]_.-]#_#g') + RUNNER_HOST_LOG_OUT="${RUNNER_BUNDLE_DIR}/host-logs/${LOG_PATH_KEY}" + mkdir -p "${RUNNER_HOST_LOG_OUT}" + if ${KUBECTL} exec ${dbgpod} -- test -d "${RUNNER_HOST_LOG_DIR}" >/dev/null 2>&1; then + RUNNER_LOG_ARCHIVE="${RUNNER_HOST_LOG_OUT}.tar" + if ${KUBECTL} exec ${dbgpod} -- tar -C "${RUNNER_HOST_LOG_DIR}" -cf - . \ + >"${RUNNER_LOG_ARCHIVE}" 2>"${RUNNER_BUNDLE_DIR}/collection-stderr.txt"; then + if tar -tf "${RUNNER_LOG_ARCHIVE}" >"${RUNNER_HOST_LOG_OUT}-files.txt" 2>/dev/null; then + tar -xf "${RUNNER_LOG_ARCHIVE}" -C "${RUNNER_HOST_LOG_OUT}" + rm -f "${RUNNER_LOG_ARCHIVE}" + log " collected test-runner host logs from ${AMD_TEST_RUNNER_LOG_DIR}" + else + echo "Failed to read the test-runner host log archive." >"${RUNNER_HOST_LOG_OUT}-collection-status.txt" + fi + else + echo "Failed to archive ${AMD_TEST_RUNNER_LOG_DIR} from the host." >"${RUNNER_HOST_LOG_OUT}-collection-status.txt" + fi + else + echo "${AMD_TEST_RUNNER_LOG_DIR} does not exist on this node." >"${RUNNER_HOST_LOG_OUT}-collection-status.txt" + fi + log " lsmod" ${KUBECTL} exec ${dbgpod} -- sh -c "lsmod || true" >${TECH_SUPPORT_FILE}/${node}/lsmod.txt log " dmesg"