diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b9dbf81 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,28 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true + +[*.{sh,bash}] +indent_style = tab + +[Makefile] +indent_style = tab + +[Kbuild] +indent_style = tab + +[*.{json,md,yml,yaml}] +indent_style = space +indent_size = 2 + +[*.{c,h}] +indent_style = tab +indent_size = 8 + +[*.ps1] +indent_style = space +indent_size = 4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..75dcc27 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,113 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +jobs: + shellcheck: + name: shellcheck + bash -n + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + + - name: bash -n (syntax check) + run: | + for f in scripts/lib/*.sh scripts/0*.sh; do + echo "::group::bash -n $f" + bash -n "$f" + echo "::endgroup::" + done + + - name: shellcheck + run: shellcheck -x scripts/lib/common.sh scripts/0*.sh + + json: + name: validate JSON + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: validate .vscode/*.json + run: | + for f in .vscode/*.json; do + echo "::group::python -m json.tool $f" + python3 -m json.tool "$f" > /dev/null + echo "::endgroup::" + done + + makefile: + name: Makefile syntax + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: make help (dry-parse) + run: make help + + build: + name: build example module against runner kernel (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: install host kernel headers + build tools + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + bc bison flex libelf-dev libssl-dev \ + "linux-headers-$(uname -r)" + + - name: use the example module + run: make use-example NAME=chuck_norise + + - name: build via top-level Makefile (default target) + # KDIR defaults to /lib/modules/$(uname -r)/build when there's no + # .kernel-cache/current, so this exercises the full prepare-build + + # KCFLAGS pipeline against a real (the runner's) kernel. + run: make modules + + - name: assert a .ko was produced + run: | + set -e + shopt -s nullglob + kos=(build/artifacts/*/*/*.ko) + if [ "${#kos[@]}" -eq 0 ]; then + echo "::error::no .ko produced under build/artifacts/" + ls -R build/ || true + exit 1 + fi + for ko in "${kos[@]}"; do + echo "built $ko ($(stat -c %s "$ko") bytes)" + file "$ko" + done + + - name: verify DWARF prefix-map rewrote paths back to module/ + # The whole point of -ffile-prefix-map is that DWARF references your + # source under module/<...>.c, not build/intermediate/<...>.c. If a + # change to the Makefile breaks the injection, this catches it. + run: | + set -e + ko=$(find build/artifacts -name '*.ko' | head -1) + if objdump --dwarf=decodedline "$ko" 2>/dev/null \ + | grep -qE '/build/intermediate/'; then + echo "::error::DWARF still references build/intermediate/ — prefix-map broken" + objdump --dwarf=decodedline "$ko" 2>/dev/null | grep -E '/build/intermediate/' | head + exit 1 + fi + echo "ok: DWARF paths point at module/" + + - name: clean-module resets cleanly + run: | + make clean-module + ls module/ + [ "$(ls module/)" = "README.md" ] || { echo "::error::clean-module did not reduce module/ to just README.md"; exit 1; } diff --git a/.gitignore b/.gitignore index 00f934b..aa02f4b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,16 @@ +# Per-machine config (lab.example.env is the tracked template). lab.local.env +# Per-target synced kernel headers, source, vmlinux. .kernel-cache/ -.gdb/*.gdb -.gdb/*.ready -.gdb/*.log + +# Generated GDB init files (regenerated by scripts/04 on every F5). +.gdb/ + +# Per-target build intermediates and final .ko artifacts. build/ +# Kbuild stragglers in case anything runs out of place. *.ko *.mod *.mod.c @@ -14,6 +19,13 @@ build/ Module.symvers modules.order +# IDE / editor noise. .vscode/ipch/ .codex +.claude/ +# Agent / session local notes (CLAUDE.md is consumed by Claude Code; keep it +# out of the public repo so workflow guidance stays separate from user docs). +CLAUDE.md +session.md +todo.md diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 1f4042d..9065b7f 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -3,20 +3,19 @@ "configurations": [ { "name": "Linux kernel module - current", - "compilerPath": "/usr/bin/gcc-13", + "compilerPath": "/usr/bin/gcc", "intelliSenseMode": "linux-gcc-x64", "cStandard": "gnu11", "defines": [ "__KERNEL__", "MODULE", "CC_USING_FENTRY", - "KBUILD_MODNAME=\"hello\"", - "KBUILD_BASENAME=\"hello\"", - "__KBUILD_MODNAME=kmod_hello" + "KBUILD_MODNAME=\"kmod\"", + "KBUILD_BASENAME=\"kmod\"", + "__KBUILD_MODNAME=kmod_kmod" ], "includePath": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", + "${workspaceFolder}/module/**", "${workspaceFolder}/.kernel-cache/current/build/include", "${workspaceFolder}/.kernel-cache/current/build/include/uapi", "${workspaceFolder}/.kernel-cache/current/build/include/generated", @@ -29,8 +28,7 @@ ], "browse": { "path": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", + "${workspaceFolder}/module", "${workspaceFolder}/.kernel-cache/current/build/ubuntu/include", "${workspaceFolder}/.kernel-cache/current/build/include", "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include" @@ -42,90 +40,6 @@ "${workspaceFolder}/.kernel-cache/current/build/include/linux/kconfig.h", "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler_types.h" ] - }, - { - "name": "Linux kernel module - desktop", - "compilerPath": "/usr/bin/gcc-13", - "intelliSenseMode": "linux-gcc-x64", - "cStandard": "gnu11", - "defines": [ - "__KERNEL__", - "MODULE", - "CC_USING_FENTRY", - "KBUILD_MODNAME=\"hello\"", - "KBUILD_BASENAME=\"hello\"", - "__KBUILD_MODNAME=kmod_hello" - ], - "includePath": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", - "${workspaceFolder}/.kernel-cache/desktop/build/include", - "${workspaceFolder}/.kernel-cache/desktop/build/include/uapi", - "${workspaceFolder}/.kernel-cache/desktop/build/include/generated", - "${workspaceFolder}/.kernel-cache/desktop/build/include/generated/uapi", - "${workspaceFolder}/.kernel-cache/desktop/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include/uapi", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include/generated", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include/generated/uapi" - ], - "browse": { - "path": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", - "${workspaceFolder}/.kernel-cache/desktop/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/desktop/build/include", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include" - ], - "limitSymbolsToIncludedHeaders": true - }, - "forcedInclude": [ - "${workspaceFolder}/.kernel-cache/desktop/build/include/linux/compiler-version.h", - "${workspaceFolder}/.kernel-cache/desktop/build/include/linux/kconfig.h", - "${workspaceFolder}/.kernel-cache/desktop/build/include/linux/compiler_types.h" - ] - }, - { - "name": "Linux kernel module - server", - "compilerPath": "/usr/bin/gcc-13", - "intelliSenseMode": "linux-gcc-x64", - "cStandard": "gnu11", - "defines": [ - "__KERNEL__", - "MODULE", - "CC_USING_FENTRY", - "KBUILD_MODNAME=\"hello\"", - "KBUILD_BASENAME=\"hello\"", - "__KBUILD_MODNAME=kmod_hello" - ], - "includePath": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", - "${workspaceFolder}/.kernel-cache/server/build/include", - "${workspaceFolder}/.kernel-cache/server/build/include/uapi", - "${workspaceFolder}/.kernel-cache/server/build/include/generated", - "${workspaceFolder}/.kernel-cache/server/build/include/generated/uapi", - "${workspaceFolder}/.kernel-cache/server/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include/uapi", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include/generated", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include/generated/uapi" - ], - "browse": { - "path": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", - "${workspaceFolder}/.kernel-cache/server/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/server/build/include", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include" - ], - "limitSymbolsToIncludedHeaders": true - }, - "forcedInclude": [ - "${workspaceFolder}/.kernel-cache/server/build/include/linux/compiler-version.h", - "${workspaceFolder}/.kernel-cache/server/build/include/linux/kconfig.h", - "${workspaceFolder}/.kernel-cache/server/build/include/linux/compiler_types.h" - ] } ] } diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 9a91c37..97d3205 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,6 +1,6 @@ { "recommendations": [ - "ms-vscode.cpptools" + "ms-vscode.cpptools", + "webfreak.debug" ] } - diff --git a/.vscode/launch.json b/.vscode/launch.json index 9c396ab..cb0c619 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,20 +1,28 @@ { "version": "0.2.0", + "inputs": [ + { + "id": "debugEndpoint", + "type": "promptString", + "description": "GDB remote endpoint (host:port). Only used by Native Debug — cppdbg reads it from the generated .gdb script.", + "default": "127.0.0.1:1234" + } + ], "configurations": [ { - "name": "Kernel: Debug server", + "name": "Kernel: cppdbg (full GDB MI)", "type": "cppdbg", "request": "launch", - "program": "${workspaceFolder}/.kernel-cache/server/vmlinux", + "program": "${workspaceFolder}/.gdb/current-vmlinux", "cwd": "${workspaceFolder}", "MIMode": "gdb", "miDebuggerPath": "gdb", "targetArchitecture": "x64", - "preLaunchTask": "Kernel: Deploy Debug server", + "preLaunchTask": "Kernel: Deploy Debug", "setupCommands": [ { - "description": "Attach KGDB and trigger deferred module load", - "text": "source ${workspaceFolder}/.gdb/server-kgdb.gdb", + "description": "Source the GDB init script generated by scripts/04", + "text": "source ${workspaceFolder}/.gdb/current-debug.gdb", "ignoreFailures": false } ], @@ -22,24 +30,18 @@ "externalConsole": false }, { - "name": "Kernel: Debug desktop", - "type": "cppdbg", - "request": "launch", - "program": "${workspaceFolder}/.kernel-cache/desktop/vmlinux", + "name": "Kernel: Native Debug (faster, fewer features)", + "type": "gdb", + "request": "attach", + "executable": "${workspaceFolder}/.gdb/current-vmlinux", + "target": "${input:debugEndpoint}", + "remote": true, "cwd": "${workspaceFolder}", - "MIMode": "gdb", - "miDebuggerPath": "gdb", - "targetArchitecture": "x64", - "preLaunchTask": "Kernel: Deploy Debug desktop", - "setupCommands": [ - { - "description": "Attach KGDB and trigger deferred module load", - "text": "source ${workspaceFolder}/.gdb/desktop-kgdb.gdb", - "ignoreFailures": false - } - ], - "launchCompleteCommand": "exec-continue", - "externalConsole": false + "preLaunchTask": "Kernel: Deploy Debug", + "valuesFormatting": "parseText", + "autorun": [ + "source ${workspaceFolder}/.gdb/current-debug-attached.gdb" + ] } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 713c7a9..9503432 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,39 +1,15 @@ { "C_Cpp.default.configurationProvider": "", "C_Cpp.default.configurationName": "Linux kernel module - current", - "C_Cpp.default.compileCommands": "${workspaceFolder}/compile_commands.json", - "C_Cpp.default.compilerPath": "/usr/bin/gcc-13", - "C_Cpp.default.cStandard": "gnu11", - "C_Cpp.default.intelliSenseMode": "linux-gcc-x64", - "C_Cpp.default.defines": [ - "__KERNEL__", - "MODULE", - "CC_USING_FENTRY", - "KBUILD_MODNAME=\"hello\"", - "KBUILD_BASENAME=\"hello\"", - "__KBUILD_MODNAME=kmod_hello" - ], - "C_Cpp.default.includePath": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", - "${workspaceFolder}/.kernel-cache/current/build/include", - "${workspaceFolder}/.kernel-cache/current/build/include/uapi", - "${workspaceFolder}/.kernel-cache/current/build/include/generated", - "${workspaceFolder}/.kernel-cache/current/build/include/generated/uapi", - "${workspaceFolder}/.kernel-cache/current/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include/uapi", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include/generated", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include/generated/uapi" - ], - "C_Cpp.default.forcedInclude": [ - "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler-version.h", - "${workspaceFolder}/.kernel-cache/current/build/include/linux/kconfig.h", - "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler_types.h" - ], "C_Cpp.errorSquiggles": "enabled", "files.associations": { "*.h": "c", - "*.c": "c" + "*.c": "c", + "Kbuild": "makefile" + }, + "files.exclude": { + ".kernel-cache": true, + ".gdb": true, + "build": true } } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 4a3e933..69e19f9 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -3,26 +3,38 @@ "inputs": [ { "id": "kernelTarget", + "type": "promptString", + "description": "Target profile (one of the names in TARGETS in lab.local.env)", + "default": "server" + }, + { + "id": "debugMethod", "type": "pickString", - "description": "Ubuntu 24.04 target VM", + "description": "Debug method", "options": [ - "desktop", - "server" + "kgdb", + "qemu" ], - "default": "server" + "default": "qemu" } ], "tasks": [ + { + "label": "Kernel: Check Target", + "type": "shell", + "command": "./scripts/00-check-target.sh ${input:kernelTarget} ${input:debugMethod}", + "problemMatcher": [] + }, { "label": "Kernel: Provision Target", "type": "shell", - "command": "./scripts/01-provision-target.sh ${input:kernelTarget}", + "command": "./scripts/01-provision-target.sh ${input:kernelTarget} ${input:debugMethod}", "problemMatcher": [] }, { - "label": "Kernel: Setup WSL Build", + "label": "Kernel: Setup Host Build", "type": "shell", - "command": "./scripts/02-setup-wsl-build.sh ${input:kernelTarget}", + "command": "./scripts/02-setup-host-build.sh ${input:kernelTarget}", "problemMatcher": [] }, { @@ -38,19 +50,20 @@ { "label": "Kernel: Deploy Debug", "type": "shell", - "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget}", - "problemMatcher": [] - }, - { - "label": "Kernel: Deploy Debug server", - "type": "shell", - "command": "./scripts/04-deploy-debug-vscode.sh server", + "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget} ${input:debugMethod}", "problemMatcher": [] }, { - "label": "Kernel: Deploy Debug desktop", + "label": "Kernel: GDB", "type": "shell", - "command": "./scripts/04-deploy-debug-vscode.sh desktop", + "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget} ${input:debugMethod} && gdb -tui ${workspaceFolder}/.gdb/current-vmlinux -x ${workspaceFolder}/.gdb/current-debug.gdb", + "presentation": { + "echo": false, + "reveal": "always", + "focus": true, + "panel": "dedicated", + "clear": true + }, "problemMatcher": [] } ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d7a590b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,68 @@ +# Contributing + +Bug reports, fixes, and improvements welcome. + +## What belongs here + +This is a **template** for building and source-debugging out-of-tree +Linux kernel modules. Lab tooling lives in `Makefile`, `scripts/`, +`.vscode/`, `host/`. The example under `examples/chuck_norise/` +demonstrates the workflow end-to-end. + +**Yes:** + +- Bugs in the lab tooling: broken scripts, wrong assumptions, scripts + that fail on re-run, unclear errors. +- New target-OS backends (currently only Ubuntu — see the `TARGET_OS` + checks in scripts 01 and 02). +- New debug methods that fit the ` → GDB` shape. +- Docs fixes. +- Quality-of-life: better errors, idempotent re-runs, faster sync. + +**No:** + +- Changes to `module/` (user's slot; stays as just a placeholder README). +- New examples under `examples/` (one focused example beats many). +- Project-management features (work logs, task tracking). + +## Reporting bugs + +Open an issue with: + +- Dev host (distro, version, kernel) and target VM (distro, version, + kernel, hypervisor). +- Debug method tried (`kgdb` or `qemu`) and endpoint configured. +- Command run and full output. For deploy failures, attach + `.gdb/--loader.log`. +- What you expected vs what happened. + +## Submitting changes + +1. Fork, branch from `main`. +2. Focused commits — one logical change per commit. +3. Keep scripts shellcheck-clean and `bash -n`-clean: + `shellcheck -x scripts/lib/common.sh scripts/0*.sh` and + `for f in scripts/lib/*.sh scripts/0*.sh; do bash -n "$f"; done`. +4. Update docs in the same PR — if you touched a script, check + `README.md` and `module/README.md` for anything the change made + inaccurate. +5. PR description explains *why*, not just what. + +## Style + +- **Shell:** tabs for indent. `set -euo pipefail` at the top. Errors via + the `die` helper. Use `lab_*` helpers from `scripts/lib/common.sh` — + don't call ssh/scp/sshpass binaries directly from numbered scripts. +- **Makefile:** tabs for recipes; explicit `.PHONY`. +- **Markdown:** wrap at ~80; fenced code blocks with language tags. +- **JSON / YAML:** 2-space indent, trailing newline. + +`.editorconfig` captures these — most editors apply it automatically. + +## Testing locally + +No automated test suite for the build/debug flow (it needs real VMs). +Before sending a PR, smoke-test the full chain against at least one +target VM: `01 --debug-symbols` → reboot → `02` → `03` → `04` → F5 in +VS Code → set breakpoints in both your module and a kernel function, +confirm both bind. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b77f1ff --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +MIT License + +Copyright (c) 2026 Dor Grosglick and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +--- + +This MIT license covers the lab tooling in this repository (Makefile, +scripts/, .vscode/, host/, docs). The example module under +examples/chuck_norise/ is also MIT-licensed for ease of reuse. Any kernel +module you place under module/ is yours — this lab's license does not impose +terms on your module's source. diff --git a/Makefile b/Makefile index ac62736..7c48b92 100644 --- a/Makefile +++ b/Makefile @@ -1,41 +1,124 @@ -MODULE_NAME := hello -HOST_KERNEL := $(shell uname -r) -CURRENT_CACHE := $(CURDIR)/.kernel-cache/current +# Top-level wrapper for the user's module under module/. +# +# Builds out-of-tree against a target-specific kernel header tree under +# .kernel-cache//. Sources are staged into +# build/intermediate/// before Kbuild runs from there, +# and `-ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR)` is injected +# via KCFLAGS so DWARF paths and `__FILE__` resolve back to module/ — +# IDE breakpoints keyed by absolute path bind to the real sources. +# +# Per-target staging also keeps each (target, kernel) pair's object +# files isolated. The user's module/Makefile stays a normal Kbuild file +# with no knowledge of the staging. + +HOST_KERNEL := $(shell uname -r) +CURRENT_CACHE := $(CURDIR)/.kernel-cache/current CURRENT_TARGET := $(shell if [ -e "$(CURRENT_CACHE)" ]; then basename "$$(readlink -f "$(CURRENT_CACHE)")"; else printf local; fi) CURRENT_KERNEL := $(shell if [ -f "$(CURRENT_CACHE)/kernel.release" ]; then cat "$(CURRENT_CACHE)/kernel.release"; else printf '$(HOST_KERNEL)'; fi) -KDIR ?= $(if $(wildcard $(CURRENT_CACHE)/build),$(CURRENT_CACHE)/build,/lib/modules/$(HOST_KERNEL)/build) -BUILD_ID ?= $(CURRENT_TARGET)/$(CURRENT_KERNEL) -BUILD_ROOT ?= $(CURDIR)/build +KDIR ?= $(if $(wildcard $(CURRENT_CACHE)/build),$(CURRENT_CACHE)/build,/lib/modules/$(HOST_KERNEL)/build) +BUILD_ID ?= $(CURRENT_TARGET)/$(CURRENT_KERNEL) +BUILD_ROOT ?= $(CURDIR)/build INTERMEDIATE_DIR ?= $(BUILD_ROOT)/intermediate/$(BUILD_ID) -ARTIFACT_DIR ?= $(BUILD_ROOT)/artifacts/$(BUILD_ID) -MODULE_DIR ?= $(CURDIR)/module +ARTIFACT_DIR ?= $(BUILD_ROOT)/artifacts/$(BUILD_ID) +MODULE_DIR ?= $(CURDIR)/module + +EXTRA_CCFLAGS ?= -g -DDEBUG +KCFLAGS_INJECT := -ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR) $(EXTRA_CCFLAGS) -.PHONY: all modules prepare-build clean +.PHONY: all modules prepare-build clean help use-example clean-module all: modules +help: + @echo "Top-level lab Makefile." + @echo "" + @echo "Normally invoked by scripts/03-build-module.sh , which sets" + @echo "KDIR, BUILD_ID, INTERMEDIATE_DIR, and ARTIFACT_DIR per-target." + @echo "" + @echo "Build targets:" + @echo " modules build module/ against KDIR (default)" + @echo " clean remove BUILD_ID's intermediate + artifact dirs" + @echo " prepare-build stage module/ into INTERMEDIATE_DIR (internal)" + @echo "" + @echo "module/ ergonomics:" + @echo " use-example NAME=chuck_norise copy an example into module/" + @echo " clean-module reset module/ to just README.md" + @echo "" + @echo "Vars (auto-resolved when .kernel-cache/current points at a target):" + @echo " MODULE_DIR $(MODULE_DIR)" + @echo " KDIR $(KDIR)" + @echo " BUILD_ID $(BUILD_ID)" + @echo " INTERMEDIATE_DIR $(INTERMEDIATE_DIR)" + @echo " ARTIFACT_DIR $(ARTIFACT_DIR)" + @echo " EXTRA_CCFLAGS $(EXTRA_CCFLAGS) (appended to KCFLAGS)" + +# Copy examples/$(NAME)/ into module/. Refuses if module/ already has a +# Makefile/Kbuild (run clean-module first). Preserves the placeholder +# module/README.md so the example's own README doesn't overwrite it. +NAME ?= chuck_norise +use-example: + @if [ ! -d "$(CURDIR)/examples/$(NAME)" ]; then \ + echo "Makefile: examples/$(NAME) does not exist;" >&2; \ + echo " available examples:" >&2; \ + ls $(CURDIR)/examples 2>/dev/null | sed 's/^/ /' >&2; \ + exit 1; \ + fi + @if [ -f "$(MODULE_DIR)/Makefile" ] || [ -f "$(MODULE_DIR)/Kbuild" ]; then \ + echo "Makefile: $(MODULE_DIR) already has a Makefile/Kbuild." >&2; \ + echo " Run 'make clean-module' first if you want to replace it." >&2; \ + exit 1; \ + fi + @echo "copying examples/$(NAME)/* -> $(MODULE_DIR)/ (keeping module/README.md placeholder)" + @find "$(CURDIR)/examples/$(NAME)" -mindepth 1 -maxdepth 1 -not -name README.md \ + -exec cp -r {} "$(MODULE_DIR)/" \; + @echo "done. next: scripts/03-build-module.sh " + @echo "(the example's docs stay at examples/$(NAME)/README.md)" + +clean-module: + @[ -d "$(MODULE_DIR)" ] || { echo "Makefile: $(MODULE_DIR) is missing." >&2; exit 1; } + @find "$(MODULE_DIR)" -mindepth 1 -maxdepth 1 -not -name README.md -print0 \ + | xargs -0 -r rm -rf + @echo "module/ reset (kept only README.md)" + modules: prepare-build - $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" modules - mkdir -p "$(ARTIFACT_DIR)" - mv "$(INTERMEDIATE_DIR)/$(MODULE_NAME).ko" "$(ARTIFACT_DIR)/$(MODULE_NAME).ko" - @echo "built $(ARTIFACT_DIR)/$(MODULE_NAME).ko" + $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" KCFLAGS="$(KCFLAGS_INJECT)" modules + @mkdir -p "$(ARTIFACT_DIR)" + @set -e; \ + kos=$$(find "$(INTERMEDIATE_DIR)" -maxdepth 2 -name '*.ko' -type f); \ + if [ -z "$$kos" ]; then \ + echo "Makefile: no .ko produced under $(INTERMEDIATE_DIR);" >&2; \ + echo " check the Kbuild output above for compile errors." >&2; \ + exit 1; \ + fi; \ + for ko in $$kos; do \ + name=$$(basename "$$ko"); \ + mv "$$ko" "$(ARTIFACT_DIR)/$$name"; \ + echo "built $(ARTIFACT_DIR)/$$name"; \ + done +# `cp -as` recursively creates dirs and absolute symlinks for each file. +# Re-runs are idempotent: we wipe the symlink scaffolding first so a +# removed source file doesn't linger as a stale symlink. prepare-build: - mkdir -p "$(INTERMEDIATE_DIR)" - if [ -L "$(INTERMEDIATE_DIR)/src" ]; then rm -f "$(INTERMEDIATE_DIR)/src"; fi - mkdir -p "$(INTERMEDIATE_DIR)/src" - ln -sfn "$(MODULE_DIR)/src/hello.c" "$(INTERMEDIATE_DIR)/src/hello.c" - ln -sfn "$(MODULE_DIR)/src/chuck_device.c" "$(INTERMEDIATE_DIR)/src/chuck_device.c" - ln -sfn "$(MODULE_DIR)/src/chuck_message.c" "$(INTERMEDIATE_DIR)/src/chuck_message.c" - if [ -L "$(INTERMEDIATE_DIR)/include" ]; then rm -f "$(INTERMEDIATE_DIR)/include"; fi - mkdir -p "$(INTERMEDIATE_DIR)/include" - ln -sfn "$(MODULE_DIR)/include/chuck_device.h" "$(INTERMEDIATE_DIR)/include/chuck_device.h" - ln -sfn "$(MODULE_DIR)/include/chuck_message.h" "$(INTERMEDIATE_DIR)/include/chuck_message.h" - ln -sfn "$(MODULE_DIR)/Kbuild" "$(INTERMEDIATE_DIR)/Kbuild" + @if [ ! -e "$(MODULE_DIR)/Makefile" ] && [ ! -e "$(MODULE_DIR)/Kbuild" ]; then \ + echo "Makefile: no Makefile or Kbuild under $(MODULE_DIR);" >&2; \ + echo " drop your kernel module project there (see $(MODULE_DIR)/README.md)" >&2; \ + echo " or try the example: make use-example NAME=chuck_norise" >&2; \ + exit 1; \ + fi + @cp --help 2>&1 | grep -q -- '--symbolic-link' || { \ + echo "Makefile: GNU 'cp' (with --symbolic-link, --archive) is required;" >&2; \ + echo " the lab targets Debian-based hosts. On macOS: brew install coreutils, then re-run with CP=gcp." >&2; \ + exit 1; \ + } + @mkdir -p "$(INTERMEDIATE_DIR)" + @find "$(INTERMEDIATE_DIR)" -depth -type l -delete + @find "$(INTERMEDIATE_DIR)" -depth -type d -empty -not -path "$(INTERMEDIATE_DIR)" -delete + @cp -as "$(MODULE_DIR)/." "$(INTERMEDIATE_DIR)/" clean: - if [ -d "$(INTERMEDIATE_DIR)" ] && [ -e "$(INTERMEDIATE_DIR)/Kbuild" ]; then \ - $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" clean; \ + @if [ -d "$(INTERMEDIATE_DIR)" ] && { [ -e "$(INTERMEDIATE_DIR)/Makefile" ] || [ -e "$(INTERMEDIATE_DIR)/Kbuild" ]; }; then \ + $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" clean || true; \ fi rm -rf "$(INTERMEDIATE_DIR)" "$(ARTIFACT_DIR)" diff --git a/README.md b/README.md index e8eac41..cdfdd3b 100644 --- a/README.md +++ b/README.md @@ -1,414 +1,286 @@ -# Ubuntu 24.04 Kernel Module Debug Lab +# Linux Kernel Module Debug Lab -This repo is a WSL-first lab for building a Linux kernel module and -experimenting with source debugging on VMware Ubuntu 24.04 LTS Desktop and -Server targets. +A template for building and **source-debugging** out-of-tree Linux kernel +modules against one or more target VMs. Drop your module under `module/`, +point the lab at a target, press F5 — step through your code and into +kernel code from VS Code or `gdb -tui`. -The sample module builds as `hello.ko` and exposes `/dev/chuck_norise`. Reads -repeat the exact string `chuck norise!`, preserving the file offset for each -open file descriptor. +What you get: -## Quick Start +- **One slot, any module.** `module/` is your project; the lab is generic. +- **Per-target builds.** Cross-build the same source against multiple + target kernels without polluting your tree. +- **Breakpoints just work.** `-ffile-prefix-map` is injected via `KCFLAGS` + so IDE breakpoints by absolute path bind to your real source files. +- **Step into the kernel.** With `--debug-symbols`, the lab pulls down + `vmlinux` + kernel source and wires GDB `substitute-path` so you can + step from your module's read handler into `vfs_read`. +- **Two debug methods.** In-kernel KGDB over serial (`kgdb`) and any + hypervisor GDB stub (`qemu`, also VMware's `debugStub`). -1. Copy `lab.example.env` to `lab.local.env`. -2. Fill in the SSH values for `desktop` and `server`. Fill in the debug - endpoint values too if you want to use F5 debugging. -3. Provision each VM once, then reboot it: +The repo ships with a worked example (`examples/chuck_norise/`) so you +can run the full flow end-to-end before writing a line of module code. -```bash -./scripts/01-provision-target.sh desktop -./scripts/01-provision-target.sh server -``` - -4. Sync each target's kernel headers into WSL: - -```bash -./scripts/02-setup-wsl-build.sh desktop -./scripts/02-setup-wsl-build.sh server -``` +--- -5. Build from VS Code Remote-WSL with `Ctrl+Shift+B`. -6. To try source debugging, configure one of the debug endpoint options below, - then choose `Kernel: Debug desktop` or `Kernel: Debug server` and press F5. +## Quickstart -## Building - -In VS Code Remote-WSL, `Ctrl+Shift+B` runs the default `Kernel: Build` task. -The task prompts for `kernelTarget` and runs: +You need a **Debian-based dev host** (Debian, Ubuntu, WSL Ubuntu) and at +least one **Ubuntu target VM** reachable over SSH. ```bash -./scripts/03-build-module.sh -``` - -The built module is written to: - -```text -build/artifacts///hello.ko -``` +# 0. clone + configure +git clone kmod-debug-lab && cd kmod-debug-lab +cp lab.example.env lab.local.env +$EDITOR lab.local.env # TARGET_SSH_*, endpoints, etc. -Build intermediates are kept under `build/intermediate///`, so -Kbuild does not leave `.o`, `.cmd`, `Module.symvers`, or `modules.order` files -in the repo root or `module/src/`. +# optional preflight (non-mutating) +./scripts/00-check-target.sh server qemu -Running plain `make` uses `.kernel-cache/current` when it exists. The setup -script updates that link to the most recently synced target. +# 1. target setup: headers, vmlinux dbg, kernel source (one-time) +./scripts/01-provision-target.sh server qemu --debug-symbols +# ...reboot the target VM so new GRUB args take effect... -## Source Debugging Status +# 2. host setup: sync everything to .kernel-cache/server/ (one-time) +./scripts/02-setup-host-build.sh server -Source debugging is work in progress. The pieces under active development are -`scripts/04-deploy-debug-vscode.sh` and the VS Code launch tasks. - -Today, F5 runs the matching deploy/debug task for the selected target. Script 04 -expects `hello.ko` to already exist, uploads it to the VM, loads it, reads the -module section addresses from `/sys/module/hello/sections/*`, writes a GDB -symbol file under `.gdb/`, and generates the GDB startup script that VS Code -uses. The current goal is to load module symbols and stop in -`module/src/hello.c`; expect rough edges while this flow is being tightened. - -The VS Code launch configs pin the debugger architecture to `x64` while the -generated GDB script sets GDB's kernel architecture to `i386:x86-64`. This -avoids debug-adapter architecture auto-detection failures and lets GDB use the -Linux kernel architecture name it expects. The deploy script also maps generated -Kbuild source paths back to `module/` with `set substitute-path`, so source -stepping opens the real module files instead of the staged build symlinks. - -If F5 fails at `target remote :` with a timeout, GDB could not reach -the endpoint configured by `_KGDB_ENDPOINT` in `lab.local.env`. Check -that the VMware debug stub or serial bridge is listening on the same host/port -and is reachable from WSL before starting the VS Code debug launch. - -## Debug Endpoint Options - -GDB runs in WSL and needs a TCP endpoint for the target VM. Use one of these -options, then put that host and port in `_KGDB_ENDPOINT` in -`lab.local.env`. The variable name says `KGDB`, but it is also used for the -VMware debug stub endpoint. - -### Option A: VMware Debug Stub (Recommended) - -Power off the VM, open the target VM's `.vmx` file, and add: - -```text -debugStub.listen.guest64 = "TRUE" -debugStub.port.guest64 = "8864" -debugStub.listen.guest64.remote = "TRUE" -debugStub.hideBreakpoints = "FALSE" -``` +# 3. try the example (or skip and drop your own module under module/) +make use-example NAME=chuck_norise +./scripts/03-build-module.sh server -Start the VM after saving the `.vmx` file. Set the matching endpoint in -`lab.local.env`, for example: +# 4. deploy + load on the target +./scripts/04-deploy-debug-vscode.sh server qemu -```env -SERVER_KGDB_ENDPOINT=127.0.0.1:8864 +# 5. open the repo in VS Code and press F5 ("Kernel: cppdbg") +code . ``` -If WSL cannot reach the Windows loopback address, use the Windows host address -visible from WSL instead. If both VMs may run at the same time, give each VM a -different `debugStub.port.guest64` value and match that port in -`lab.local.env`. +Set a breakpoint in `module/src/hello.c` and one in `vfs_read` — both +bind. You're stepping through kernel code from your module. -This option does not need a VMware serial port or the PowerShell bridge. +After the one-time `01 + 02`, the inner loop is **03 → 04 → F5** (or +just F5 — the launch's pre-task runs 04, and 03 if needed). -### Option B: VMware Serial Port And Bridge +--- -Use this if you prefer Linux KGDB over a virtual serial port, or if the VMware -debug stub is not available in your VMware setup. The data path is: +## Repo layout -```text -VS Code in WSL -> GDB -> TCP port on Windows -> named pipe -> VMware serial port -> Ubuntu KGDB ``` +module/ your module project (Makefile + sources) +examples/chuck_norise/ worked example; `make use-example` copies it into module/ -The VM exposes a virtual serial port as a Windows named pipe. GDB cannot connect -to that pipe directly from WSL, so `host/bridge-kgdb.ps1` listens on a TCP port -and forwards bytes between TCP and the VMware named pipe. - -#### Configure The VMware Serial Port - -Power off the VM before changing virtual hardware. In VMware Workstation: - -1. Open the VM settings. -2. Add a `Serial Port` if the VM does not already have one. -3. Select `Use named pipe`. -4. Set the pipe name. -5. Select `This end is the server`. -6. Select `The other end is an application`. -7. Enable `Connect at power on`. +scripts/00-check-target.sh preflight: ssh / sudo / headers / vmlinux / endpoint +scripts/01-provision-target.sh target-side: headers, vmlinux dbg, kernel source, GRUB args + (`--uninstall` to undo the GRUB args) +scripts/02-setup-host-build.sh dev-host: sync headers + vmlinux + source into .kernel-cache/ +scripts/03-build-module.sh dev-host: build module/ against the synced headers +scripts/04-deploy-debug-vscode.sh dev-host: upload, insmod, write .gdb/-.gdb +scripts/lib/common.sh shared helpers (lab_ssh, lab_ssh_sudo, lab_rsync_from, ...) -Use one pipe per VM. The repo defaults are: +.kernel-cache// synced build/source/vmlinux (gitignored) +build/ per-target intermediate + final .ko (gitignored) +.gdb/ generated GDB init files (gitignored, regenerated each F5) -```text -desktop VM: \\.\pipe\kgdb-desktop -server VM: \\.\pipe\kgdb-server +host/ optional VMware-on-Windows helpers +lab.example.env -> lab.local.env per-machine config (gitignored copy) ``` -The first serial port in the VM normally appears as `ttyS0` inside Ubuntu. If -you add more serial ports or change the VM hardware order, adjust -`_KGDB_TTY` in `lab.local.env`. +Useful `make` targets: `make use-example NAME=`, `make clean-module`, +`make help`. -#### Configure The Lab Environment +--- -`lab.local.env` connects the VM serial device to the TCP endpoint used by GDB. -The default server values are: +## The `module/` contract -```env -SERVER_KGDB_ENDPOINT=127.0.0.1:5520 -SERVER_KGDB_TTY=ttyS0 -SERVER_KGDB_BAUD=115200 -``` +`module/` is the lab's only slot for your module. The lab does not know +its name, source layout, or behavior. Expectations: -The default desktop values are: +- **`module/Makefile`** (or `Kbuild`) following standard out-of-tree + conventions: -```env -DESKTOP_KGDB_ENDPOINT=127.0.0.1:5510 -DESKTOP_KGDB_TTY=ttyS0 -DESKTOP_KGDB_BAUD=115200 -``` + ```makefile + obj-m += my_module.o + my_module-y := src/main.o src/util.o + ccflags-y := -I$(src)/include -g -DDEBUG + ``` -The endpoint port must match the TCP port passed to `host/bridge-kgdb.ps1`. -The TTY and baud rate must match the guest kernel boot argument configured by -the provisioning script. + `examples/chuck_norise/Makefile` shows the canonical shape, including + the optional `ifndef KERNELRELEASE` wrapper that lets `make` work + directly in `module/`. -#### Provision The Guest For KGDB +- **Exactly one `obj-m` entry per build.** The lab discovers the module + name from the produced `.ko`. -From WSL, run provisioning for the target and reboot the VM: +- **Optional `debug_delay_ms` module parameter** so the host has time to + attach GDB before init runs: -```bash -./scripts/01-provision-target.sh server -``` + ```c + static unsigned int debug_delay_ms = 5000; + module_param(debug_delay_ms, uint, 0644); + ``` -Provisioning installs the target kernel headers and updates GRUB with KGDB boot -arguments like: + Script 04 passes `DEBUG_LOAD_DELAY_MS` (from `lab.local.env`) on + `insmod`. Modules that don't declare the parameter fall back to a + plain `insmod` automatically. -```text -kgdboc=ttyS0,115200 nokaslr sysrq_always_enabled=1 -``` +Source layout is yours. The lab passes through whatever your Kbuild file +declares. -Reboot is required. Without the reboot, the target kernel is still running -without KGDB on the serial port. +--- -For source debugging against full kernel symbols, provision with debug symbols: +## Configuring the lab -```bash -./scripts/01-provision-target.sh server --debug-symbols -``` - -Then sync the target kernel headers and `vmlinux` into WSL: +Targets are **data, not variable prefixes**. Add a profile name to +`TARGETS`, then add entries to the `TARGET_*` maps under that name: ```bash -./scripts/02-setup-wsl-build.sh server -``` - -#### Start The Windows Pipe-To-TCP Bridge - -Run the bridge from Windows PowerShell, not from WSL. Start it before pressing -F5 in VS Code. - -For the server target: - -```powershell -powershell -ExecutionPolicy Bypass -File .\host\bridge-kgdb.ps1 -Target server -``` - -This forwards: +TARGETS=(desktop server) -```text -\\.\pipe\kgdb-server <-> 127.0.0.1:5520 -``` - -For the desktop target: +declare -A TARGET_SSH_HOST=([desktop]=ubuntu-desktop.local [server]=ubuntu-server.local) +declare -A TARGET_SSH_USER=([desktop]=user [server]=user) +declare -A TARGET_SSH_PASS=([desktop]= [server]=) # empty when using SSH keys -```powershell -powershell -ExecutionPolicy Bypass -File .\host\bridge-kgdb.ps1 -Target desktop +declare -A TARGET_DEBUG_ENDPOINT_QEMU=( + [desktop]=127.0.0.1:1234 + [server]=127.0.0.1:1234 +) ``` -This forwards: +Endpoints are split per debug method (`TARGET_DEBUG_ENDPOINT_KGDB`, +`TARGET_DEBUG_ENDPOINT_QEMU`); either map can be empty per target. The +VS Code task picker accepts any target name you type — no edits to +`.vscode/` when adding targets. -```text -\\.\pipe\kgdb-desktop <-> 127.0.0.1:5510 -``` +--- -If your VMware pipe or TCP port is different, override the defaults: +## Debug methods -```powershell -powershell -ExecutionPolicy Bypass -File .\host\bridge-kgdb.ps1 ` - -Target server ` - -PipeName "\\.\pipe\my-kgdb-pipe" ` - -Port 5520 -``` +- **`qemu`** — the hypervisor's built-in GDB stub + (QEMU's `-gdb tcp::PORT` or `-s`, VMware's `debugStub.listen.guest64`). + No KGDB in the guest; the hypervisor halts the vCPU directly. +- **`kgdb`** — in-kernel KGDB over the guest's serial port. Script 01 + adds `kgdboc=ttyS0,115200` and `sysrq_always_enabled=1` to GRUB. Break + with `echo g | sudo tee /proc/sysrq-trigger` on the target. Useful + when you can't change the hypervisor's command line (VMware + Workstation is the common case). -Leave the PowerShell window open while debugging. It should print that it is -waiting for a GDB TCP connection. When VS Code starts debugging, it should print -that GDB connected and that the named pipe connected. +Neither method assumes any in-module `kgdb_breakpoint()` call. In both, +you break manually after 04 finishes. -## Verify The Debug Endpoint +**QEMU stub:** `qemu-system-x86_64 ... -gdb tcp::1234` (`-S` to pause +at boot). -From Windows PowerShell: +**VMware debug stub:** in the VM's `.vmx`: -```powershell -Test-NetConnection 127.0.0.1 -Port 5520 ``` - -From WSL, if `nc` is installed: - -```bash -nc -vz 127.0.0.1 5520 +debugStub.listen.guest64 = "TRUE" +debugStub.port.guest64 = "8864" +debugStub.listen.guest64.remote = "TRUE" +debugStub.hideBreakpoints = "FALSE" ``` -If Windows can connect but WSL cannot, use the Windows host address visible from -WSL instead of `127.0.0.1`: +Then set `TARGET_DEBUG_ENDPOINT_QEMU[server]=127.0.0.1:8864`. -```bash -grep nameserver /etc/resolv.conf -``` +**KGDB over serial → TCP:** bridge the guest's `/dev/ttyS0` to a TCP +listener. For QEMU: `-serial tcp:127.0.0.1:5520,server,nowait`. For +VMware Workstation on Windows: named-pipe serial port + +`host/bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520`. `TARGET_KGDB_TTY` +and `TARGET_KGDB_BAUD` in `lab.local.env` must match the GRUB args +script 01 adds. -Then update `lab.local.env`: +Sanity check before F5: `nc -vz 127.0.0.1 `. -```env -SERVER_KGDB_ENDPOINT=:5520 -``` +--- -Run F5 again after changing `lab.local.env` so -`scripts/04-deploy-debug-vscode.sh` regenerates `.gdb/server-kgdb.gdb` with the -new endpoint. +## Stepping into kernel code -## Start VS Code Debugging +Pass `--debug-symbols` to script 01 (installs the matching `vmlinux` +debug image and the `linux-source-X` package). Script 02 syncs and +extracts both. Script 04 then asks `addr2line` where `start_kernel` +lives in `vmlinux` (always `/init/main.c`) and emits the +corresponding `set substitute-path` into the generated `.gdb` file. -In VS Code Remote-WSL: +That's it — set a breakpoint in `vfs_read` and step through it. -1. Press `Ctrl+Shift+B` and choose the target when prompted. This runs - `Kernel: Build`. -2. Choose `Kernel: Debug server` or `Kernel: Debug desktop`. -3. Put a breakpoint in `module/src/hello.c` or let the module stop at - `kgdb_breakpoint()`. -4. Press F5. +Gracefully degrades: missing `vmlinux` is fatal (script 04 errors with +remediation); missing kernel source prints a one-line note and module +debugging still works (kernel step-into shows disassembly instead of +source). -The build task runs: +--- -```bash -./scripts/03-build-module.sh server -``` +## Undoing target-side changes -The matching debug prelaunch task then runs: +`./scripts/01-provision-target.sh --uninstall` removes every +boot arg the lab added (`nokaslr`, `kgdboc=`, `sysrq_always_enabled=1`, +`maxcpus=`) and re-runs `update-grub`. Installed packages are left in +place. A timestamped GRUB backup is left on the target under +`/etc/default/grub.kmod-debug-lab..bak`. -```bash -./scripts/04-deploy-debug-vscode.sh server -``` - -Script 04 uploads `hello.ko`, removes any old `hello` module, inserts the new -module with a short `debug_delay_ms`, discovers the module section addresses -from `/sys/module/hello/sections/*`, writes `.gdb/server-module-symbols.gdb`, -and generates `.gdb/server-kgdb.gdb`. VS Code then starts GDB and sources that -generated script. - -The expected order is: - -1. VM is running with either the VMware debug stub enabled or the serial bridge - connected. -2. TCP endpoint is reachable from WSL. -3. `Ctrl+Shift+B` builds `hello.ko`. -4. F5 runs the matching deploy prelaunch task. -5. Script 04 loads `hello.ko` and prepares the generated GDB files. -6. VS Code starts GDB and sources the generated script. -7. GDB connects to the configured endpoint while the module is still inside - `debug_delay_ms`. -8. The target should stop at `kgdb_breakpoint()` in `module/src/hello.c`. +--- ## Troubleshooting -If F5 fails with `target remote ... Connection timed out`, the endpoint in -`.gdb/server-kgdb.gdb` is not reachable from WSL. Check that -`SERVER_KGDB_ENDPOINT` or `DESKTOP_KGDB_ENDPOINT` uses the reachable host and -port. For the recommended VMware debug stub path, confirm the `.vmx` debugStub -settings and port. For the serial bridge path, confirm `host/bridge-kgdb.ps1` -is still running. +**`sshpass: command not found`.** `TARGET_SSH_PASS` is set. On the dev +host: `sudo apt-get install -y sshpass`. -If the bridge says the TCP client connected but named-pipe connection fails, -check the VMware serial-port pipe name and make sure the VM is powered on with -the serial port connected at power on. +**F5 fails with `target remote ... Connection timed out`.** Endpoint +not reachable from the dev host. `nc -vz ` to confirm. On +WSL, the Windows-host address is in `/etc/resolv.conf`'s `nameserver` +line. Re-run F5 after fixing `lab.local.env` so 04 regenerates the +`.gdb/` files. -If GDB connects but the module does not load, inspect: +**GDB connects but the module never loads.** Inspect +`.gdb/--loader.log`. Reproduce by hand: +`ssh sudo insmod /tmp/kmod-debug-lab-/.ko` then +`ssh sudo dmesg | tail -40`. -```bash -cat .gdb/server-loader.log -``` +**Breakpoints bind under `build/intermediate/...`.** Re-run F5; 04 +regenerates `.gdb/current-debug.gdb` with the correct +`substitute-path`. Don't edit `.gdb/` by hand. -If breakpoints bind to files under `build/intermediate/...`, the generated GDB -script should map them back to `module/` using `set substitute-path`. Regenerate -the script by pressing F5 again after any script changes. +**VS Code shows include squiggles for `linux/module.h`.** Script 02 +hasn't synced the headers yet, or you switched targets and +IntelliSense is caching old paths. Re-run script 02 and run +`C/C++: Reset IntelliSense Database`. -The module source is self-contained under `module/`: C files in `module/src/`, -headers in `module/include/`, and the module Kbuild fragment in -`module/Kbuild`. The top-level scripts and Makefile are lab/build tooling. +**`/sys/module//sections/.text` didn't appear within 30s.** Your +module failed to load, or it's slow to init. Loader log shows insmod +output. Bump with `INSMOD_WAIT_SECS=60 ./scripts/04-... ...` if needed. -For full kernel symbols, run provisioning with `--debug-symbols`: +**`Cannot access memory at address …` when stepping into the kernel.** +Either vmlinux is missing (re-run `01 --debug-symbols && 02`) or the +cached vmlinux is for a different kernel than the target now runs +(script 04 normally catches this — if you bypassed the guard, re-sync). -```bash -./scripts/01-provision-target.sh desktop --debug-symbols -``` - -Without that flag, provisioning only installs target headers, `rsync`, and KGDB -boot arguments. Module builds still work. +**Spaces in your repo path.** Clone to a path without spaces. GDB's +`set substitute-path FROM TO` splits on whitespace; Kbuild is also +fragile with spaces. -If you use SSH passwords instead of keys, set `DESKTOP_SSH_PASS` or -`SERVER_SSH_PASS` in `lab.local.env` and install `sshpass` in WSL: - -```bash -sudo apt-get install -y sshpass -``` +--- -Leave `_SUDO_PASS` empty when the sudo password is the same as the SSH -password. - -## VS Code Include Errors - -The C/C++ extension reads `.vscode/c_cpp_properties.json`. It expects target -kernel headers under `.kernel-cache//build`, which are created by: - -```bash -./scripts/02-setup-wsl-build.sh desktop -./scripts/02-setup-wsl-build.sh server -``` +## Tool versions -Before that sync runs, VS Code can still show include squiggles for kernel -headers such as `linux/module.h` or `linux/fs.h`. If the squiggles remain after -syncing headers, run `C/C++: Reset IntelliSense Database` from the command -palette and select the matching configuration: `Linux kernel module - desktop` -or `Linux kernel module - server`. +Dev host: bash 5+, GNU coreutils, GDB ≥ 10, gcc ≥ 8 (for +`-ffile-prefix-map`). Target VM: Ubuntu 22.04 / 24.04 (kernel 5.15 / +6.8 tested; CI also exercises ubuntu-24.04 runners with 6.17). macOS +dev hosts need `brew install coreutils` and `make CP=gcp` for the +recursive-symlink staging. -The IntelliSense configuration deliberately mirrors Kbuild's compile context: -GNU C mode, `__KERNEL__`, `MODULE`, `CC_USING_FENTRY`, Ubuntu's kernel include -directory, and the forced kernel headers `compiler-version.h`, `kconfig.h`, and -`compiler_types.h`. Without those forced headers, VS Code parses kernel headers -as ordinary C and reports false errors for `CONFIG_*`, `IS_ENABLED()`, ftrace, -and other kernel-only macros even when `make` builds successfully. +--- -## Reading The Device +## Adding a new target OS -For a quick smoke test after loading the module: - -```bash -head -c 10 /dev/chuck_norise -``` - -Expected output: - -```text -chuck nori -``` - -Use one open file descriptor to see offset-preserving reads: - -```bash -exec 9/dev/null -dd bs=1 count=5 <&9 2>/dev/null -dd bs=1 count=7 <&9 2>/dev/null -exec 9<&- -``` +Only `TARGET_OS[*]=ubuntu` is implemented in scripts 01 and 02. Adding +(say) Fedora means teaching script 01 to install headers / dbgsym / +source via `dnf`, and teaching script 02 to discover the header tree +paths under `/usr/src/`. Both scripts check `target_os` near the top +and fail fast on unknown values — that's where to add a backend. PRs +welcome. -Expected output chunks are `chu`, `ck no`, and `rise!ch`. +--- -## Snapshot Restore +## License -VMware snapshot restore is host-only. Run `host/restore-snapshot.ps1` manually -from Windows PowerShell, not from WSL or VS Code Remote-WSL. +MIT — see [LICENSE](LICENSE). The lab's license does not impose terms +on your module under `module/`. diff --git a/compile_commands.json b/compile_commands.json deleted file mode 100644 index 04ec83f..0000000 --- a/compile_commands.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"hello\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/src/hello.c", - "file": "/home/dor/small-ko/module/src/hello.c" - }, - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"chuck_device\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/src/chuck_device.c", - "file": "/home/dor/small-ko/module/src/chuck_device.c" - }, - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"chuck_message\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/src/chuck_message.c", - "file": "/home/dor/small-ko/module/src/chuck_message.c" - }, - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -x c-header -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"chuck_device\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/include/chuck_device.h", - "file": "/home/dor/small-ko/module/include/chuck_device.h" - }, - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -x c-header -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"chuck_message\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/include/chuck_message.h", - "file": "/home/dor/small-ko/module/include/chuck_message.h" - } -] diff --git a/examples/chuck_norise/Makefile b/examples/chuck_norise/Makefile new file mode 100644 index 0000000..c196f49 --- /dev/null +++ b/examples/chuck_norise/Makefile @@ -0,0 +1,28 @@ +# Standard out-of-tree Linux kernel module Makefile. +# +# Read in two modes: +# 1. As a Kbuild fragment when the kernel build system pulls it in via +# `make -C $KDIR M=$THIS_DIR modules` (KERNELRELEASE is set; Kbuild +# assignments run, wrapper recipe is skipped). This is the mode the +# lab's outer build uses. +# 2. As a standalone Makefile when you run `make` in this dir directly +# (KERNELRELEASE is unset; the wrapper recipe re-invokes mode 1 +# against the host kernel). + +obj-m += chuck_norise.o +chuck_norise-y := src/hello.o src/chuck_device.o src/chuck_message.o + +ccflags-y := -I$(src)/include -g -DDEBUG + +ifndef KERNELRELEASE +KDIR ?= /lib/modules/$(shell uname -r)/build +PWD := $(CURDIR) + +.PHONY: default clean + +default: + $(MAKE) -C $(KDIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(KDIR) M=$(PWD) clean +endif diff --git a/examples/chuck_norise/README.md b/examples/chuck_norise/README.md new file mode 100644 index 0000000..c3f8e7b --- /dev/null +++ b/examples/chuck_norise/README.md @@ -0,0 +1,38 @@ +# chuck_norise — example module + +Worked example for the lab. Builds as `chuck_norise.ko` and exposes +`/dev/chuck_norise`. Reads return `chuck norise!` repeated, preserving +the file offset. + +## Run it through the lab + +From the repo root: + +```bash +make use-example NAME=chuck_norise +``` + +Then follow the [top-level Quickstart](../../README.md#quickstart). After +the module loads on the target VM: + +```bash +head -c 10 /dev/chuck_norise # chuck nori + +# offset-preserving reads from a single open fd +exec 9/dev/null # chu +dd bs=1 count=5 <&9 2>/dev/null # ck no +dd bs=1 count=7 <&9 2>/dev/null # rise!ch +exec 9<&- +``` + +## What this example shows + +- The conventional out-of-tree layout (`src/`, `include/`, `Makefile`). +- A `Makefile` that doubles as a Kbuild fragment AND a standalone + wrapper (`ifndef KERNELRELEASE` clause). +- A `debug_delay_ms` module parameter so the host has time to attach + GDB before init runs. +- A `LINUX_VERSION_CODE` shim for the 6.4 `class_create()` signature + change — a real-world example of the compat carrying out-of-tree + modules have to do. diff --git a/module/include/chuck_device.h b/examples/chuck_norise/include/chuck_device.h similarity index 100% rename from module/include/chuck_device.h rename to examples/chuck_norise/include/chuck_device.h diff --git a/module/include/chuck_message.h b/examples/chuck_norise/include/chuck_message.h similarity index 100% rename from module/include/chuck_message.h rename to examples/chuck_norise/include/chuck_message.h diff --git a/module/src/chuck_device.c b/examples/chuck_norise/src/chuck_device.c similarity index 91% rename from module/src/chuck_device.c rename to examples/chuck_norise/src/chuck_device.c index 4705ac9..d0d3dcf 100644 --- a/module/src/chuck_device.c +++ b/examples/chuck_norise/src/chuck_device.c @@ -29,11 +29,14 @@ static ssize_t chuck_read(struct file *file, char __user *buffer, size_t count, return chuck_message_read(buffer, count, position); } +/* .llseek deliberately unset — kernel installs default_llseek for char + * devices, which matches chuck_read's behavior (it respects *position). + * Don't reach for no_llseek; it was removed in 6.12 (commit 868941b14441). + */ static const struct file_operations chuck_fops = { .owner = THIS_MODULE, .open = chuck_open, .read = chuck_read, - .llseek = no_llseek, }; static struct class *chuck_class_create(void) diff --git a/module/src/chuck_message.c b/examples/chuck_norise/src/chuck_message.c similarity index 100% rename from module/src/chuck_message.c rename to examples/chuck_norise/src/chuck_message.c diff --git a/module/src/hello.c b/examples/chuck_norise/src/hello.c similarity index 66% rename from module/src/hello.c rename to examples/chuck_norise/src/hello.c index 91942fb..d29c98e 100644 --- a/module/src/hello.c +++ b/examples/chuck_norise/src/hello.c @@ -2,14 +2,12 @@ #include "chuck_message.h" #include -#include -#include #include static unsigned int debug_delay_ms = 5000; module_param(debug_delay_ms, uint, 0644); MODULE_PARM_DESC(debug_delay_ms, - "Milliseconds to wait before kgdb_breakpoint so symbols can load"); + "Milliseconds to sleep at the start of init so the host can attach GDB and set breakpoints"); static int hello_init(void) { @@ -18,12 +16,6 @@ static int hello_init(void) if (debug_delay_ms) msleep(debug_delay_ms); -#if IS_ENABLED(CONFIG_KGDB) - kgdb_breakpoint(); -#else - pr_warn("hello: CONFIG_KGDB is disabled; continuing without breakpoint\n"); -#endif - ret = chuck_device_register(); if (ret) { pr_err("hello: failed to register /dev/%s: %d\n", @@ -46,5 +38,5 @@ module_init(hello_init); module_exit(hello_exit); MODULE_LICENSE("GPL"); -MODULE_AUTHOR("small-ko lab"); -MODULE_DESCRIPTION("Multi-file char device module for KGDB source debugging"); +MODULE_AUTHOR("Linux Kernel Module Debug Lab"); +MODULE_DESCRIPTION("Multi-file char device example for kernel source debugging"); diff --git a/host/bridge-kgdb.ps1 b/host/bridge-kgdb.ps1 index fb63d39..0b45997 100644 --- a/host/bridge-kgdb.ps1 +++ b/host/bridge-kgdb.ps1 @@ -1,11 +1,27 @@ +# Generic VMware-Workstation-on-Windows bridge from a guest serial port (exposed +# as a Windows named pipe) to a TCP listener that GDB connects to. +# +# This is the Windows-host side of the `kgdb` debug method when the hypervisor +# is VMware Workstation. Configure a serial port in the VM as: +# Use named pipe = \\.\pipe\ +# This end is the server +# The other end is an application +# Connect at power on +# +# Then run this script on the Windows host whenever you want to debug: +# .\bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520 +# +# The lab does not bundle target-specific defaults here on purpose; pipe names +# and ports are user choices that should live in your own wrapper, in your VM +# settings, or in your shell history. Match TARGET_DEBUG_ENDPOINT_KGDB[] +# in lab.local.env to the -ListenAddress:-Port you pass here. + param( [Parameter(Mandatory = $true)] - [ValidateSet("desktop", "server")] - [string]$Target, - [string]$PipeName, - [int]$Port = 0, + [Parameter(Mandatory = $true)] + [int]$Port, [string]$ListenAddress = "127.0.0.1", @@ -14,25 +30,6 @@ param( $ErrorActionPreference = "Stop" -$Defaults = @{ - desktop = @{ - PipeName = "kgdb-desktop" - Port = 5510 - } - server = @{ - PipeName = "kgdb-server" - Port = 5520 - } -} - -if (-not $PipeName) { - $PipeName = $Defaults[$Target].PipeName -} - -if ($Port -eq 0) { - $Port = $Defaults[$Target].Port -} - if ($PipeName -match '^\\\\[^\\]+\\pipe\\(.+)$') { $PipeName = $Matches[1] } @@ -49,7 +46,7 @@ function Close-Quietly($Resource) { } } -Write-Host "Starting KGDB bridge for $Target" +Write-Host "Starting KGDB bridge" Write-Host " TCP: ${ListenAddress}:${Port}" Write-Host " Pipe: \\.\pipe\$PipeName" Write-Host "Press Ctrl+C to stop." diff --git a/host/restore-snapshot.ps1 b/host/restore-snapshot.ps1 index b614803..a4b4605 100644 --- a/host/restore-snapshot.ps1 +++ b/host/restore-snapshot.ps1 @@ -1,62 +1,54 @@ +# Generic helper for reverting a VMware Workstation VM to a named snapshot +# before each debug session, so each run starts from a known-clean state. +# +# This is optional and intended as a starting point. Wire it into your own +# pre-debug routine if you find it useful. +# +# Example: +# .\restore-snapshot.ps1 -VmxPath 'C:\VMs\target\target.vmx' ` +# -Snapshot clean-debug ` +# -StartAfterRestore + param( [Parameter(Mandatory = $true)] - [ValidateSet("desktop", "server")] - [string]$Target, + [string]$VmxPath, + + [Parameter(Mandatory = $true)] + [string]$Snapshot, + + [string]$VmrunPath = "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe", [switch]$StartAfterRestore ) $ErrorActionPreference = "Stop" -$Vmrun = "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe" - -$Targets = @{ - desktop = @{ - VmxPath = "C:\Users\dor\Documents\Virtual Machines\Ubuntu-Main\Ubuntu-Main.vmx" - Snapshot = "clean-debug" - } - server = @{ - VmxPath = "C:\Users\dor\Documents\Virtual Machines\ubuntu-server\ubuntu-server.vmx" - Snapshot = "clean-debug" - } -} - -if (-not (Test-Path -LiteralPath $Vmrun)) { - throw "vmrun.exe was not found at '$Vmrun'" -} - -$Config = $Targets[$Target] -if (-not $Config) { - throw "Unknown target '$Target'" +if (-not (Test-Path -LiteralPath $VmrunPath)) { + throw "vmrun.exe was not found at '$VmrunPath'" } -if ($Config.VmxPath -like "C:\Path\To\*") { - throw "Edit host/restore-snapshot.ps1 and set the real VMX path for '$Target'" +if (-not (Test-Path -LiteralPath $VmxPath)) { + throw "VMX path does not exist: '$VmxPath'" } -if (-not (Test-Path -LiteralPath $Config.VmxPath)) { - throw "VMX path does not exist: '$($Config.VmxPath)'" -} - -Write-Host "Stopping $Target if it is running..." -& $Vmrun -T ws stop $Config.VmxPath soft +Write-Host "Stopping VM if it is running..." +& $VmrunPath -T ws stop $VmxPath soft if ($LASTEXITCODE -ne 0) { Write-Host "Soft stop failed or VM was not running; continuing to snapshot revert." } -Write-Host "Reverting $Target to snapshot '$($Config.Snapshot)'..." -& $Vmrun -T ws revertToSnapshot $Config.VmxPath $Config.Snapshot +Write-Host "Reverting VM to snapshot '$Snapshot'..." +& $VmrunPath -T ws revertToSnapshot $VmxPath $Snapshot if ($LASTEXITCODE -ne 0) { throw "Snapshot revert failed with exit code $LASTEXITCODE" } if ($StartAfterRestore) { - Write-Host "Starting $Target..." - & $Vmrun -T ws start $Config.VmxPath + Write-Host "Starting VM..." + & $VmrunPath -T ws start $VmxPath if ($LASTEXITCODE -ne 0) { throw "VM start failed with exit code $LASTEXITCODE" } } -Write-Host "Snapshot restore complete for $Target." - +Write-Host "Snapshot restore complete." diff --git a/lab.example.env b/lab.example.env index 1e96ffc..498bfd6 100644 --- a/lab.example.env +++ b/lab.example.env @@ -1,41 +1,118 @@ -# Copy this file to lab.local.env and edit the values for your machines. -# Supported targets are exactly: desktop, server. +# Copy this file to lab.local.env and edit it for your machines. +# lab.local.env is gitignored; lab.example.env is the only tracked copy. -# Common local tools. -GDB_BIN=gdb +# Local tools on the Debian-based development host. The defaults below +# expect each to be on $PATH; override here only if you need an absolute +# path or an alternate binary. SSH_BIN=ssh SCP_BIN=scp RSYNC_BIN=rsync SSHPASS_BIN=sshpass -# GDB connects to these endpoints from WSL. If VMware exposes KGDB through a -# Windows named pipe, bridge that pipe to TCP on the host and put the bridge -# endpoint here. -DESKTOP_KGDB_ENDPOINT=127.0.0.1:5510 -SERVER_KGDB_ENDPOINT=127.0.0.1:5520 - -# SSH details for the Ubuntu 24.04 Desktop VM. -DESKTOP_SSH_HOST=ubuntu-desktop.local -DESKTOP_SSH_PORT=22 -DESKTOP_SSH_USER=user -DESKTOP_SSH_PASS= -# Leave empty to reuse DESKTOP_SSH_PASS for sudo. -DESKTOP_SUDO_PASS= -DESKTOP_REMOTE_DIR=/tmp/small-ko -DESKTOP_KGDB_TTY=ttyS0 -DESKTOP_KGDB_BAUD=115200 - -# SSH details for the Ubuntu 24.04 Server VM. -SERVER_SSH_HOST=ubuntu-server.local -SERVER_SSH_PORT=22 -SERVER_SSH_USER=user -SERVER_SSH_PASS= -# Leave empty to reuse SERVER_SSH_PASS for sudo. -SERVER_SUDO_PASS= -SERVER_REMOTE_DIR=/tmp/small-ko -SERVER_KGDB_TTY=ttyS0 -SERVER_KGDB_BAUD=115200 - -# The module waits briefly before kgdb_breakpoint so the deploy script can read -# /sys/module/hello/sections/* and prepare module symbols for VS Code. +# Target profiles. These names are data, not variable prefixes — every +# TARGET_* map below is keyed by these names. Add a target by adding entries +# to TARGETS and to every map you care about for that target. +# +# Pick whatever names you like ([letters][letters digits _ -]*). The VS Code +# task pickers accept any name you type; there is no per-target editing +# required there. +TARGETS=(desktop server) + +# Only ubuntu targets are implemented for now. Add a new target OS backend in +# the scripts before using another value here. +declare -A TARGET_OS=( + [desktop]=ubuntu + [server]=ubuntu +) + +# SSH connection to each target VM. +declare -A TARGET_SSH_HOST=( + [desktop]=ubuntu-desktop.local + [server]=ubuntu-server.local +) + +declare -A TARGET_SSH_PORT=( + [desktop]=22 + [server]=22 +) + +declare -A TARGET_SSH_USER=( + [desktop]=user + [server]=user +) + +# SSH password. Leave empty when using SSH keys (recommended for anything +# beyond throwaway dev VMs). The lab pipes any configured password into ssh +# via SSHPASS; it is not written to disk and not visible in `ps` on either +# end. +declare -A TARGET_SSH_PASS=( + [desktop]= + [server]= +) + +# Sudo password. Leave empty to reuse TARGET_SSH_PASS. The lab pipes this +# into `sudo -S` over ssh's stdin, so it is not visible in `ps` on the +# target. For long-lived dev targets, configure NOPASSWD for the lab user +# (` ALL=(ALL) NOPASSWD: ALL` in sudoers) and leave this empty. +declare -A TARGET_SUDO_PASS=( + [desktop]= + [server]= +) + +# Remote staging directory used for uploading the built module. Any writable +# path is fine; /tmp/* gets cleaned on reboot which is usually what you want. +# Leave empty to use the per-user default `/tmp/kmod-debug-lab-`, +# which avoids collisions when multiple devs share a target host. +declare -A TARGET_REMOTE_DIR=( + [desktop]= + [server]= +) + +# Debug endpoints — one per debug method. Scripts 01 and 04 take a +# argument and pick the matching endpoint from these maps. +# Either map may be left empty per target if you do not use that method. + +# KGDB-over-serial: a TCP bridge to the target's serial port (kgdboc=ttyS*). +# Typically opened by VMware's named-pipe-to-TCP bridge, socat, or QEMU's +# `-serial tcp:HOST:PORT,server,nowait`. +declare -A TARGET_DEBUG_ENDPOINT_KGDB=( + [desktop]=127.0.0.1:5510 + [server]=127.0.0.1:5520 +) + +# QEMU's built-in gdbstub: typically opened by QEMU's `-gdb tcp::PORT` (or +# `-s`). VMware's `debugStub.listen.guest64` is the same shape and goes here +# too. Does not require kgdb in the guest. +declare -A TARGET_DEBUG_ENDPOINT_QEMU=( + [desktop]=127.0.0.1:1234 + [server]=127.0.0.1:1234 +) + +# Serial KGDB settings — only consulted when script 01 is run with `kgdb`. +declare -A TARGET_KGDB_TTY=( + [desktop]=ttyS0 + [server]=ttyS0 +) + +declare -A TARGET_KGDB_BAUD=( + [desktop]=115200 + [server]=115200 +) + +# When your module declares `module_param(debug_delay_ms, uint, 0644);`, +# scripts/04 passes this value (in milliseconds) on `insmod`. Your +# `module_init` should sleep that long so the host can read +# /sys/module//sections/* and you have a window to break in GDB +# (Ctrl+C against the qemu stub, sysrq+g for kgdb) before init runs. +# Modules that do not declare the parameter get loaded plainly; no harm. DEBUG_LOAD_DELAY_MS=5000 + +# Optional. Sets `maxcpus=` on the target's GRUB command line — useful +# when you want to pin debugging to a known CPU count (e.g. 1 to avoid +# CPU migration confusing breakpoints). Unset to leave it off. +#DEBUG_MAXCPUS=1 + +# Optional. Override how long scripts/04 polls for the loaded module's +# section addresses to appear under /sys/module//sections/. +# Increase on slow target VMs. Default 30. +#INSMOD_WAIT_SECS=30 diff --git a/module/Kbuild b/module/Kbuild deleted file mode 100644 index a49ea0a..0000000 --- a/module/Kbuild +++ /dev/null @@ -1,6 +0,0 @@ -MODULE_NAME ?= hello - -obj-m += $(MODULE_NAME).o -$(MODULE_NAME)-y := src/hello.o src/chuck_device.o src/chuck_message.o - -ccflags-y := -I$(src)/include -g -DDEBUG diff --git a/module/README.md b/module/README.md new file mode 100644 index 0000000..397d41e --- /dev/null +++ b/module/README.md @@ -0,0 +1,27 @@ +# module/ — your kernel module goes here + +The lab's only slot for the module being built and debugged. The repo is +otherwise generic — it doesn't know your module's name, source layout, +or what it does. + +**Contract:** see the [top-level README](../README.md#the-module-contract). +TL;DR: a standard out-of-tree `Makefile` (or `Kbuild`) with one `obj-m` +entry; sources anywhere you like; an optional `debug_delay_ms` module +parameter so the host has time to attach GDB. + +## Try the example + +```bash +make use-example NAME=chuck_norise # copy examples/chuck_norise/ into module/ +make clean-module # reset module/ to just this README +``` + +## One caveat: no symlinks inside `module/` + +The lab stages your sources into `build/intermediate///` +as a tree of absolute symlinks (`cp -as`) and runs Kbuild there. If +`module/` contains its OWN symlinks, the staged copy ends up with +symlinks-to-symlinks; the DWARF prefix-map remap (which assumes sources +live under `module/`) won't produce the paths your IDE expects. Keep +`module/` symlink-free, or copy in real files for external sources you +need. diff --git a/scripts/00-check-target.sh b/scripts/00-check-target.sh new file mode 100755 index 0000000..23551d5 --- /dev/null +++ b/scripts/00-check-target.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Non-mutating health check for a target. Run before scripts/01 to catch +# configuration mistakes before any state on the target changes. +# +# Reports ssh / sudo / os / kernel / headers / vmlinux / kernel-source / +# debug endpoint. The debug-method arg is optional — both kgdb and qemu +# endpoints are checked when omitted. +set -euo pipefail + +usage() { + echo "usage: $0 [kgdb|qemu]" >&2 + exit 2 +} + +target="${1:-}" +debug_method="${2:-}" +case "$debug_method" in ""|kgdb|qemu) ;; *) usage ;; esac + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 [kgdb|qemu]" + +env_file="$(lab_env_file "$repo_root")" +# shellcheck source=/dev/null +source "$env_file" +lab_load_target "$target" + +# Colored OK / WARN / FAIL on a TTY; plain text when piped. +if [[ -t 1 ]]; then + GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[0;33m'; RESET='\033[0m' +else + GREEN=''; RED=''; YELLOW=''; RESET='' +fi +ok() { printf " ${GREEN}OK${RESET} %s\n" "$1"; } +fail() { printf " ${RED}FAIL${RESET} %s\n" "$1"; failures=$((failures+1)); } +warn() { printf " ${YELLOW}WARN${RESET} %s\n" "$1"; } +failures=0 + +echo "checking $LAB_TARGET ($LAB_SSH_TARGET:$LAB_SSH_PORT)" + +# --- ssh --- +if lab_ssh 'true' 2>/dev/null; then + ok "ssh: reachable" +else + fail "ssh: cannot reach $LAB_SSH_TARGET on port $LAB_SSH_PORT" + echo + echo "fix one of:" + echo " - TARGET_SSH_HOST/PORT/USER for '$target' in lab.local.env" + echo " - the target VM is up and openssh-server is running" + echo " - if password auth: TARGET_SSH_PASS[$target] is set" + exit 1 +fi + +# --- sudo --- +if lab_ssh_sudo 'true' 2>/dev/null; then + ok "sudo: works" +else + fail "sudo: configured password rejected or sudo not installed" + echo + echo "fix one of:" + echo " - TARGET_SUDO_PASS[$target] in lab.local.env (defaults to TARGET_SSH_PASS)" + echo " - configure NOPASSWD for the lab user on the target" +fi + +# --- os --- +# shellcheck disable=SC2016 # $ID and $VERSION_CODENAME are intentionally evaluated on the remote. +remote_os="$(lab_ssh '. /etc/os-release 2>/dev/null && printf "%s %s" "$ID" "${VERSION_CODENAME:-unknown}"' || echo unknown)" +case "$remote_os" in + "ubuntu "*) ok "os: $remote_os" ;; + *) fail "os: $remote_os (only ubuntu targets are implemented)" ;; +esac + +# --- kernel --- +live_kernel="$(lab_ssh 'uname -r' 2>/dev/null || echo unknown)" +ok "kernel: $live_kernel" + +cache_dir="$repo_root/.kernel-cache/$target" +cached_kernel="$(cat "$cache_dir/kernel.release" 2>/dev/null || echo)" +if [[ -n "$cached_kernel" && "$cached_kernel" != "$live_kernel" ]]; then + warn "cached kernel ($cached_kernel) differs from live; re-run scripts/02-setup-host-build.sh $target" +fi + +# --- headers --- +if lab_ssh "test -d /lib/modules/$live_kernel/build" 2>/dev/null; then + ok "headers: /lib/modules/$live_kernel/build present" +else + warn "headers: missing on target; run scripts/01-provision-target.sh $target " +fi + +# --- vmlinux (debug image) --- +if lab_ssh_sudo "test -r /usr/lib/debug/boot/vmlinux-$live_kernel" 2>/dev/null; then + ok "vmlinux: /usr/lib/debug/boot/vmlinux-$live_kernel readable (debug symbols installed)" +else + warn "vmlinux: missing on target; for source debugging, re-run scripts/01-... with --debug-symbols" +fi + +# --- kernel source --- +src_found="$(lab_ssh 'ls -1d /usr/src/linux-source-*/ 2>/dev/null | head -1 || true' 2>/dev/null)" +if [[ -n "$src_found" ]]; then + ok "kernel-source: ${src_found%/} present (step-into-kernel will resolve source)" +else + warn "kernel-source: missing on target; for step-into-kernel, re-run scripts/01-... with --debug-symbols" +fi + +# --- debug endpoint(s) reachable from this host --- +check_endpoint() { + local label="$1" key="$2" + local endpoint + endpoint="$(target_cfg "$target" "$key")" + if [[ -z "$endpoint" ]]; then + warn "$label: TARGET_${key}[$target] is unset (skip if you don't use this method)" + return + fi + local host="${endpoint%:*}" port="${endpoint##*:}" + if command -v nc >/dev/null 2>&1; then + if nc -z -w 3 "$host" "$port" 2>/dev/null; then + ok "$label: $endpoint reachable from this host" + else + warn "$label: $endpoint NOT reachable from this host (start your GDB stub / serial bridge)" + fi + else + warn "$label: cannot test (nc not installed); endpoint configured as $endpoint" + fi +} + +case "$debug_method" in + kgdb) check_endpoint "debug-kgdb" DEBUG_ENDPOINT_KGDB ;; + qemu) check_endpoint "debug-qemu" DEBUG_ENDPOINT_QEMU ;; + "") + check_endpoint "debug-kgdb" DEBUG_ENDPOINT_KGDB + check_endpoint "debug-qemu" DEBUG_ENDPOINT_QEMU + ;; +esac + +echo +if [[ $failures -eq 0 ]]; then + echo "all checks passed." + echo "next: scripts/01-provision-target.sh $target [--debug-symbols]" +else + echo "$failures check(s) FAILED; resolve before running scripts/01." + exit 1 +fi diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index ff34a16..6802ed7 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -1,122 +1,224 @@ #!/usr/bin/env bash +# Provision a target VM for kernel module debugging. +# +# Installs kernel headers (so the host build can sync them), optionally the +# matching vmlinux debug image + kernel source for step-into-kernel, and +# adds the boot args this lab needs to the guest's GRUB command line: +# - nokaslr (both methods; lets vmlinux symbols line up) +# - kgdboc + sysrq (kgdb method only) +# - maxcpus (only if DEBUG_MAXCPUS is set in the env) +# +# `--uninstall` removes every boot arg the lab added and leaves installed +# packages alone. Reboot the target after either mode for boot args to +# take effect — this script never reboots on its own. set -euo pipefail +usage() { + echo "usage: $0 [--debug-symbols]" >&2 + echo " $0 --uninstall" >&2 + exit 2 +} + target="${1:-}" -symbols="${2:-}" -case "$target" in desktop|server) ;; *) echo "usage: $0 [--debug-symbols]" >&2; exit 2 ;; esac -case "$symbols" in ""|--debug-symbols) ;; *) echo "usage: $0 [--debug-symbols]" >&2; exit 2 ;; esac +arg2="${2:-}" +arg3="${3:-}" + +uninstall=0 +debug_method="" +symbols="" +case "$arg2" in + --uninstall) + uninstall=1 + [[ -z "$arg3" ]] || usage + ;; + kgdb|qemu) + debug_method="$arg2" + case "$arg3" in + ""|--debug-symbols) symbols="$arg3" ;; + *) usage ;; + esac + ;; + *) usage ;; +esac repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -env_file="$repo_root/lab.local.env" -[[ -f "$env_file" ]] || { echo "missing $env_file; copy lab.example.env first" >&2; exit 1; } +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 [--debug-symbols]" -set -a +env_file="$(lab_env_file "$repo_root")" # shellcheck source=/dev/null source "$env_file" -set +a - -prefix="${target^^}" -cfg() { local name="${prefix}_$1"; printf '%s' "${!name:-}"; } - -ssh_bin="${SSH_BIN:-ssh}" -sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(cfg SSH_HOST)" -ssh_port="$(cfg SSH_PORT)" -ssh_user="$(cfg SSH_USER)" -ssh_pass="$(cfg SSH_PASS)" -sudo_pass="$(cfg SUDO_PASS)" -kgdb_tty="$(cfg KGDB_TTY)" -kgdb_baud="$(cfg KGDB_BAUD)" - -[[ -n "$ssh_host" && -n "$ssh_user" ]] || { - echo "missing ${prefix}_SSH_HOST or ${prefix}_SSH_USER in lab.local.env" >&2 - exit 1 -} +lab_load_target "$target" -ssh_port="${ssh_port:-22}" +target_os="$(target_cfg "$target" OS)" +kgdb_tty="$(target_cfg "$target" KGDB_TTY)" +kgdb_baud="$(target_cfg "$target" KGDB_BAUD)" +target_os="${target_os:-ubuntu}" kgdb_tty="${kgdb_tty:-ttyS0}" kgdb_baud="${kgdb_baud:-115200}" -sudo_pass="${sudo_pass:-$ssh_pass}" -sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" -if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it in WSL: sudo apt-get install -y sshpass" >&2 - exit 1 -fi +case "$target_os" in + ubuntu) ;; + *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; +esac -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port") -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") -fi +# Skip the sudo check — the user might be running 01 *to* configure sudo. +# The provisioning shell below will exercise it and fail with apt's own +# diagnostics if the password is wrong. +lab_check_connection --no-sudo -echo "provisioning $target at $ssh_user@$ssh_host:$ssh_port" +if (( uninstall )); then + echo "uninstalling lab GRUB args from $target ($LAB_SSH_TARGET:$LAB_SSH_PORT)" +else + symbols_blurb="${symbols:+, with debug symbols}" + echo "provisioning $target ($LAB_SSH_TARGET:$LAB_SSH_PORT) for debug method: $debug_method$symbols_blurb" +fi -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" -t "$ssh_user@$ssh_host" \ - "KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$sudo_pass_b64' bash -s" <<'REMOTE' +# We run the remote script as root via `sudo -A bash -s` so stdin stays a +# clean pipe of env-var-assignments + script body for bash to read. The +# straightforward "pipe password to sudo -S" pattern doesn't work here +# because sudo skips the stdin read whenever policy doesn't require auth +# (NOPASSWD, cached creds), and the password line then leaks into bash as +# its first command. SUDO_ASKPASS routes the password through a helper +# script so stdin is never contended. +askpass="$LAB_REMOTE_DIR/.kmod-askpass-$$" +pwfile="$LAB_REMOTE_DIR/.kmod-sudo-pw-$$" +trap 'lab_ssh "rm -f $askpass $pwfile 2>/dev/null" || true' EXIT + +lab_ssh "mkdir -p '$LAB_REMOTE_DIR'; umask 077; cat > '$pwfile'" <<<"$LAB_SUDO_PASS" +lab_ssh "umask 077; cat > '$askpass'; chmod 700 '$askpass'" <&2; exit 1; } -sudo_auth +[[ "${TARGET_OS:-ubuntu}" == "ubuntu" ]] || + { echo "unsupported target OS: ${TARGET_OS:-}" >&2; exit 1; } +[[ "${ID:-}" == "ubuntu" ]] || + { echo "target is not Ubuntu (os-release ID=$ID); only ubuntu targets are implemented" >&2; exit 1; } -echo "installing packages for running kernel $kernel" -apt_retry update -apt_retry install -y linux-headers-"$kernel" rsync -[[ -d "/lib/modules/$kernel/build" ]] || { - echo "missing /lib/modules/$kernel/build after installing headers" >&2 - exit 1 +grub_file=/etc/default/grub +# Lab-managed boot args; stripped on every run so reruns replace rather +# than append, and `--uninstall` cleans them all out. +grub_managed_keys='kgdboc=|maxcpus=|sysrq_always_enabled=|nokaslr' + +read_current_cmdline() { + local raw stripped + raw="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" + stripped="$(printf '%s' "$raw" | sed -E "s/(^| )($grub_managed_keys)([^ ]*)?/ /g; s/ */ /g; s/^ +//; s/ +$//")" + printf '%s' "$stripped" } -vmlinux="/usr/lib/debug/boot/vmlinux-${kernel}" -if [[ "$INSTALL_DEBUG_SYMBOLS" == "--debug-symbols" && ! -r "$vmlinux" ]]; then - apt_retry install -y ubuntu-dbgsym-keyring - cat </dev/null -deb http://ddebs.ubuntu.com ${codename} main restricted universe multiverse +write_cmdline() { + cp "$grub_file" "$grub_file.kmod-debug-lab.$(date +%Y%m%d%H%M%S).bak" + if grep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' "$grub_file"; then + sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$1\"|" "$grub_file" + else + printf 'GRUB_CMDLINE_LINUX_DEFAULT="%s"\n' "$1" >> "$grub_file" + fi + update-grub +} + +if [[ "${UNINSTALL:-0}" == "1" ]]; then + echo "stripping lab boot args from $grub_file" + current="$(read_current_cmdline)" + write_cmdline "$current" + echo + echo "done. reboot to drop the lab boot args." + echo " boot args now: $current" + exit 0 +fi + +echo "[1/3] installing kernel headers for running kernel $kernel" +apt_retry update +apt_retry install -y "linux-headers-$kernel" rsync +[[ -d "/lib/modules/$kernel/build" ]] || + { echo "missing /lib/modules/$kernel/build after installing headers; check apt output above" >&2; exit 1; } + +if [[ "$INSTALL_DEBUG_SYMBOLS" == "--debug-symbols" ]]; then + echo "[2/3] installing vmlinux debug image + kernel source for step-into-kernel" + vmlinux="/usr/lib/debug/boot/vmlinux-$kernel" + if [[ ! -r "$vmlinux" ]]; then + apt_retry install -y ubuntu-dbgsym-keyring + cat > /etc/apt/sources.list.d/ddebs.list <&2 - echo "retry later if ddebs.ubuntu.com is returning 503" >&2 - exit 1 - } +DDEBS + apt-get clean + apt_retry update + apt_retry install -y "linux-image-${kernel}-dbgsym" || + apt_retry install -y "linux-image-unsigned-${kernel}-dbgsym" || + { echo "failed to install dbgsym for $kernel; retry later if ddebs.ubuntu.com is returning 503" >&2; exit 1; } + fi + [[ -r "$vmlinux" ]] || echo "warning: $vmlinux still missing after install; kernel source debugging will be unavailable" + + # Try the major.minor.patch-suffixed package first (linux-source-6.8.0), + # falling back to the meta-package. We don't extract here — script 02 + # does it host-side to avoid needing tar on the target. + short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" + src_pkg="" + for candidate in "linux-source-$short_kver" "linux-source"; do + [[ -z "$candidate" || "$candidate" == "linux-source-" ]] && continue + apt_retry install -y "$candidate" && { src_pkg="$candidate"; break; } + done + if [[ -z "$src_pkg" ]]; then + echo "warning: could not install a linux-source package; step-into-kernel will only show disassembly" >&2 + elif ls -1 /usr/src/linux-source-*.tar.* /usr/src/linux-source-*/linux-source-*.tar.* 2>/dev/null | head -1 >/dev/null; then + echo " kernel source tarball is in /usr/src; script 02 will sync and extract it" + else + echo "warning: $src_pkg installed but no linux-source tarball found under /usr/src/" >&2 + fi +else + echo "[2/3] skipping debug symbols (pass --debug-symbols to enable step-into-kernel)" fi -[[ -r "$vmlinux" ]] || echo "note: $vmlinux is missing; F5 KGDB source debugging may need it" -grub_file=/etc/default/grub -current="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" -for arg in "kgdboc=${KGDB_TTY},${KGDB_BAUD}" "nokaslr" "sysrq_always_enabled=1"; do +echo "[3/3] updating GRUB command line" +current="$(read_current_cmdline)" + +args=("nokaslr") +[[ "$DEBUG_METHOD" == "kgdb" ]] && args+=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "sysrq_always_enabled=1") +[[ -n "${DEBUG_MAXCPUS:-}" ]] && args+=("maxcpus=${DEBUG_MAXCPUS}") +for arg in "${args[@]}"; do case " $current " in *" $arg "*) ;; *) current="${current:+$current }$arg" ;; esac done -sudo_run cp "$grub_file" "$grub_file.small-ko.$(date +%Y%m%d%H%M%S).bak" -if grep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' "$grub_file"; then - sudo_run sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$current\"|" "$grub_file" +write_cmdline "$current" + +echo +echo "done. reboot the target VM before debugging." +echo " boot args: $current" +REMOTE_SCRIPT +} | SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "SUDO_ASKPASS='$askpass' sudo -A bash -s" + +echo +if (( uninstall )); then + echo "uninstall complete. reboot the $target VM to drop the lab boot args." else - echo "GRUB_CMDLINE_LINUX_DEFAULT=\"$current\"" | sudo_run tee -a "$grub_file" >/dev/null + echo "provisioning complete. reboot the $target VM, then:" + echo "next: scripts/02-setup-host-build.sh $target" fi -sudo_run update-grub - -echo "done. reboot the VM before debugging. boot args: $current" -REMOTE diff --git a/scripts/02-setup-host-build.sh b/scripts/02-setup-host-build.sh new file mode 100755 index 0000000..c064a41 --- /dev/null +++ b/scripts/02-setup-host-build.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +# Sync the target's kernel build tree to the host so an out-of-tree module +# can be cross-built against the target's exact kernel. +# +# Populates .kernel-cache// with: +# build/ -> synced linux-headers tree +# source/ -> synced linux-source tree (if --debug-symbols was used in 01) +# vmlinux -> debug-symbol vmlinux from the target (if available) +# kernel.release -> running kernel release +# remote.{build,header}.* -> bookkeeping +# Also updates .kernel-cache/current -> so plain `make` and the +# IntelliSense config track the most recently synced target. +set -euo pipefail + +target="${1:-}" + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 " +require_debian_host + +env_file="$(lab_env_file "$repo_root")" +# shellcheck source=/dev/null +source "$env_file" +lab_load_target "$target" + +target_os="$(target_cfg "$target" OS)" +target_os="${target_os:-ubuntu}" +case "$target_os" in + ubuntu) ;; + *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; +esac + +# --- Install dev-host build prerequisites --------------------------------- + +apt_get() { + sudo apt-get \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=20 \ + -o Acquire::https::Timeout=20 \ + "$@" +} + +echo "[1/5] installing host build prerequisites" +apt_get update +apt_get install -y \ + bc bison build-essential dwarves flex gdb \ + libelf-dev libssl-dev openssh-client rsync sshpass + +# --- Discover the target's kernel + header layout ------------------------- + +echo "[2/5] discovering target kernel" +lab_check_connection +kernel="$(lab_ssh 'uname -r')" +echo " target $target is running $kernel" + +cache_dir="$repo_root/.kernel-cache/$target" +usr_src_dir="$cache_dir/usr-src" +build_dir="$cache_dir/build" +remote_vmlinux="/usr/lib/debug/boot/vmlinux-$kernel" + +mkdir -p "$cache_dir" "$usr_src_dir" + +build_real="$(lab_ssh "readlink -f '/lib/modules/$kernel/build'")" +if ! lab_ssh "test -d '$build_real'"; then + die "missing target build directory $build_real; +- on the target, check: ls -la /lib/modules/$kernel/build +- if it's missing, re-run: scripts/01-provision-target.sh $target " +fi + +mapfile -t remote_header_dirs < <(lab_ssh "bash -s" < ${dest#"$repo_root"/}" + # Preserve symlinks inside Ubuntu's header trees (e.g. dangling rust + # links — harmless for external C builds). + lab_rsync_from "$remote_dir/" "$dest/" --delete +} + +sync_source_path() { + local remote_path="$1" kind="$2" dest + dest="$usr_src_dir/$(basename "$remote_path")" + echo " $remote_path -> ${dest#"$repo_root"/}" + if [[ "$kind" == "dir" ]]; then + lab_rsync_from "$remote_path/" "$dest/" --delete + else + lab_rsync_from "$remote_path" "$dest" + fi +} + +echo "[3/5] syncing kernel headers and source" +# Dedup discovered paths — two map entries can canonicalize to the same +# dir via symlink chains. +printf '%s\n' "${remote_header_dirs[@]}" | awk 'NF && !seen[$0]++' | +while IFS= read -r remote_dir; do + sync_header_dir "$remote_dir" +done + +for entry in "${remote_source_entries[@]}"; do + [[ -z "$entry" ]] && continue + sync_source_path "${entry%$'\t'*}" "${entry##*$'\t'}" +done + +build_name="$(basename "$build_real")" +rm -rf "$build_dir" +ln -s "usr-src/$build_name" "$build_dir" +[[ -f "$build_dir/Makefile" ]] || die "synced build tree is missing Makefile: $build_dir" + +# --- vmlinux ------------------------------------------------------------- + +echo "[4/5] checking vmlinux debug image" +if lab_ssh_sudo "test -r '$remote_vmlinux'"; then + # rsync-over-sudo: skips unchanged, restartable, checksummed. Replaces + # an earlier cat-over-ssh approach that silently corrupted vmlinux on + # any sudo banner or connection drop (file is ~415MB). + echo " rsync $remote_vmlinux -> ${cache_dir#"$repo_root"/}/vmlinux" + lab_rsync_from "$remote_vmlinux" "$cache_dir/vmlinux" --rsync-path='sudo rsync' +else + rm -f "$cache_dir/vmlinux" + echo " no vmlinux on target ($remote_vmlinux is missing)" + echo " module debugging will work; kernel source debugging will not" + echo " to enable: scripts/01-provision-target.sh $target --debug-symbols" +fi + +# --- Kernel source resolution -------------------------------------------- +# +# Ubuntu has shipped three linux-source layouts: +# 1. /usr/src/linux-source-X.tar.bz2 (tarball, no enclosing dir) +# 2. /usr/src/linux-source-X/ (extracted, files at top) +# 3. /usr/src/linux-source-X/linux-source-X/ (extracted, nested one deep) +# Plus the variant where the tarball lives inside the same-named dir. +# +# Strategy: look for a Makefile + init/main.c (the kernel-source markers) +# at depth ≤ 3 under usr-src/. If found, symlink to it. Otherwise extract +# a tarball into source-tree/ on the host (self-healing on rerun, and +# avoids needing tar on the target). + +echo "[5/5] resolving kernel source for step-into-kernel" +source_link="$cache_dir/source" +source_tree="$cache_dir/source-tree" +rm -f "$source_link" + +is_kernel_source_root() { + [[ -f "$1/Makefile" && -d "$1/init" && -f "$1/init/main.c" ]] +} + +# Prefer the linux-source-X dir whose X matches the live kernel; fall +# back to the highest version installed. +short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" +found_root="" +matching_root="" +while IFS= read -r mf; do + candidate="$(dirname "$mf")" + if is_kernel_source_root "$candidate"; then + [[ -z "$found_root" ]] && found_root="$candidate" + if [[ -n "$short_kver" && "$candidate" == *"linux-source-$short_kver"* ]]; then + matching_root="$candidate" + break + fi + fi +done < <(find "$usr_src_dir" -mindepth 1 -maxdepth 3 -name Makefile -path '*linux-source-*' 2>/dev/null | sort -Vr) +found_root="${matching_root:-$found_root}" + +if [[ -n "$found_root" ]]; then + rm -rf "$source_tree" + rel="${found_root#"$cache_dir"/}" + ln -s "$rel" "$source_link" + echo " kernel source: $source_link -> $rel" +else + shopt -s nullglob + tarballs=("$usr_src_dir"/linux-source-*.tar.* "$usr_src_dir"/linux-source-*/linux-source-*.tar.*) + shopt -u nullglob + if [[ ${#tarballs[@]} -gt 0 ]]; then + tarball="${tarballs[0]}" + if is_kernel_source_root "$source_tree"; then + echo " kernel source already extracted at $source_tree" + else + echo " extracting $(basename "$tarball") -> source-tree/ (one-time, ~30s)" + rm -rf "$source_tree" + mkdir -p "$source_tree" + tar -C "$source_tree" --strip-components=1 -xf "$tarball" + fi + ln -s source-tree "$source_link" + echo " kernel source: $source_link -> source-tree" + else + rm -rf "$source_tree" + echo " no kernel source on target; kernel step-into will show disassembly only" + echo " to enable: scripts/01-provision-target.sh $target --debug-symbols" + fi +fi + +# --- Bookkeeping --------------------------------------------------------- + +printf '%s\n' "$kernel" > "$cache_dir/kernel.release" +printf '%s\n' "$build_real" > "$cache_dir/remote.build.path" +printf '%s\n' "${remote_header_dirs[@]}" > "$cache_dir/remote.header.paths" +ln -sfn "$target" "$repo_root/.kernel-cache/current" + +echo +echo "synced $target kernel $kernel" +echo " build -> $build_dir" +[[ -L "$source_link" ]] && echo " source -> $(readlink -f "$source_link")" +[[ -f "$cache_dir/vmlinux" ]] && echo " vmlinux: $cache_dir/vmlinux" +echo "next: scripts/03-build-module.sh $target" diff --git a/scripts/02-setup-wsl-build.sh b/scripts/02-setup-wsl-build.sh deleted file mode 100755 index e425107..0000000 --- a/scripts/02-setup-wsl-build.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - echo "usage: $0 " >&2 - exit 2 -} - -target="${1:-}" -case "$target" in - desktop|server) ;; - *) usage ;; -esac - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -env_file="$repo_root/lab.local.env" - -if [[ ! -f "$env_file" ]]; then - echo "missing $env_file; copy lab.example.env to lab.local.env and edit it" >&2 - exit 1 -fi - -set -a -# shellcheck source=/dev/null -source "$env_file" -set +a - -prefix="${target^^}" -cfg() { - local name="${prefix}_$1" - printf '%s' "${!name:-}" -} - -ssh_bin="${SSH_BIN:-ssh}" -rsync_bin="${RSYNC_BIN:-rsync}" -sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(cfg SSH_HOST)" -ssh_port="$(cfg SSH_PORT)" -ssh_user="$(cfg SSH_USER)" -ssh_pass="$(cfg SSH_PASS)" -sudo_pass="$(cfg SUDO_PASS)" -ssh_port="${ssh_port:-22}" -sudo_pass="${sudo_pass:-$ssh_pass}" -sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" - -[[ -n "$ssh_host" && -n "$ssh_user" ]] || { - echo "missing ${prefix}_SSH_HOST or ${prefix}_SSH_USER in lab.local.env" >&2 - exit 1 -} - -if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it in WSL: sudo apt-get install -y sshpass" >&2 - exit 1 -fi - -ssh_target="$ssh_user@$ssh_host" -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -ssh_transport=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port") -rsync_rsh="$ssh_bin -o StrictHostKeyChecking=accept-new -p $ssh_port" -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") - ssh_transport=("$sshpass_bin" -e "${ssh_transport[@]}") - rsync_rsh="$sshpass_bin -e $ssh_bin -o StrictHostKeyChecking=accept-new -p $ssh_port" -fi - -remote_sudo_prefix() { - if [[ -n "$sudo_pass_b64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$sudo_pass_b64" - else - printf "sudo " - fi -} - -apt_update() { - sudo apt-get \ - -o Acquire::Retries=3 \ - -o Acquire::http::Timeout=20 \ - -o Acquire::https::Timeout=20 \ - update -} - -apt_install() { - sudo apt-get \ - -o Acquire::Retries=3 \ - -o Acquire::http::Timeout=20 \ - -o Acquire::https::Timeout=20 \ - install -y "$@" -} - -echo "installing/validating WSL kernel-module build packages" -apt_update -apt_install \ - bc \ - bison \ - build-essential \ - dwarves \ - flex \ - gdb \ - libelf-dev \ - libssl-dev \ - openssh-client \ - rsync \ - sshpass - -kernel="$(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "uname -r")" -cache_dir="$repo_root/.kernel-cache/$target" -build_dir="$cache_dir/build" -usr_src_dir="$cache_dir/usr-src" -remote_vmlinux="/usr/lib/debug/boot/vmlinux-$kernel" - -mkdir -p "$cache_dir" "$usr_src_dir" - -echo "target $target is currently running kernel $kernel" -echo "discovering target kernel header layout" -build_real="$(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "readlink -f '/lib/modules/$kernel/build'")" -if ! SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "test -d '$build_real'"; then - echo "missing target build directory: $build_real" >&2 - echo "run scripts/01-provision-target.sh $target first" >&2 - exit 1 -fi - -mapfile -t remote_header_dirs < <(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "bash -s" < $dest" - # Preserve symlinks inside Ubuntu's kernel header trees. Some optional - # symlinks, such as rust support links, may be dangling and are harmless for - # external C module builds. - SSHPASS="$ssh_pass" "$rsync_bin" -a --delete -e "$rsync_rsh" \ - "$ssh_target:$remote_dir/" "$dest/" -} - -printf '%s\n' "${remote_header_dirs[@]}" | awk 'NF && !seen[$0]++' | -while IFS= read -r remote_dir; do - sync_header_dir "$remote_dir" -done - -build_name="$(basename "$build_real")" -rm -rf "$build_dir" -ln -s "usr-src/$build_name" "$build_dir" - -if [[ ! -f "$build_dir/Makefile" ]]; then - echo "synced build tree is missing Makefile: $build_dir" >&2 - exit 1 -fi - -echo "checking vmlinux debug image" -remote_sudo="$(remote_sudo_prefix)" -if SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}test -r '$remote_vmlinux'"; then - echo "copying $remote_vmlinux" - SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}cat '$remote_vmlinux'" > "$cache_dir/vmlinux" -else - rm -f "$cache_dir/vmlinux" - echo "warning: missing readable $remote_vmlinux on target" >&2 - echo "module builds can still work, but VS Code KGDB source debugging needs it" >&2 - echo "run scripts/01-provision-target.sh $target --debug-symbols to install it" >&2 -fi - -printf '%s\n' "$kernel" > "$cache_dir/kernel.release" -printf '%s\n' "$build_real" > "$cache_dir/remote.build.path" -printf '%s\n' "${remote_header_dirs[@]}" > "$cache_dir/remote.header.paths" -ln -sfn "$target" "$repo_root/.kernel-cache/current" - -echo "cached $target kernel $kernel in $cache_dir" -echo "local KDIR: $build_dir" -echo "current IntelliSense target: $target" diff --git a/scripts/03-build-module.sh b/scripts/03-build-module.sh index 2a0fb81..d34d096 100755 --- a/scripts/03-build-module.sh +++ b/scripts/03-build-module.sh @@ -1,38 +1,46 @@ #!/usr/bin/env bash +# Build module/ against the synced headers for . +# +# Delegates to the top-level Makefile, which stages module/ into +# build/intermediate/// and runs Kbuild from there with +# -ffile-prefix-map injected via KCFLAGS. Result: one .ko under +# build/artifacts///. set -euo pipefail -usage() { - echo "usage: $0 " >&2 - exit 2 -} - target="${1:-}" -case "$target" in - desktop|server) ;; - *) usage ;; -esac - repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 " + cache_dir="$repo_root/.kernel-cache/$target" kernel_file="$cache_dir/kernel.release" kdir="$cache_dir/build" -if [[ ! -f "$kernel_file" || ! -d "$kdir" ]]; then - echo "missing kernel cache for $target" >&2 - echo "run scripts/02-setup-wsl-build.sh $target first" >&2 - exit 1 -fi +[[ -f "$kernel_file" && -d "$kdir" ]] || + die "missing kernel cache for $target; run: scripts/02-setup-host-build.sh $target" + +module_dir="$repo_root/module" +[[ -e "$module_dir/Makefile" || -e "$module_dir/Kbuild" ]] || + die "no Makefile or Kbuild under $module_dir; drop your module project there (see $module_dir/README.md), or try the example: make use-example NAME=chuck_norise" kernel="$(<"$kernel_file")" intermediate_dir="$repo_root/build/intermediate/$target/$kernel" artifact_dir="$repo_root/build/artifacts/$target/$kernel" -echo "building hello.ko for $target kernel $kernel" +echo "building $module_dir for $target kernel $kernel" make -C "$repo_root" \ KDIR="$kdir" \ BUILD_ID="$target/$kernel" \ INTERMEDIATE_DIR="$intermediate_dir" \ ARTIFACT_DIR="$artifact_dir" \ + MODULE_DIR="$module_dir" \ clean modules -echo "built $artifact_dir/hello.ko" +mapfile -t kos < <(find "$artifact_dir" -maxdepth 1 -name '*.ko' -type f | sort) +[[ ${#kos[@]} -gt 0 ]] || die "build produced no .ko under $artifact_dir; check the make output above" + +for ko in "${kos[@]}"; do + echo "built $ko" +done +echo "next: scripts/04-deploy-debug-vscode.sh $target " diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index ced8c6d..976f0b0 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -1,127 +1,247 @@ #!/usr/bin/env bash +# Deploy the built module to a target, load it, and write the GDB init files +# VS Code (or `gdb -tui`) needs to attach and source-debug it. +# +# Phases: +# 1. Best-effort kill of any local gdb holding the debug-endpoint TCP +# port (KGDB serial bridge and QEMU gdbstub each serve one client). +# 2. Upload module/.ko, verify size matches. +# 3. Insmod in the background; poll /sys/module//sections/ for the +# runtime load addresses and emit an `add-symbol-file ...` line. +# 4. Emit .gdb/-.gdb (+ -attached variant) and the +# current-* symlinks the IDE launch configs point at. +# +# Env knobs: +# DEBUG_LOAD_DELAY_MS (5000) passed to the module if it declares debug_delay_ms +# INSMOD_WAIT_SECS (30) poll deadline for /sys/module/.../sections/.text set -euo pipefail usage() { - echo "usage: $0 " >&2 + echo "usage: $0 " >&2 exit 2 } target="${1:-}" -case "$target" in - desktop|server) ;; - *) usage ;; -esac +debug_method="${2:-}" +case "$debug_method" in kgdb|qemu) ;; *) usage ;; esac repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -env_file="$repo_root/lab.local.env" - -if [[ ! -f "$env_file" ]]; then - echo "missing $env_file; copy lab.example.env to lab.local.env and edit it" >&2 - exit 1 -fi +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 " -set -a +env_file="$(lab_env_file "$repo_root")" # shellcheck source=/dev/null source "$env_file" -set +a +lab_load_target "$target" -prefix="${target^^}" -cfg() { - local name="${prefix}_$1" - printf '%s' "${!name:-}" -} - -ssh_bin="${SSH_BIN:-ssh}" -scp_bin="${SCP_BIN:-scp}" -sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(cfg SSH_HOST)" -ssh_port="$(cfg SSH_PORT)" -ssh_user="$(cfg SSH_USER)" -ssh_pass="$(cfg SSH_PASS)" -sudo_pass="$(cfg SUDO_PASS)" -remote_dir="$(cfg REMOTE_DIR)" -kgdb_endpoint="$(cfg KGDB_ENDPOINT)" +debug_endpoint="$(target_require_cfg "$target" "DEBUG_ENDPOINT_${debug_method^^}")" debug_delay_ms="${DEBUG_LOAD_DELAY_MS:-5000}" +insmod_wait_secs="${INSMOD_WAIT_SECS:-30}" -ssh_port="${ssh_port:-22}" -remote_dir="${remote_dir:-/tmp/small-ko}" -sudo_pass="${sudo_pass:-$ssh_pass}" -sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" - -[[ -n "$ssh_host" && -n "$ssh_user" && -n "$kgdb_endpoint" ]] || { - echo "missing ${prefix}_SSH_HOST, ${prefix}_SSH_USER, or ${prefix}_KGDB_ENDPOINT" >&2 - exit 1 -} - -if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it in WSL: sudo apt-get install -y sshpass" >&2 - exit 1 -fi - -ssh_target="$ssh_user@$ssh_host" -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -scp_cmd=("$scp_bin" -o StrictHostKeyChecking=accept-new -P "$ssh_port") -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") - scp_cmd=("$sshpass_bin" -e "${scp_cmd[@]}") -fi - -remote_sudo_prefix() { - if [[ -n "$sudo_pass_b64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$sudo_pass_b64" - else - printf "sudo " - fi -} +# --- Locate artifact + vmlinux, verify target is on the cached kernel ---- cache_dir="$repo_root/.kernel-cache/$target" kernel_file="$cache_dir/kernel.release" vmlinux="$cache_dir/vmlinux" -if [[ ! -f "$kernel_file" ]]; then - echo "missing kernel cache for $target" >&2 - echo "run scripts/02-setup-wsl-build.sh $target first" >&2 - exit 1 -fi +[[ -f "$kernel_file" ]] || + die "missing kernel cache for $target; run: scripts/02-setup-host-build.sh $target" +[[ -f "$vmlinux" ]] || + die "missing $vmlinux — source debugging needs the matching vmlinux from the target. +- install + sync it with: + scripts/01-provision-target.sh $target $debug_method --debug-symbols + scripts/02-setup-host-build.sh $target" -if [[ ! -f "$vmlinux" ]]; then - echo "missing $vmlinux" >&2 - echo "module builds may work, but VS Code KGDB debugging needs the matching vmlinux" >&2 - echo "install the target's linux-image-*-dbgsym package, then rerun:" >&2 - echo " scripts/02-setup-wsl-build.sh $target" >&2 - exit 1 +kernel="$(<"$kernel_file")" + +lab_check_connection +live_kernel="$(lab_ssh 'uname -r')" +if [[ "$live_kernel" != "$kernel" ]]; then + die "kernel mismatch: target now runs '$live_kernel' but cache has '$kernel'. +- if the target rebooted into a new kernel, re-sync: + scripts/01-provision-target.sh $target $debug_method --debug-symbols # for a new vmlinux + scripts/02-setup-host-build.sh $target +- or reboot the target back into $kernel" fi -kernel="$(<"$kernel_file")" -artifact="$repo_root/build/artifacts/$target/$kernel/hello.ko" +artifact_dir="$repo_root/build/artifacts/$target/$kernel" intermediate_dir="$repo_root/build/intermediate/$target/$kernel" +module_dir="$repo_root/module" -if [[ ! -f "$artifact" ]]; then - echo "missing $artifact" >&2 - echo "run scripts/03-build-module.sh $target first" >&2 - exit 1 -fi +mapfile -t artifacts < <(find "$artifact_dir" -maxdepth 1 -name '*.ko' -type f | sort) +case ${#artifacts[@]} in + 0) die "no .ko under $artifact_dir; run: scripts/03-build-module.sh $target" ;; + 1) ;; + *) + echo "expected one .ko under $artifact_dir; found:" >&2 + printf ' %s\n' "${artifacts[@]}" >&2 + die "the lab assumes a single obj-m per build. Merge into one module (obj-m += foo.o; foo-y := a.o b.o ...) or split into separate module/ trees." + ;; +esac +artifact="${artifacts[0]}" +module_name="$(basename "$artifact" .ko)" + +# --- Output paths --------------------------------------------------------- gdb_dir="$repo_root/.gdb" mkdir -p "$gdb_dir" -ready_file="$gdb_dir/$target.ready" -symbols_file="$gdb_dir/$target-module-symbols.gdb" -gdb_file="$gdb_dir/$target-kgdb.gdb" -loader_log="$gdb_dir/$target-loader.log" -loader_script="$gdb_dir/$target-loader.sh" +symbols_file="$gdb_dir/$target-$debug_method-module-symbols.gdb" +gdb_file="$gdb_dir/$target-$debug_method.gdb" +gdb_attached_file="$gdb_dir/$target-$debug_method-attached.gdb" +loader_log="$gdb_dir/$target-$debug_method-loader.log" -rm -f "$ready_file" "$symbols_file" "$loader_log" "$loader_script" +rm -f "$symbols_file" "$loader_log" -remote_module="$remote_dir/hello.ko" -remote_sudo="$(remote_sudo_prefix)" +# --- Free the debug endpoint from stale GDB clients ---------------------- +# +# Best-effort: lsof to find sockets on the endpoint, ps to confirm comm=gdb, +# kill -9. Errors here never fail the deploy. +free_debug_endpoint() { + command -v lsof >/dev/null 2>&1 || return 0 + local host="${debug_endpoint%:*}" port="${debug_endpoint##*:}" + local pids + pids="$(lsof -ti "@$host:$port" 2>/dev/null || true)" + local gdb_pids=() pid + for pid in $pids; do + [[ "$(ps -p "$pid" -o comm= 2>/dev/null || true)" == "gdb" ]] && gdb_pids+=("$pid") + done + if [[ ${#gdb_pids[@]} -gt 0 ]]; then + echo "killing leftover gdb clients on $debug_endpoint:" + ps -p "${gdb_pids[@]}" -o pid,cmd 2>/dev/null | tail -n +2 | sed 's/^/ /' || true + kill -9 "${gdb_pids[@]}" 2>/dev/null || true + sleep 0.3 + fi +} +free_debug_endpoint || true + +# --- Upload + load ------------------------------------------------------- + +remote_module="$LAB_REMOTE_DIR/$module_name.ko" + +echo "uploading $artifact -> $LAB_SSH_TARGET:$remote_module" +lab_ssh "mkdir -p '$LAB_REMOTE_DIR'" +lab_scp_to "$artifact" "$remote_module" +# Verify size — a truncated .ko produces cryptic "invalid module format" errors. +local_size="$(stat -c %s "$artifact")" +remote_size="$(lab_ssh "stat -c %s '$remote_module' 2>/dev/null" | tr -d '[:space:]')" +[[ "$remote_size" == "$local_size" ]] || + die "upload size mismatch: local $local_size bytes, remote ${remote_size:-missing} bytes; retry scripts/04 or check disk space on the target" + +# Background insmod on the target so we can poll for sections in parallel +# with the module's debug_delay_ms sleep. +# +# Three load-bearing details: +# - `<&0` keeps sudo's stdin attached to the SSH-inherited pipe (where +# the password arrives). Non-job-control bash auto-redirects async +# stdin to /dev/null without an explicit redirection — sudo would +# see EOF and silently fail to authenticate. +# - `& sleep 1` keeps the SSH session open long enough for sudo to +# authenticate and exec nohup before the channel closes. +# - `nohup` + non-interactive bash → the orphaned chain survives +# session exit (bash doesn't huponexit non-interactively). +# +# `|| insmod ...` falls back when the module doesn't declare a +# debug_delay_ms parameter (kv arg would otherwise reject with EINVAL). +load_and_discover_symbols() { + echo "loading $module_name on $target (debug_delay_ms=$debug_delay_ms)" + lab_ssh_sudo "rmmod '$module_name' >/dev/null 2>&1 || true" + lab_ssh_sudo "nohup sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$LAB_REMOTE_DIR/insmod.log' 2>&1 <&0 & sleep 1" + + echo "polling /sys/module/$module_name/sections/ (timeout ${insmod_wait_secs}s)" -echo "uploading $artifact to $ssh_target:$remote_module" -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "mkdir -p '$remote_dir'" -SSHPASS="$ssh_pass" "${scp_cmd[@]}" "$artifact" "$ssh_target:$remote_module" + # Dump every readable file under sections/. Dynamic discovery handles + # non-standard sections (.text.hot, custom __ksymtab subsections, ...). + # + # The sh -c body MUST be single-quoted in the SSH command so the outer + # remote bash doesn't expand $(ls -A) and $f before sh -c sees them — + # with double quotes the outer bash would expand in its own CWD and + # the for loop would iterate over the wrong filenames. + local dump_body + # shellcheck disable=SC2016 # $(...) is intentionally deferred to sh -c. + dump_body='cd "/sys/module/'"$module_name"'/sections" 2>/dev/null && for f in $(ls -A 2>/dev/null); do [ -r "$f" ] && printf "%s %s\n" "$f" "$(cat "$f")"; done' + + local deadline=$((SECONDS + insmod_wait_secs)) tmp + tmp="$(mktemp)" + while ((SECONDS < deadline)); do + # No sleep — each SSH round-trip already takes 0.3-1s, which is + # the right polling cadence. + if lab_ssh_sudo "sh -c '$dump_body'" > "$tmp" 2>/dev/null; then + local text_addr + text_addr="$(awk '$1 == ".text" { print $2 }' "$tmp")" + if [[ -n "$text_addr" ]]; then + local cmd="add-symbol-file $artifact $text_addr" + while read -r sec addr; do + [[ "$sec" != ".text" && -n "$addr" ]] && cmd+=" -s $sec $addr" + done < "$tmp" + { + echo "$cmd" + printf 'echo loaded %s module symbols for %s\\n\n' "$module_name" "$target" + } > "$symbols_file" + rm -f "$tmp" + echo "wrote $symbols_file" + return 0 + fi + fi + done + + rm -f "$tmp" + echo "timeout: /sys/module/$module_name/sections/.text did not appear within ${insmod_wait_secs}s" + echo "remote insmod.log (empty = insmod produced no output; module may still have failed to load):" + lab_ssh "cat '$LAB_REMOTE_DIR/insmod.log' 2>/dev/null || true" | sed 's/^/ /' + echo "hint: ssh into the target and try the load by hand:" + echo " ssh $LAB_SSH_TARGET sudo insmod $remote_module" + echo " ssh $LAB_SSH_TARGET sudo dmesg | tail -40" + echo " ssh $LAB_SSH_TARGET ls -la /sys/module/$module_name/sections/ 2>/dev/null" + return 1 +} + +if load_and_discover_symbols > "$loader_log" 2>&1; then + tail -1 "$loader_log" +else + echo "module load / symbol discovery failed (full log: $loader_log):" + sed 's/^/ /' "$loader_log" + exit 1 +fi -cat > "$gdb_file" </init/main.c`; stripping the +# known suffix yields the prefix. +kernel_substitute_line="" +kernel_directory_line="" +kernel_src_link="$cache_dir/source" +if [[ -L "$kernel_src_link" || -d "$kernel_src_link" ]]; then + kernel_src_root="$(readlink -f "$kernel_src_link")" + kernel_directory_line="directory $kernel_src_root" + if command -v nm >/dev/null 2>&1 && command -v addr2line >/dev/null 2>&1; then + # `|| true` because awk's `exit` SIGPIPEs nm, and pipefail would + # kill the script. We still capture awk's output before exit. + sym_addr="$(nm "$vmlinux" 2>/dev/null | awk 'NF==3 && $3=="start_kernel" {print $1; exit}' || true)" + if [[ -n "$sym_addr" ]]; then + sym_loc="$(addr2line -e "$vmlinux" "$sym_addr" 2>/dev/null | head -1 | cut -d: -f1 || true)" + build_prefix="${sym_loc%/init/main.c}" + if [[ -n "$build_prefix" && "$build_prefix" != "$sym_loc" ]]; then + kernel_substitute_line="set substitute-path $build_prefix $kernel_src_root" + echo "kernel source mapping: $build_prefix -> $kernel_src_root" + else + echo "warning: addr2line returned unexpected location for start_kernel: $sym_loc" + fi + else + echo "warning: could not find start_kernel in vmlinux; kernel step-into will show disassembly" + fi + fi +else + echo "note: no kernel source synced; kernel step-into will show disassembly" + echo " to enable: scripts/01-... --debug-symbols && scripts/02-... $target" +fi + +{ + cat < "$gdb_file" -{ - printf '#!/usr/bin/env bash\n' - printf 'set -euo pipefail\n' - printf 'target=%q\n' "$target" - printf 'ready_file=%q\n' "$ready_file" - printf 'symbols_file=%q\n' "$symbols_file" - printf 'artifact=%q\n' "$artifact" - printf 'ssh_bin=%q\n' "$ssh_bin" - printf 'sshpass_bin=%q\n' "$sshpass_bin" - printf 'ssh_pass=%q\n' "$ssh_pass" - printf 'ssh_port=%q\n' "$ssh_port" - printf 'ssh_target=%q\n' "$ssh_target" - printf 'sudo_pass_b64=%q\n' "$sudo_pass_b64" - printf 'remote_module=%q\n' "$remote_module" - printf 'remote_dir=%q\n' "$remote_dir" - printf 'debug_delay_ms=%q\n' "$debug_delay_ms" - cat <<'LOADER' - -remote_sudo_prefix() { - if [[ -n "$sudo_pass_b64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$sudo_pass_b64" - else - printf "sudo " - fi -} +# Native Debug and cppdbg-with-miDebuggerServerAddress already called +# `target remote` and loaded the executable by the time they source this +# script — so commands that change global gdb state error with +# "Cannot change this setting while the inferior is running". Strip them. +grep -vE '^(target remote |set mi-async |set target-async |set tcp connect-timeout |set remotetimeout |set architecture |symbol-file )' \ + "$gdb_file" > "$gdb_attached_file" -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") -fi -remote_sudo="$(remote_sudo_prefix)" - -echo "loading module on $target and discovering symbols" -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}rmmod hello >/dev/null 2>&1 || true" -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "nohup ${remote_sudo}insmod '$remote_module' debug_delay_ms='$debug_delay_ms' > '$remote_dir/insmod.log' 2>&1 &" - -tmp_sections="$(mktemp)" -trap 'rm -f "$tmp_sections"' EXIT - -section_list=".text .data .bss .rodata .init.text .exit.text .text.unlikely .rodata.str1.1 .rodata.str1.8 .init.data .exit.data" -deadline=$((SECONDS + 30)) -while ((SECONDS < deadline)); do - if SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "${remote_sudo}sh -c 'for sec in $section_list; do path=/sys/module/hello/sections/\$sec; if [ -r \"\$path\" ]; then printf \"%s %s\n\" \"\$sec\" \"\$(cat \"\$path\")\"; fi; done'" \ - > "$tmp_sections"; then - text_addr="$(awk '$1 == ".text" { print $2 }' "$tmp_sections")" - if [[ -n "$text_addr" ]]; then - cmd="add-symbol-file $artifact $text_addr" - while read -r sec addr; do - if [[ "$sec" != ".text" && -n "$addr" ]]; then - cmd="$cmd -s $sec $addr" - fi - done < "$tmp_sections" - { - echo "$cmd" - echo "echo loaded hello.ko module symbols for $target\\n" - } > "$symbols_file" - echo "wrote module symbols to $symbols_file" - exit 0 - fi - fi - sleep 0.2 -done - -echo "failed to discover /sys/module/hello/sections/.text before breakpoint" -exit 1 -LOADER -} > "$loader_script" -chmod +x "$loader_script" -"$loader_script" > "$loader_log" 2>&1 - -echo "generated $gdb_file" -echo "loaded module and wrote symbols; log: $loader_log" +ln -sfn "$target-$debug_method.gdb" "$gdb_dir/current-debug.gdb" +ln -sfn "$target-$debug_method-attached.gdb" "$gdb_dir/current-debug-attached.gdb" +ln -sfn "../.kernel-cache/$target/vmlinux" "$gdb_dir/current-vmlinux" + +echo +echo "ready. GDB endpoint: $debug_endpoint" +echo " gdb script: $gdb_file" +echo " loader log: $loader_log" +echo "next: F5 in VS Code (Kernel: cppdbg) — or: gdb -tui $vmlinux -x $gdb_file" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh new file mode 100644 index 0000000..220c1b0 --- /dev/null +++ b/scripts/lib/common.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# Shared helpers sourced by every numbered script. +# +# `die`, `validate_target`, `target_*` are state-free. +# `lab_*` helpers rely on globals set by `lab_load_target`. +# `LAB_*` globals are set by `lab_load_target`. Use the helpers, not the +# raw ssh/scp/sshpass binaries. + +die() { + echo "$*" >&2 + exit 1 +} + +# --- Target validation ---------------------------------------------------- + +validate_target() { + local target="$1" + local usage="$2" + + [[ -n "$target" ]] || die "$usage" + [[ "$target" =~ ^[A-Za-z][A-Za-z0-9_-]*$ ]] || + die "invalid target '$target'; use letters, numbers, '_' or '-', starting with a letter" +} + +target_is_configured() { + local target="$1" + local known + + declare -p TARGETS >/dev/null 2>&1 || + die "missing TARGETS array in lab.local.env" + + for known in "${TARGETS[@]}"; do + [[ "$known" == "$target" ]] && return 0 + done + + die "unknown target '$target'; add it to TARGETS in lab.local.env" +} + +target_cfg() { + local target="$1" + local key="$2" + local map_name="TARGET_${key}" + + declare -p "$map_name" >/dev/null 2>&1 || return 0 + local -n map="$map_name" + printf '%s' "${map[$target]:-}" +} + +target_require_cfg() { + local target="$1" + local key="$2" + local value + + value="$(target_cfg "$target" "$key")" + [[ -n "$value" ]] || die "missing TARGET_${key}[$target] in lab.local.env" + printf '%s' "$value" +} + +require_debian_host() { + command -v apt-get >/dev/null 2>&1 || + die "the development host must be Debian-based (apt-get not found)" +} + +# --- Lab env loading ------------------------------------------------------ + +# Callers `source` the returned path directly so TARGET_* assignments land +# in script-global scope (sourcing from inside a function would scope them +# to the function). +lab_env_file() { + local env_file="$1/lab.local.env" + [[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it for your machines" + printf '%s' "$env_file" +} + +# --- Per-target connection setup ------------------------------------------ + +# Read TARGET_* config and populate LAB_* globals: LAB_TARGET, LAB_SSH_HOST, +# LAB_SSH_PORT, LAB_SSH_USER, LAB_SSH_PASS, LAB_SUDO_PASS, LAB_REMOTE_DIR, +# LAB_SSH_TARGET, LAB_SSH_CMD[], LAB_SCP_CMD[], LAB_RSYNC_RSH. +# +# LAB_SSH_CMD / LAB_SCP_CMD are flag-only — they do NOT include the host. +# Helpers (and the few direct callers) append it themselves, so options +# like `-t` can be inserted before the host (ssh treats anything after +# the host as the remote command). +lab_load_target() { + local target="$1" + target_is_configured "$target" + + # shellcheck disable=SC2034 # LAB_TARGET is consumed by sourcing scripts. + LAB_TARGET="$target" + LAB_SSH_HOST="$(target_require_cfg "$target" SSH_HOST)" + LAB_SSH_PORT="$(target_cfg "$target" SSH_PORT)" + LAB_SSH_USER="$(target_require_cfg "$target" SSH_USER)" + LAB_SSH_PASS="$(target_cfg "$target" SSH_PASS)" + LAB_SUDO_PASS="$(target_cfg "$target" SUDO_PASS)" + LAB_REMOTE_DIR="$(target_cfg "$target" REMOTE_DIR)" + + LAB_SSH_PORT="${LAB_SSH_PORT:-22}" + LAB_SUDO_PASS="${LAB_SUDO_PASS:-$LAB_SSH_PASS}" + LAB_REMOTE_DIR="${LAB_REMOTE_DIR:-/tmp/kmod-debug-lab-$LAB_SSH_USER}" + + local ssh_bin scp_bin sshpass_bin + ssh_bin="${SSH_BIN:-ssh}" + scp_bin="${SCP_BIN:-scp}" + sshpass_bin="${SSHPASS_BIN:-sshpass}" + + if [[ -n "$LAB_SSH_PASS" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then + die "TARGET_SSH_PASS[$target] is set but '$sshpass_bin' isn't installed; install with: sudo apt-get install -y sshpass" + fi + + LAB_SSH_TARGET="$LAB_SSH_USER@$LAB_SSH_HOST" + + # ConnectTimeout: fail fast on dead targets instead of multi-minute TCP wait. + # ServerAliveInterval: keep long-lived connections (e.g. the polling loop) + # alive through NAT idle timers. + local ssh_opts=( + -o StrictHostKeyChecking=accept-new + -o ConnectTimeout=10 + -o ServerAliveInterval=15 + ) + LAB_SSH_CMD=("$ssh_bin" "${ssh_opts[@]}" -p "$LAB_SSH_PORT") + LAB_SCP_CMD=("$scp_bin" "${ssh_opts[@]}" -P "$LAB_SSH_PORT") + LAB_RSYNC_RSH="$ssh_bin ${ssh_opts[*]} -p $LAB_SSH_PORT" + if [[ -n "$LAB_SSH_PASS" ]]; then + LAB_SSH_CMD=("$sshpass_bin" -e "${LAB_SSH_CMD[@]}") + LAB_SCP_CMD=("$sshpass_bin" -e "${LAB_SCP_CMD[@]}") + LAB_RSYNC_RSH="$sshpass_bin -e $LAB_RSYNC_RSH" + fi +} + +# --- Remote command helpers ----------------------------------------------- +# All require a prior `lab_load_target` call. + +# Inherits stdin from the caller (so a caller can pipe data into the remote +# command — see lab_ssh_sudo). +lab_ssh() { + SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "$@" +} + +# Run a command on the target as root. The password reaches `sudo -S` via +# ssh's stdin, so it never appears in argv on either host. +# +# `-k` ignores any cached sudo timestamp (e.g. left by `scripts/00-check-target.sh`) +# for THIS invocation only — without it, sudo would skip the stdin read and +# the password would leak into the next reader. It does NOT invalidate the +# user's existing sudo cache. +# +# For callers where the remote command itself reads stdin (e.g. `bash -s`), +# this helper is not enough — NOPASSWD policy also skips the stdin read, +# leaking the password. Use SUDO_ASKPASS instead (see scripts/01 for the +# pattern). +lab_ssh_sudo() { + printf '%s\n' "$LAB_SUDO_PASS" | + SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "sudo -k -S -p '' $*" +} + +lab_scp_to() { + SSHPASS="$LAB_SSH_PASS" "${LAB_SCP_CMD[@]}" "$1" "$LAB_SSH_TARGET:$2" +} + +# Extra rsync flags can be appended — notably `--rsync-path='sudo rsync'` +# for paths only root can read (vmlinux), and `--delete` for tree mirroring. +lab_rsync_from() { + local remote_src="$1" + local local_dst="$2" + shift 2 + SSHPASS="$LAB_SSH_PASS" "${RSYNC_BIN:-rsync}" -a -e "$LAB_RSYNC_RSH" "$@" \ + "$LAB_SSH_TARGET:$remote_src" "$local_dst" +} + +# --- Preflight ------------------------------------------------------------ + +# `--no-sudo` skips the sudo test, for scripts/01 which may be running TO +# configure sudo for the first time. +# shellcheck disable=SC2120 # --no-sudo arg is optional. +lab_check_connection() { + local check_sudo=1 + [[ "${1:-}" == "--no-sudo" ]] && check_sudo=0 + + if ! lab_ssh 'true' 2>/dev/null; then + die "cannot reach $LAB_SSH_TARGET over ssh (port $LAB_SSH_PORT). +- check TARGET_SSH_HOST/PORT/USER for '$LAB_TARGET' in lab.local.env +- try by hand: ssh -p $LAB_SSH_PORT $LAB_SSH_TARGET +- if using passwords, confirm TARGET_SSH_PASS[$LAB_TARGET] is set" + fi + if (( check_sudo )) && ! lab_ssh_sudo 'true' 2>/dev/null; then + die "ssh reaches $LAB_SSH_TARGET but sudo doesn't work there. +- check TARGET_SUDO_PASS[$LAB_TARGET] in lab.local.env (defaults to TARGET_SSH_PASS) +- on the target, confirm: sudo -n -v +- consider NOPASSWD for the lab user on long-lived dev targets" + fi +}