diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b9dbf81 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,28 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true + +[*.{sh,bash}] +indent_style = tab + +[Makefile] +indent_style = tab + +[Kbuild] +indent_style = tab + +[*.{json,md,yml,yaml}] +indent_style = space +indent_size = 2 + +[*.{c,h}] +indent_style = tab +indent_size = 8 + +[*.ps1] +indent_style = space +indent_size = 4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..8c84bff --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,114 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +jobs: + shellcheck: + name: shellcheck + bash -n + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + + - name: bash -n (syntax check) + run: | + for f in scripts/lib/*.sh scripts/0*.sh; do + echo "::group::bash -n $f" + bash -n "$f" + echo "::endgroup::" + done + + - name: shellcheck + run: shellcheck -x scripts/lib/common.sh scripts/0*.sh + + json: + name: validate JSON + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: validate .vscode/*.json + run: | + for f in .vscode/*.json; do + echo "::group::python -m json.tool $f" + python3 -m json.tool "$f" > /dev/null + echo "::endgroup::" + done + + makefile: + name: Makefile syntax + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: make help (dry-parse) + run: make help + + build: + name: build example module against runner kernel (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + # Surface drift across LTS kernels. ubuntu-22.04 ships 5.15; 24.04 ships 6.8. + os: [ubuntu-22.04, ubuntu-24.04] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: install host kernel headers + build tools + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + bc bison flex libelf-dev libssl-dev \ + "linux-headers-$(uname -r)" + + - name: use the example module + run: make use-example NAME=chuck_norise + + - name: build via top-level Makefile (default target) + # KDIR defaults to /lib/modules/$(uname -r)/build when there's no + # .kernel-cache/current, so this exercises the full prepare-build + + # KCFLAGS pipeline against a real (the runner's) kernel. + run: make modules + + - name: assert a .ko was produced + run: | + set -e + shopt -s nullglob + kos=(build/artifacts/*/*/*.ko) + if [ "${#kos[@]}" -eq 0 ]; then + echo "::error::no .ko produced under build/artifacts/" + ls -R build/ || true + exit 1 + fi + for ko in "${kos[@]}"; do + echo "built $ko ($(stat -c %s "$ko") bytes)" + file "$ko" + done + + - name: verify DWARF prefix-map rewrote paths back to module/ + # The whole point of -ffile-prefix-map is that DWARF references your + # source under module/<...>.c, not build/intermediate/<...>.c. If a + # change to the Makefile breaks the injection, this catches it. + run: | + set -e + ko=$(find build/artifacts -name '*.ko' | head -1) + if objdump --dwarf=decodedline "$ko" 2>/dev/null \ + | grep -qE '/build/intermediate/'; then + echo "::error::DWARF still references build/intermediate/ — prefix-map broken" + objdump --dwarf=decodedline "$ko" 2>/dev/null | grep -E '/build/intermediate/' | head + exit 1 + fi + echo "ok: DWARF paths point at module/" + + - name: clean-module resets cleanly + run: | + make clean-module + ls module/ + [ "$(ls module/)" = "README.md" ] || { echo "::error::clean-module did not reduce module/ to just README.md"; exit 1; } diff --git a/.gitignore b/.gitignore index b7f1afc..aa02f4b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,16 @@ +# Per-machine config (lab.example.env is the tracked template). lab.local.env +# Per-target synced kernel headers, source, vmlinux. .kernel-cache/ + +# Generated GDB init files (regenerated by scripts/04 on every F5). .gdb/ + +# Per-target build intermediates and final .ko artifacts. build/ +# Kbuild stragglers in case anything runs out of place. *.ko *.mod *.mod.c @@ -12,10 +19,13 @@ build/ Module.symvers modules.order -compile_commands.json +# IDE / editor noise. .vscode/ipch/ .codex -kernel.gdb +.claude/ + +# Agent / session local notes (CLAUDE.md is consumed by Claude Code; keep it +# out of the public repo so workflow guidance stays separate from user docs). CLAUDE.md session.md -.claude/ +todo.md diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 9065b7f..248ab5d 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -41,5 +41,10 @@ "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler_types.h" ] } + ], + "_notes": [ + "KBUILD_MODNAME is a placeholder; the actual value at compile time is your module's real name. IntelliSense only needs the macro to resolve, not match exactly.", + "ubuntu/include is Ubuntu-kernel-specific (where the distro's extra headers live); on non-Ubuntu kernels VS Code silently ignores it.", + "All paths under .kernel-cache/current/ resolve only after scripts/02-setup-host-build.sh has run. Before that, expect include squiggles." ] } diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 9a91c37..97d3205 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,6 +1,6 @@ { "recommendations": [ - "ms-vscode.cpptools" + "ms-vscode.cpptools", + "webfreak.debug" ] } - diff --git a/.vscode/launch.json b/.vscode/launch.json index abfbc34..cb0c619 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,13 +4,13 @@ { "id": "debugEndpoint", "type": "promptString", - "description": "GDB remote endpoint (host:port). Used only by the Native Debug launch; cppdbg reads it from the generated .gdb script.", + "description": "GDB remote endpoint (host:port). Only used by Native Debug — cppdbg reads it from the generated .gdb script.", "default": "127.0.0.1:1234" } ], "configurations": [ { - "name": "Kernel: Cppdbg Debug", + "name": "Kernel: cppdbg (full GDB MI)", "type": "cppdbg", "request": "launch", "program": "${workspaceFolder}/.gdb/current-vmlinux", @@ -21,7 +21,7 @@ "preLaunchTask": "Kernel: Deploy Debug", "setupCommands": [ { - "description": "Attach GDB to the selected target", + "description": "Source the GDB init script generated by scripts/04", "text": "source ${workspaceFolder}/.gdb/current-debug.gdb", "ignoreFailures": false } @@ -30,7 +30,7 @@ "externalConsole": false }, { - "name": "Kernel: Native Debug", + "name": "Kernel: Native Debug (faster, fewer features)", "type": "gdb", "request": "attach", "executable": "${workspaceFolder}/.gdb/current-vmlinux", diff --git a/.vscode/settings.json b/.vscode/settings.json index 1d56719..9503432 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,38 +1,15 @@ { "C_Cpp.default.configurationProvider": "", "C_Cpp.default.configurationName": "Linux kernel module - current", - "C_Cpp.default.compileCommands": "", - "C_Cpp.default.compilerPath": "/usr/bin/gcc", - "C_Cpp.default.cStandard": "gnu11", - "C_Cpp.default.intelliSenseMode": "linux-gcc-x64", - "C_Cpp.default.defines": [ - "__KERNEL__", - "MODULE", - "CC_USING_FENTRY", - "KBUILD_MODNAME=\"kmod\"", - "KBUILD_BASENAME=\"kmod\"", - "__KBUILD_MODNAME=kmod_kmod" - ], - "C_Cpp.default.includePath": [ - "${workspaceFolder}/module/**", - "${workspaceFolder}/.kernel-cache/current/build/include", - "${workspaceFolder}/.kernel-cache/current/build/include/uapi", - "${workspaceFolder}/.kernel-cache/current/build/include/generated", - "${workspaceFolder}/.kernel-cache/current/build/include/generated/uapi", - "${workspaceFolder}/.kernel-cache/current/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include/uapi", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include/generated", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include/generated/uapi" - ], - "C_Cpp.default.forcedInclude": [ - "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler-version.h", - "${workspaceFolder}/.kernel-cache/current/build/include/linux/kconfig.h", - "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler_types.h" - ], "C_Cpp.errorSquiggles": "enabled", "files.associations": { "*.h": "c", - "*.c": "c" + "*.c": "c", + "Kbuild": "makefile" + }, + "files.exclude": { + ".kernel-cache": true, + ".gdb": true, + "build": true } } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index b9190e9..69e19f9 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -19,6 +19,12 @@ } ], "tasks": [ + { + "label": "Kernel: Check Target", + "type": "shell", + "command": "./scripts/00-check-target.sh ${input:kernelTarget} ${input:debugMethod}", + "problemMatcher": [] + }, { "label": "Kernel: Provision Target", "type": "shell", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e518573 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,83 @@ +# Contributing + +Bug reports, fixes, and improvements are welcome. This file describes what +that looks like for this repo. + +## What this repo is (and isn't) + +This is a **template** for building and source-debugging out-of-tree Linux +kernel modules. The lab tooling lives in `Makefile`, `scripts/`, +`.vscode/`, `host/`, and the docs. The example module under +`examples/chuck_norise/` exists to demonstrate the workflow end-to-end. + +PRs that **belong** here: + +- Bugs in the lab tooling — broken scripts, wrong assumptions about target + state, scripts that fail on re-run, unclear error messages. +- Support for a new target OS family (currently only Ubuntu). See the + `TARGET_OS` checks in scripts 01 and 02 for where to add a backend. +- New debug methods that fit the ` -> GDB` shape (anything + speaking the GDB remote serial protocol). +- Documentation that fixes misleading or missing information. +- Quality-of-life improvements: better error messages, more idempotent + re-runs, faster sync, smaller artifact dirs. + +PRs that **do not** belong here: + +- Changes to `module/` (that's the user's slot; we keep it as a placeholder + with just a README). +- New example modules under `examples/`. We keep one example focused and + small so it stays useful as a walkthrough. +- Project-management features (work logs, task tracking, etc.). The lab + stays focused on build + debug. + +## Reporting bugs + +Open an issue with: + +- Your **dev host** (distro, version, kernel) and **target VM** (distro, + version, kernel, hypervisor). +- The debug method you tried (`kgdb` or `qemu`) and the endpoint you + pointed it at. +- The command you ran and the full output. For deploy failures, attach + `.gdb/--loader.log`. +- What you expected, what happened instead. + +## Submitting changes + +1. Fork and branch from `main`. +2. Make focused commits — one logical change per commit. The git history + should read top-to-bottom as a small set of intentional steps. +3. Keep scripts shellcheck-clean (`shellcheck -x scripts/lib/common.sh + scripts/0*.sh`) and bash-syntax-clean (`bash -n`). +4. Update docs in the same PR. If you touched a script, check `README.md` + and `module/README.md` for anything the change makes inaccurate. +5. Open a PR with a description that explains *why* the change is needed, + not just what it does. The diff already says what. + +## Style + +- **Shell:** tabs for indentation. `set -euo pipefail` at the top of every + script. Errors via the `die` helper. Use `lab_*` helpers from + `scripts/lib/common.sh` for SSH/SCP/rsync — do not call those binaries + directly from numbered scripts. +- **Makefile:** tabs for recipes. Explicit `.PHONY` declarations. +- **Markdown:** wrap prose at ~80 columns. Use fenced code blocks with + language tags. +- **JSON (VS Code config):** 2-space indent, trailing newline. + +The repo includes an `.editorconfig` that captures these — most editors +will apply it automatically. + +## Testing changes locally + +There is no automated test suite for the build/debug flow (it requires real +hardware-or-VM targets). Before sending a PR: + +- Run `bash -n scripts/0*.sh scripts/lib/*.sh` to catch syntax errors. +- If you have `shellcheck`, run it too: `shellcheck -x scripts/lib/common.sh + scripts/0*.sh`. +- Smoke-test the full chain against at least one target VM you have access + to: `01 --debug-symbols` → reboot → `02` → `03` → `04` → F5 in VS Code + → set breakpoints in both your module and a kernel function and confirm + both bind. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b77f1ff --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +MIT License + +Copyright (c) 2026 Dor Grosglick and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +--- + +This MIT license covers the lab tooling in this repository (Makefile, +scripts/, .vscode/, host/, docs). The example module under +examples/chuck_norise/ is also MIT-licensed for ease of reuse. Any kernel +module you place under module/ is yours — this lab's license does not impose +terms on your module's source. diff --git a/Makefile b/Makefile index 5893e0f..7560971 100644 --- a/Makefile +++ b/Makefile @@ -29,20 +29,78 @@ INTERMEDIATE_DIR ?= $(BUILD_ROOT)/intermediate/$(BUILD_ID) ARTIFACT_DIR ?= $(BUILD_ROOT)/artifacts/$(BUILD_ID) MODULE_DIR ?= $(CURDIR)/module -EXTRA_CCFLAGS ?= -g -DDEBUG +EXTRA_CCFLAGS ?= -g -DDEBUG KCFLAGS_INJECT := -ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR) $(EXTRA_CCFLAGS) -.PHONY: all modules prepare-build clean +.PHONY: all modules prepare-build clean help use-example clean-module all: modules +help: + @echo "Top-level lab Makefile." + @echo "" + @echo "Normally invoked by scripts/03-build-module.sh , which sets" + @echo "KDIR, BUILD_ID, INTERMEDIATE_DIR, and ARTIFACT_DIR to per-target paths." + @echo "" + @echo "Build targets:" + @echo " modules build module/ against KDIR (default)" + @echo " clean remove intermediate and artifact dirs for BUILD_ID" + @echo " prepare-build stage module/ into INTERMEDIATE_DIR (internal)" + @echo "" + @echo "module/ ergonomics:" + @echo " use-example NAME=chuck_norise copy an example into module/" + @echo " clean-module reset module/ to just README.md" + @echo "" + @echo "Vars (auto-resolved when .kernel-cache/current points at a target):" + @echo " MODULE_DIR $(MODULE_DIR)" + @echo " KDIR $(KDIR)" + @echo " BUILD_ID $(BUILD_ID)" + @echo " INTERMEDIATE_DIR $(INTERMEDIATE_DIR)" + @echo " ARTIFACT_DIR $(ARTIFACT_DIR)" + @echo " EXTRA_CCFLAGS $(EXTRA_CCFLAGS) (appended to KCFLAGS)" + +# Copy examples/$(NAME)/ into module/. Refuses if module/ already contains a +# Makefile/Kbuild (use clean-module first). NAME=chuck_norise unless overridden. +NAME ?= chuck_norise +use-example: + @if [ ! -d "$(CURDIR)/examples/$(NAME)" ]; then \ + echo "Makefile: examples/$(NAME) does not exist;" >&2; \ + echo " available examples:" >&2; \ + ls $(CURDIR)/examples 2>/dev/null | sed 's/^/ /' >&2; \ + exit 1; \ + fi + @if [ -f "$(MODULE_DIR)/Makefile" ] || [ -f "$(MODULE_DIR)/Kbuild" ]; then \ + echo "Makefile: $(MODULE_DIR) already has a Makefile/Kbuild." >&2; \ + echo " Run 'make clean-module' first if you want to replace it." >&2; \ + exit 1; \ + fi + @echo "copying examples/$(NAME)/* -> $(MODULE_DIR)/ (keeping module/README.md placeholder)" + @find "$(CURDIR)/examples/$(NAME)" -mindepth 1 -maxdepth 1 -not -name README.md \ + -exec cp -r {} "$(MODULE_DIR)/" \; + @echo "done. next: scripts/03-build-module.sh " + @echo "(the example's docs stay at examples/$(NAME)/README.md)" + +# Remove everything from module/ except its placeholder README, so you can +# drop a fresh module project in. Refuses to touch tracked files outside +# module/. +clean-module: + @if [ ! -d "$(MODULE_DIR)" ]; then \ + echo "Makefile: $(MODULE_DIR) is missing — nothing to clean." >&2; \ + exit 1; \ + fi + @find "$(MODULE_DIR)" -mindepth 1 -maxdepth 1 -not -name README.md -print0 \ + | xargs -0 -r rm -rf + @echo "module/ reset (kept only README.md)" + modules: prepare-build $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" KCFLAGS="$(KCFLAGS_INJECT)" modules @mkdir -p "$(ARTIFACT_DIR)" @set -e; \ kos=$$(find "$(INTERMEDIATE_DIR)" -maxdepth 2 -name '*.ko' -type f); \ if [ -z "$$kos" ]; then \ - echo "no .ko produced under $(INTERMEDIATE_DIR)" >&2; exit 1; \ + echo "Makefile: no .ko produced under $(INTERMEDIATE_DIR);" >&2; \ + echo " check the Kbuild output above for compile errors." >&2; \ + exit 1; \ fi; \ for ko in $$kos; do \ name=$$(basename "$$ko"); \ @@ -50,17 +108,23 @@ modules: prepare-build echo "built $(ARTIFACT_DIR)/$$name"; \ done -# Mirror module/ into INTERMEDIATE_DIR as a tree of absolute symlinks so -# Kbuild's M= sees the user's sources in a writeable scratch dir without -# polluting module/. `cp -as` is GNU coreutils; it recursively creates dirs -# and symlinks each file. Re-running is safe because we wipe the symlink -# scaffolding first. +# Mirror $(MODULE_DIR) into $(INTERMEDIATE_DIR) as a tree of absolute +# symlinks so Kbuild's `M=` sees the user's sources in a writeable scratch +# dir without polluting module/. `cp -as` is GNU coreutils; it recursively +# creates dirs and symlinks each file. Re-running is idempotent because we +# wipe the symlink scaffolding first. prepare-build: @if [ ! -e "$(MODULE_DIR)/Makefile" ] && [ ! -e "$(MODULE_DIR)/Kbuild" ]; then \ - echo "no Makefile or Kbuild under $(MODULE_DIR)" >&2; \ - echo "see $(MODULE_DIR)/README.md for the expected layout" >&2; \ + echo "Makefile: no Makefile or Kbuild under $(MODULE_DIR);" >&2; \ + echo " drop your kernel module project there (see $(MODULE_DIR)/README.md)" >&2; \ + echo " or try the example: make use-example NAME=chuck_norise" >&2; \ exit 1; \ fi + @cp --help 2>&1 | grep -q -- '--symbolic-link' || { \ + echo "Makefile: GNU 'cp' (with --symbolic-link, --archive) is required;" >&2; \ + echo " the lab targets Debian-based hosts. On macOS: brew install coreutils, then re-run with CP=gcp." >&2; \ + exit 1; \ + } @mkdir -p "$(INTERMEDIATE_DIR)" @find "$(INTERMEDIATE_DIR)" -depth -type l -delete @find "$(INTERMEDIATE_DIR)" -depth -type d -empty -not -path "$(INTERMEDIATE_DIR)" -delete diff --git a/README.md b/README.md index 9301728..538ee56 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,109 @@ -# Linux Kernel Module Debug Lab — Template +# Linux Kernel Module Debug Lab -A reusable development and source-debugging setup for out-of-tree Linux kernel -modules. The repo is generic: drop your module project under `module/`, point -the lab at a target VM, and source-debug from VS Code or `gdb -tui`. +A template for building and **source-debugging** out-of-tree Linux kernel +modules against one or more target VMs. Drop your module under `module/`, +point the lab at a target, press F5 — step through your code (and into +kernel code) in VS Code or `gdb -tui`. -A worked example lives under `examples/chuck_norise/` — a tiny multi-file -char device. Copy it into `module/` to run the full flow end-to-end without -writing any module code first. +What you get out of the box: -## What This Template Gives You +- **One slot, any module.** `module/` is your project; the lab is generic. +- **Per-target builds.** Cross-build the same source against multiple + target kernels without polluting your tree. +- **Breakpoints just work.** DWARF `-ffile-prefix-map` is injected for + you so IDE breakpoints by absolute path bind to your real source files. +- **Step into the kernel.** With one flag (`--debug-symbols`), the lab + pulls down `vmlinux` + kernel source and wires GDB `substitute-path` + so you can step from your module's read handler into `vfs_read`. +- **Two debug methods.** In-kernel KGDB over serial (`kgdb`) and any + hypervisor GDB stub (`qemu`, also VMware's `debugStub`). +- **One-command flow.** Four numbered scripts + a VS Code F5. -- A per-target, out-of-tree build that stages your module/ tree into - `build/intermediate///` and writes the final `.ko` under - `build/artifacts///`. -- DWARF path rewriting (`-ffile-prefix-map`) injected via `KCFLAGS`, so - breakpoints set in your IDE bind to the real files under `module/` and - not to the staged copies. -- Provisioning, header sync, build, deploy, load, and symbol discovery - driven from four numbered scripts plus the VS Code task/launch wiring - that calls them. -- Support for two debug methods against the same target — in-kernel KGDB - over serial (`kgdb`) and a hypervisor GDB stub (`qemu`, also covers - VMware's `debugStub.listen.guest64`). +The repo ships with a small worked example (`examples/chuck_norise/`) so +you can run the whole flow end-to-end before writing a line of module +code. -## Machine Model +--- -The lab uses two kinds of machines: +## Quickstart -1. A Debian-based development host. - This can be Debian, Ubuntu Desktop, Ubuntu under WSL, or another - Debian-based distro. The scripts assume `apt`/`apt-get`, `bash`, `ssh`, - `scp`, `rsync`, `make`, and standard GNU userland tools. +You need a **Debian-based dev host** (Debian, Ubuntu, WSL Ubuntu, …) and +at least one **Ubuntu target VM** reachable over SSH. From fresh clone +to a stopped breakpoint: -2. A target VM. - The target is where the module is loaded and tested. Communication with - the target is over SSH. For now, the supported target OS is Ubuntu — add - a target OS backend in the scripts to support another distro. +```bash +# 0. clone + configure +git clone kmod-debug-lab && cd kmod-debug-lab +cp lab.example.env lab.local.env +$EDITOR lab.local.env # set TARGET_SSH_*, endpoints, etc. + +# 0.5 (optional) sanity-check the target before changing anything on it +./scripts/00-check-target.sh server qemu + +# 1. one-time target setup (installs headers, vmlinux dbg, kernel source) +./scripts/01-provision-target.sh server qemu --debug-symbols +# ...reboot the target VM so new GRUB args take effect... + +# 2. one-time host setup (syncs everything to .kernel-cache/server/) +./scripts/02-setup-host-build.sh server + +# 3. try the example module (replace with your own when ready) +make use-example NAME=chuck_norise +./scripts/03-build-module.sh server + +# 4. deploy + load on the target +./scripts/04-deploy-debug-vscode.sh server qemu + +# 5. open the repo in VS Code and press F5 ("Kernel: cppdbg") +code . +``` + +Set a breakpoint in `module/src/hello.c` and one in `vfs_read` — both +bind. You're stepping through kernel code from your module. + +--- + +## How the pieces fit + +``` +module/ your module project (Makefile + sources) +examples/chuck_norise/ worked example; copy into module/ with `make use-example` + +scripts/00-check-target.sh non-mutating preflight: ssh / sudo / headers / vmlinux / endpoint +scripts/01-provision-target.sh target-side: install headers, vmlinux dbg, source, GRUB args + (also `--uninstall` to undo the GRUB args) +scripts/02-setup-host-build.sh dev-host: sync headers + vmlinux + kernel source into .kernel-cache/ +scripts/03-build-module.sh dev-host: build module/ via Kbuild against the synced headers +scripts/04-deploy-debug-vscode.sh dev-host: upload, insmod, write .gdb/-.gdb +scripts/lib/common.sh shared bash helpers (lab_ssh, lab_ssh_sudo, lab_rsync_from, …) + +.kernel-cache// per-target build/source/vmlinux cache (gitignored) +build/ per-target intermediate + final .ko (gitignored) +.gdb/ generated GDB init files (gitignored, regenerated on F5) + +host/ optional helpers for VMware-on-Windows users +lab.example.env -> lab.local.env per-machine config (gitignored copy) +``` + +Top-level `Makefile` exposes a few non-build targets too: +`make use-example NAME=` (copy `examples//` into `module/`), +`make clean-module` (reset `module/` to just its placeholder README), +`make help` (full var/target listing). -## The module/ Contract +The numbered scripts are designed to be run in order. After the one-time +01 + 02 against a target, the inner loop is **03 → 04 → F5** (or just F5, +which runs 04 as a prelaunch task and rebuilds via 03 if you ran it +manually first). -`module/` is the single slot for your module project. The lab is otherwise -generic — it does not know your module's name, source layout, or behavior. +--- -The build pipeline expects: +## The `module/` contract -- `module/Makefile` (or `module/Kbuild`) following standard out-of-tree - Kbuild conventions: +`module/` is the lab's only slot for your kernel module project. The lab +does not know your module's name, source layout, or behavior; it expects +exactly: + +- **A standard Kbuild file** at `module/Makefile` (or `module/Kbuild`): ```makefile obj-m += my_module.o @@ -54,384 +112,287 @@ The build pipeline expects: ``` See `examples/chuck_norise/Makefile` for the canonical shape, including - the optional `ifndef KERNELRELEASE` wrapper that also lets you run plain + the optional `ifndef KERNELRELEASE` wrapper that lets you also run plain `make` directly in `module/`. -- Exactly one `obj-m` entry per build. The artifact name becomes - `.ko`. The lab discovers the module name from the produced `.ko`, - so you do not declare it anywhere else. - -- Optional: a `debug_delay_ms` module parameter that sleeps in `module_init` - before doing anything observable. The lab passes the value from - `DEBUG_LOAD_DELAY_MS` (default 5000 ms) to `insmod`. Modules without the - parameter are loaded plainly. - -You decide everything else: source layout, header layout, license, exported -symbols. - -## Try The Example Module - -```bash -cp -r examples/chuck_norise/. module/ -``` - -Then run the lab flow described below. +- **Exactly one `obj-m` entry per build.** The lab discovers the module + name from the produced `.ko` — you don't declare it anywhere + else. -## Debug Methods +- **Optional:** a `debug_delay_ms` module parameter: -This repo supports two debug methods. Provisioning (script 01) and the -deploy step (script 04) both take a `` argument so you can switch -between them per run. + ```c + static unsigned int debug_delay_ms = 5000; + module_param(debug_delay_ms, uint, 0644); + ``` -- **`kgdb`** — in-kernel KGDB talking over the guest's serial port. The - provisioning step adds `kgdboc=ttyS*,115200` and `sysrq_always_enabled=1` - to the guest's kernel command line. To halt the running kernel into GDB, - run `echo g | sudo tee /proc/sysrq-trigger` on the target. Useful when - you cannot change the hypervisor's command line, e.g. VMware running as - your only option. + When present, scripts/04 passes the value from `DEBUG_LOAD_DELAY_MS` + (in `lab.local.env`, default 5000 ms) so your `module_init` sleeps + long enough for the host to read `/sys/module//sections/*` and + for you to break in GDB before init does anything. Modules without + the parameter would reject `insmod foo.ko debug_delay_ms=...` with + `-EINVAL`; the loader catches that and retries with a plain `insmod`, + so undeclared modules still load. -- **`qemu`** — QEMU's built-in gdbstub (`-gdb tcp::PORT` or `-s`). No KGDB - in the guest is required; the hypervisor halts the vCPU directly. - VMware's `debugStub.listen.guest64` is the same shape and slots into this - method too. Connecting GDB to the stub halts the vCPU; set breakpoints, - then `continue`. +Source layout, headers, license — those are yours. The lab passes +through whatever your Kbuild file declares. -Neither method assumes the module calls `kgdb_breakpoint()` on insmod. In -both methods you break manually after the deploy step finishes. The lab's -convention is for the module to sleep `debug_delay_ms` (default 5000 ms) at -the start of init so you have a window to break before init runs and so the -host has time to read `/sys/module//sections/*`. +--- -## Configure The Lab +## Configuring the lab -Copy the example environment file and edit it for your machines: +Copy the example env and edit it for your machines: ```bash cp lab.example.env lab.local.env ``` -Targets are data, not variable prefixes. Add profile names to `TARGETS`, -then fill the `TARGET_*` maps with entries keyed by that profile name: +Targets are **data, not variable prefixes**. Add a profile name to +`TARGETS`, then add entries to the `TARGET_*` maps under that name: ```bash TARGETS=(desktop server) -declare -A TARGET_OS=( - [desktop]=ubuntu - [server]=ubuntu -) - declare -A TARGET_SSH_HOST=( [desktop]=ubuntu-desktop.local [server]=ubuntu-server.local ) -``` - -Target names may contain letters, numbers, `_`, or `-`. A target named -`ubuntu-server` is configured with map keys like `[ubuntu-server]=...`. - -Endpoints are split per debug method. Fill the one(s) you intend to use: - -```bash -declare -A TARGET_DEBUG_ENDPOINT_KGDB=( - [server]=127.0.0.1:5520 -) +declare -A TARGET_SSH_USER=([desktop]=user [server]=user) +declare -A TARGET_SSH_PASS=([desktop]= [server]=) # empty when using SSH keys declare -A TARGET_DEBUG_ENDPOINT_QEMU=( + [desktop]=127.0.0.1:1234 [server]=127.0.0.1:1234 ) ``` -Either map may be left empty per target. Script 04 errors clearly if the -endpoint for the method you picked is missing. - -The VS Code task pickers prompt for target name as free text, so adding a -target is just an edit to `lab.local.env` — no other edits required for -VS Code to pick it up. +Endpoints are split per debug method +(`TARGET_DEBUG_ENDPOINT_KGDB` and `TARGET_DEBUG_ENDPOINT_QEMU`). Either +map may be left empty per target if you don't use that method. Script 04 +errors clearly if you pick a method without an endpoint. -## Prepare A Target - -Provisioning is target-side setup. It currently supports Ubuntu targets and -takes the debug method as a second argument. - -```bash -./scripts/01-provision-target.sh server kgdb # KGDB path -./scripts/01-provision-target.sh server qemu # QEMU stub path -``` +Target names accept `[A-Za-z][A-Za-z0-9_-]*`. The VS Code task picker +accepts any name you type — no editing of `.vscode/` needed when adding +a target. -Both modes install the running target kernel's headers and `rsync`, and add -`nokaslr` to the kernel command line so vmlinux symbols line up with running -addresses. `kgdb` mode additionally adds: +--- -```text -kgdboc=ttyS0,115200 sysrq_always_enabled=1 -``` +## Debug methods -Reboot the target VM after provisioning. +Provisioning (script 01) and deploy (script 04) both take a +`` argument: -For source debugging with full kernel symbols *and the ability to step into -kernel code* (not just disassembly), provision with debug symbols: +- **`kgdb`** — in-kernel KGDB over the guest's serial port. Script 01 + adds `kgdboc=ttyS0,115200` and `sysrq_always_enabled=1` to the guest's + GRUB command line. Break with `echo g | sudo tee /proc/sysrq-trigger` + on the target. Useful when you cannot change the hypervisor's command + line (VMware Workstation is the common case). -```bash -./scripts/01-provision-target.sh server qemu --debug-symbols -``` +- **`qemu`** — the hypervisor's built-in GDB stub + (QEMU's `-gdb tcp::PORT` / `-s`, VMware's `debugStub.listen.guest64`). + No KGDB in the guest; the hypervisor halts the vCPU directly. -`--debug-symbols` installs both the matching `vmlinux` debug image and the -`linux-source-` package, then extracts the source tarball in place under -`/usr/src/`. Script 02 then syncs both alongside the headers. +Neither method assumes any in-module `kgdb_breakpoint()` call. In both, +you break manually after script 04 finishes. -Sync the target kernel headers, build tree, `vmlinux`, and kernel source -into the development host (script 02 is the same regardless of debug method): +### QEMU gdbstub -```bash -./scripts/02-setup-host-build.sh server +```text +qemu-system-x86_64 ... -gdb tcp::1234 ``` -After sync, `.kernel-cache//source` is a symlink to the synced -kernel source tree (or absent if the source wasn't installed on the target). -Script 04 picks it up automatically. - -## Build - -In VS Code, press `Ctrl+Shift+B`. The default `Kernel: Build` task prompts -for `kernelTarget` and runs: +Use `-S` to pause the vCPU at boot. Without `-S`, the guest runs until +GDB connects. -```bash -./scripts/03-build-module.sh -``` +### VMware debug stub -The module is written to: +Power off the VM, edit the VM's `.vmx`: ```text -build/artifacts///.ko +debugStub.listen.guest64 = "TRUE" +debugStub.port.guest64 = "8864" +debugStub.listen.guest64.remote = "TRUE" +debugStub.hideBreakpoints = "FALSE" ``` -Build intermediates are kept under: +Same shape as the QEMU stub — set +`TARGET_DEBUG_ENDPOINT_QEMU[server]=127.0.0.1:8864`. -```text -build/intermediate/// -``` +### KGDB over serial → TCP -Running plain `make` at the repo root uses `.kernel-cache/current` when it -exists, otherwise the host kernel. The setup script updates that link to -the most recently synced target. +Set up a serial bridge from the guest's `/dev/ttyS0` to a TCP listener +GDB can connect to. How depends on the hypervisor: -## Debug From VS Code - -The VS Code debug tasks share two inputs: `kernelTarget` (free-text prompt) -and `debugMethod` (kgdb|qemu). +- **QEMU:** `-serial tcp:127.0.0.1:5520,server,nowait` +- **VMware Workstation on Windows:** named pipe in the VM's serial port + + `host/bridge-kgdb.ps1`: -1. Select `Kernel: Cppdbg Debug` (or `Kernel: Native Debug`). -2. Press F5. -3. VS Code runs the `Kernel: Deploy Debug` prelaunch task. -4. The task prompts for `kernelTarget` and `debugMethod`. -5. `scripts/04-deploy-debug-vscode.sh ` uploads and loads - your module, prepares module symbols, and writes the current GDB files - under `.gdb/`. -6. VS Code starts GDB and sources `.gdb/current-debug.gdb`. + ```powershell + .\host\bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520 + ``` -The generated GDB script sets the kernel architecture to `i386:x86-64`, -loads the target `vmlinux`, adds module symbols from -`/sys/module//sections/*`, and maps staged Kbuild paths back to the -real files under `module/`. +Provision the target with `kgdb`: -## Stepping Into Kernel Code +```bash +./scripts/01-provision-target.sh server kgdb +``` -With both `vmlinux` debug symbols and the kernel source tree synced (i.e. -script 01 was run with `--debug-symbols` and script 02 has picked up the -extracted source under `/usr/src/linux-source-*`), GDB can show kernel -source on step-into instead of just disassembly. +`TARGET_KGDB_TTY` and `TARGET_KGDB_BAUD` in `lab.local.env` must match +the GRUB args added by provisioning (defaults `ttyS0` / `115200`). -Script 04 discovers Ubuntu's build-time source prefix by asking `addr2line` -where `start_kernel` lives in `vmlinux` — the path is always -`/init/main.c`, so the prefix is whatever comes before the -known suffix. It then emits: +Useful sanity check before F5: -```text -set substitute-path -directory +```bash +nc -vz 127.0.0.1 5520 ``` -into the generated `.gdb` file, where `` is -`.kernel-cache//source`. GDB transparently remaps any DWARF source -references in `vmlinux` to your local copy. +--- -If the kernel source wasn't installed on the target, script 04 prints a -note and skips the substitute-path — module debugging still works, but -stepping into kernel functions shows disassembly. +## Stepping into kernel code -## Debug Endpoint Options +Pass `--debug-symbols` to script 01 to install both the matching +`vmlinux` debug image and the `linux-source-X` package on the target. +Script 02 syncs both, extracts the source on the host, and creates a +stable `.kernel-cache//source` symlink. -GDB only needs an endpoint in this form: +Script 04 then asks `addr2line` where `start_kernel` lives in `vmlinux` +(always `/init/main.c`) and emits the corresponding GDB +remap into the generated `.gdb` file: ```text -: +set substitute-path +directory ``` -Put that value in `TARGET_DEBUG_ENDPOINT_KGDB` or `TARGET_DEBUG_ENDPOINT_QEMU` -in `lab.local.env`, depending on the method you want to use. +That's it — set a breakpoint in `vfs_read` and step through it. -### QEMU gdbstub +This step gracefully degrades. If `vmlinux` is missing, script 04 errors +clearly. If the kernel source wasn't synced, script 04 prints a one-line +note and continues — module debugging still works; you just see +disassembly when stepping into kernel functions. Check +`.gdb/--loader.log` for details when something is off. -Launch QEMU with the built-in stub exposed on a TCP port, e.g.: +--- -```text -qemu-system-x86_64 ... -gdb tcp::1234 -``` +## Undoing target-side changes -(Use `-S` if you want the vCPU paused at boot. Without `-S`, the guest runs -until GDB connects and halts it.) +Script 01 modifies the target's GRUB command line. To remove every boot +arg the lab added (`nokaslr`, `kgdboc=...`, `sysrq_always_enabled=1`, +`maxcpus=...`) and restore a clean kernel command line: ```bash -declare -A TARGET_DEBUG_ENDPOINT_QEMU=( - [server]=127.0.0.1:1234 -) +./scripts/01-provision-target.sh server --uninstall ``` -### VMware Debug Stub +This leaves installed packages alone — only the GRUB args are reverted. +A timestamped backup of `/etc/default/grub` is left on the target +(`/etc/default/grub.kmod-debug-lab..bak`) for paranoid recovery. -Power off the VM, open the target VM's `.vmx` file, and add: - -```text -debugStub.listen.guest64 = "TRUE" -debugStub.port.guest64 = "8864" -debugStub.listen.guest64.remote = "TRUE" -debugStub.hideBreakpoints = "FALSE" -``` +--- -Start the VM after saving the `.vmx` file. The VMware stub is the same shape -as QEMU's gdbstub — use the `qemu` debug method: +## Build outputs -```bash -declare -A TARGET_DEBUG_ENDPOINT_QEMU=( - [server]=127.0.0.1:8864 -) ``` - -### KGDB Over Serial → TCP - -Use this path if you want Linux KGDB over a virtual serial port: - -```text -VS Code -> GDB -> TCP port -> serial bridge -> guest /dev/ttyS0 -> Ubuntu KGDB +build/intermediate/// staged Kbuild tree (recursive symlinks back to module/) +build/artifacts///.ko final module +.kernel-cache//build -> synced linux-headers +.kernel-cache//source -> synced linux-source (when --debug-symbols) +.kernel-cache//vmlinux debug-symbol vmlinux from the target +.kernel-cache/current -> drives bare `make` and IntelliSense +.gdb/-.gdb full GDB init: arch, vmlinux, target remote, sourced symbols +.gdb/--attached.gdb trimmed variant for IDEs that already attach themselves +.gdb/--loader.log output of remote insmod + section discovery +.gdb/current-debug{.gdb,-attached.gdb} symlinks the VS Code launch configs read (re-pointed each F5) ``` -How you build the bridge depends on the hypervisor: +`build/`, `.kernel-cache/`, `.gdb/`, `lab.local.env`, and +`compile_commands.json` are gitignored. -- **QEMU:** `-serial tcp:127.0.0.1:5520,server,nowait` -- **VMware Workstation:** named pipe + `host/bridge-kgdb.ps1`. Configure a - serial port with `Use named pipe`, `This end is the server`, `The other - end is an application`, `Connect at power on`. Then run the bridge - script on the Windows host: - - ```powershell - .\host\bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520 - ``` - -Provision the target with the `kgdb` method and set the matching endpoint: - -```bash -./scripts/01-provision-target.sh server kgdb - -declare -A TARGET_DEBUG_ENDPOINT_KGDB=( - [server]=127.0.0.1:5520 -) - -declare -A TARGET_KGDB_TTY=( - [server]=ttyS0 -) - -declare -A TARGET_KGDB_BAUD=( - [server]=115200 -) -``` +--- -The TTY and baud rate must match the boot arguments added by provisioning. - -## Verify The Debug Endpoint - -From the development host, check the endpoint before launching GDB: +## Troubleshooting -```bash -nc -vz 127.0.0.1 1234 -``` +**`sshpass: command not found`.** You configured `TARGET_SSH_PASS`. On +the dev host: `sudo apt-get install -y sshpass`. Leave +`TARGET_SUDO_PASS` empty when the sudo password matches the SSH +password. -If the hypervisor host can connect but the development host cannot, use an -address for the hypervisor host that the development host can reach instead -of `127.0.0.1`. On WSL, the Windows host address is often listed as the -resolver: +**F5 fails with `target remote ... Connection timed out`.** The endpoint +in `.gdb/current-debug.gdb` isn't reachable from the dev host. Verify +with `nc -vz `. On WSL, the Windows host address is often +in `/etc/resolv.conf`: ```bash grep nameserver /etc/resolv.conf ``` -After changing `lab.local.env`, run F5 again so script 04 regenerates the -GDB files under `.gdb/`. - -## Troubleshooting +After editing `lab.local.env`, re-run F5 so script 04 regenerates the +`.gdb/` files. -If a script says `sshpass` is missing and you use SSH passwords, install it -on the development host: +**GDB connects but the module never loads.** Check +`.gdb/--loader.log` for the insmod error. You can also +SSH in and try the insmod by hand: ```bash -sudo apt-get install -y sshpass +ssh sudo insmod /tmp/kmod-debug-lab/.ko +ssh sudo dmesg | tail -40 ``` -Leave `TARGET_SUDO_PASS[target]` empty when the sudo password is the same -as the SSH password. +**Breakpoints bind to files under `build/intermediate/...`.** Re-run F5 +so `.gdb/current-debug.gdb` is regenerated with the correct +`substitute-path`. Don't edit `.gdb/` by hand — it's regenerated every +F5. -If F5 fails with `target remote ... Connection timed out`, the endpoint in -`.gdb/current-debug.gdb` is not reachable from the development host. Check -`TARGET_DEBUG_ENDPOINT_KGDB[target]` or `TARGET_DEBUG_ENDPOINT_QEMU[target]` -(depending on the method you picked) and verify that your endpoint provider -is listening. +**VS Code shows include squiggles for `linux/module.h`.** Script 02 +hasn't synced the headers yet, or you switched targets and IntelliSense +is caching old paths. Re-run `scripts/02-setup-host-build.sh ` +and `C/C++: Reset IntelliSense Database` from the command palette. -If GDB connects but the module does not load, inspect: +**`/sys/module//sections/.text` didn't appear within 30s.** Your +module either failed to load, or it's slow to init on the target. The +loader log will show `insmod` output. Bump the timeout with +`INSMOD_WAIT_SECS=60 ./scripts/04-... ...` if needed. -```bash -cat .gdb/--loader.log -``` +**GDB shows `Cannot access memory at address …` when stepping into the +kernel.** Either `vmlinux` is missing (re-run `scripts/01-... --debug-symbols` +then `scripts/02-...`) or the cached `vmlinux` belongs to a different +kernel than the one running on the target. The latter is caught by +script 04's stale-kernel guard, but if you bypassed it, re-sync. -If breakpoints bind to files under `build/intermediate/...`, regenerate the -GDB files by pressing F5 again. The generated script maps those staged paths -back to `module/` with `set substitute-path`. +**Spaces in your repo path.** Clone to a path WITHOUT spaces. GDB's +`set substitute-path FROM TO` splits on whitespace, so a `repo_root` like +`/home/me/path with spaces/...` produces a substitute-path that gets +parsed as four arguments and silently fails to remap module sources. +Kbuild is also notoriously fragile with spaces in paths. -## VS Code Include Errors +--- -The C/C++ extension reads `.vscode/c_cpp_properties.json`. It expects target -kernel headers under `.kernel-cache//build`, which are created by: +## Tool versions -```bash -./scripts/02-setup-host-build.sh -``` +Tested with: -Before that sync runs, VS Code can show include squiggles for kernel -headers such as `linux/module.h` or `linux/fs.h`. If the squiggles remain -after syncing headers, run `C/C++: Reset IntelliSense Database` from the -command palette. +- **Dev host:** Ubuntu 24.04, bash 5+, GNU coreutils, GDB ≥ 10, gcc 13. +- **Target VM:** Ubuntu 22.04 / 24.04, kernel 5.15 / 6.8. +- The `cp -as` recursive-symlink staging assumes GNU coreutils on the dev + host; macOS would need `brew install coreutils` and a `CP=gcp` override. +- `set substitute-path` for kernel-source remapping needs GDB ≥ 7.5 + (every supported distro ships much newer). +- `-ffile-prefix-map` (DWARF rewriting) requires GCC ≥ 8 or Clang ≥ 7. -`KBUILD_MODNAME` is set to a placeholder (`"kmod"`) in the IntelliSense -defines. The actual value at compile time is your real module name; the -placeholder is only there so the C/C++ extension can resolve macros that -reference it. +--- -## Repository Layout +## Adding a new target OS -```text -module/ # your module project lives here -examples/chuck_norise/ # worked example you can copy into module/ -scripts/01-provision-target.sh # target-side: headers, grub args -scripts/02-setup-host-build.sh # host-side: sync headers + vmlinux -scripts/03-build-module.sh # host-side: build via Kbuild -scripts/04-deploy-debug-vscode.sh # host-side: upload, insmod, gen .gdb -scripts/lib/common.sh # shared bash helpers -host/bridge-kgdb.ps1 # Windows-side KGDB serial-to-TCP bridge -host/restore-snapshot.ps1 # Windows-side VMware snapshot helper -Makefile # generic out-of-tree wrapper -lab.example.env # copy to lab.local.env and edit -.vscode/ # tasks, launch, IntelliSense -``` +Currently only `TARGET_OS[*]=ubuntu` is implemented in scripts 01 and 02. +Adding (say) Fedora means teaching script 01 how to install headers / +dbgsym / source via `dnf`, and teaching script 02 how to discover the +header tree paths under `/usr/src/`. Both scripts check `target_os` near +the top and fail fast on unknown values — that's where to add a backend. +PRs welcome. + +--- + +## License -The top-level scripts and `Makefile` are the lab's tooling — generic across -modules. The optional PowerShell scripts under `host/` are helpers for -VMware Workstation on Windows. +MIT — see [LICENSE](LICENSE). Your module under `module/` is yours; the +lab's license does not impose terms on what you build. diff --git a/examples/chuck_norise/README.md b/examples/chuck_norise/README.md index 0576a73..92b2a7d 100644 --- a/examples/chuck_norise/README.md +++ b/examples/chuck_norise/README.md @@ -13,15 +13,16 @@ userspace buffer" routine. ## Use it as a template The repo's build pipeline operates on whatever lives under `module/` at the -repo root. To try this example end-to-end, copy it into place: +repo root. To try this example end-to-end, copy it into place from the +repo root: ```bash -cp -r examples/chuck_norise/. module/ +make use-example NAME=chuck_norise ``` -Then run the normal lab flow from the repo root: `scripts/01-...` through -`scripts/04-...`, or press F5 in VS Code. After the module loads on the -target VM: +Then run the normal lab flow (`scripts/00-...` through `scripts/04-...`, +or press F5 in VS Code — see the top-level [README](../../README.md) for +the full quickstart). After the module loads on the target VM: ```bash head -c 10 /dev/chuck_norise # prints "chuck nori" diff --git a/examples/chuck_norise/src/chuck_device.c b/examples/chuck_norise/src/chuck_device.c index 4705ac9..cb6758d 100644 --- a/examples/chuck_norise/src/chuck_device.c +++ b/examples/chuck_norise/src/chuck_device.c @@ -33,7 +33,13 @@ static const struct file_operations chuck_fops = { .owner = THIS_MODULE, .open = chuck_open, .read = chuck_read, - .llseek = no_llseek, + /* + * .llseek deliberately unset. The kernel installs default_llseek for + * char devices, which matches chuck_read's behavior (it advances and + * respects *position). Earlier versions of this file pinned + * .llseek = no_llseek, but that symbol was removed in Linux 6.12 + * (mainline commit 868941b14441 "fs: remove no_llseek"). + */ }; static struct class *chuck_class_create(void) diff --git a/lab.example.env b/lab.example.env index aaa62da..498bfd6 100644 --- a/lab.example.env +++ b/lab.example.env @@ -1,7 +1,9 @@ # Copy this file to lab.local.env and edit it for your machines. +# lab.local.env is gitignored; lab.example.env is the only tracked copy. -# Common local tools on the Debian-based development host. -GDB_BIN=gdb +# Local tools on the Debian-based development host. The defaults below +# expect each to be on $PATH; override here only if you need an absolute +# path or an alternate binary. SSH_BIN=ssh SCP_BIN=scp RSYNC_BIN=rsync @@ -39,13 +41,19 @@ declare -A TARGET_SSH_USER=( [server]=user ) -# Leave empty when using SSH keys. +# SSH password. Leave empty when using SSH keys (recommended for anything +# beyond throwaway dev VMs). The lab pipes any configured password into ssh +# via SSHPASS; it is not written to disk and not visible in `ps` on either +# end. declare -A TARGET_SSH_PASS=( [desktop]= [server]= ) -# Leave empty to reuse TARGET_SSH_PASS for sudo. +# Sudo password. Leave empty to reuse TARGET_SSH_PASS. The lab pipes this +# into `sudo -S` over ssh's stdin, so it is not visible in `ps` on the +# target. For long-lived dev targets, configure NOPASSWD for the lab user +# (` ALL=(ALL) NOPASSWD: ALL` in sudoers) and leave this empty. declare -A TARGET_SUDO_PASS=( [desktop]= [server]= @@ -53,9 +61,11 @@ declare -A TARGET_SUDO_PASS=( # Remote staging directory used for uploading the built module. Any writable # path is fine; /tmp/* gets cleaned on reboot which is usually what you want. +# Leave empty to use the per-user default `/tmp/kmod-debug-lab-`, +# which avoids collisions when multiple devs share a target host. declare -A TARGET_REMOTE_DIR=( - [desktop]=/tmp/kmod-debug-lab - [server]=/tmp/kmod-debug-lab + [desktop]= + [server]= ) # Debug endpoints — one per debug method. Scripts 01 and 04 take a @@ -89,9 +99,20 @@ declare -A TARGET_KGDB_BAUD=( [server]=115200 ) -# Module sleeps this long at the start of init, IF it exposes the -# `debug_delay_ms` module parameter. The lab uses this convention so the host -# can read /sys/module//sections/* and so users have a window to break -# in GDB (Ctrl+C against the qemu stub, sysrq+g for kgdb) before init runs. -# Modules that do not declare the parameter get loaded without it; no harm. +# When your module declares `module_param(debug_delay_ms, uint, 0644);`, +# scripts/04 passes this value (in milliseconds) on `insmod`. Your +# `module_init` should sleep that long so the host can read +# /sys/module//sections/* and you have a window to break in GDB +# (Ctrl+C against the qemu stub, sysrq+g for kgdb) before init runs. +# Modules that do not declare the parameter get loaded plainly; no harm. DEBUG_LOAD_DELAY_MS=5000 + +# Optional. Sets `maxcpus=` on the target's GRUB command line — useful +# when you want to pin debugging to a known CPU count (e.g. 1 to avoid +# CPU migration confusing breakpoints). Unset to leave it off. +#DEBUG_MAXCPUS=1 + +# Optional. Override how long scripts/04 polls for the loaded module's +# section addresses to appear under /sys/module//sections/. +# Increase on slow target VMs. Default 30. +#INSMOD_WAIT_SECS=30 diff --git a/module/README.md b/module/README.md index 0576a73..3026dcd 100644 --- a/module/README.md +++ b/module/README.md @@ -1,52 +1,75 @@ -# chuck_norise — example module +# module/ — your kernel module goes here -A small multi-file char device module used as the worked example for this lab. -Builds as `chuck_norise.ko` and exposes `/dev/chuck_norise`. Reads return the -exact string `chuck norise!` repeated indefinitely, preserving the file offset -for each open file descriptor. +This directory is the lab's only slot for the kernel module being built and +debugged. The repo is otherwise generic — it does not know your module's +name, source layout, or what it does. -It is intentionally tiny so it stays useful as a template walk-through: an init -that sleeps long enough to give you time to attach GDB, a cdev/class lifecycle -split into its own file, and a separately-testable "produce bytes into a -userspace buffer" routine. +## Contract -## Use it as a template +Drop a standard out-of-tree Linux kernel module project here. The lab +expects: -The repo's build pipeline operates on whatever lives under `module/` at the -repo root. To try this example end-to-end, copy it into place: +- **`module/Makefile`** (or `module/Kbuild`) following standard Kbuild + conventions for out-of-tree modules: -```bash -cp -r examples/chuck_norise/. module/ -``` + ```makefile + obj-m += my_module.o + my_module-y := src/main.o src/util.o + ccflags-y := -I$(src)/include -g -DDEBUG + ``` + + See `examples/chuck_norise/Makefile` for the canonical shape, including + the optional `ifndef KERNELRELEASE` wrapper that lets the same file be + driven by `make` directly. + +- **Sources anywhere you like.** The lab does not impose a `src/` or + `include/` layout — your Makefile's `*-y` line picks the source files + and `ccflags-y` picks the include paths. + +- **Exactly one `obj-m` entry per build.** The lab discovers the module + name from the produced `.ko`, so you don't declare it anywhere + else. + +- **Optionally a `debug_delay_ms` module parameter** that sleeps in + `module_init` so the host has time to attach GDB before init runs. + Declared as: -Then run the normal lab flow from the repo root: `scripts/01-...` through -`scripts/04-...`, or press F5 in VS Code. After the module loads on the -target VM: + ```c + static unsigned int debug_delay_ms = 5000; + module_param(debug_delay_ms, uint, 0644); + ``` + + When declared, scripts/04 passes `DEBUG_LOAD_DELAY_MS` from + `lab.local.env` to `insmod`. When not declared, the module loads + without it. + +That's the entire contract. The lab handles per-target staging, DWARF +path rewriting, uploading, loading, and symbol discovery on the target. + +## Try the example ```bash -head -c 10 /dev/chuck_norise # prints "chuck nori" +make use-example NAME=chuck_norise ``` -To see offset-preserving reads from a single open fd: +Then build and debug as in the top-level `README.md`. To reset: ```bash -exec 9/dev/null # chu -dd bs=1 count=5 <&9 2>/dev/null # ck no -dd bs=1 count=7 <&9 2>/dev/null # rise!ch -exec 9<&- +make clean-module ``` -## What this example demonstrates - -- A standard out-of-tree Linux kernel module project layout (`src/`, `include/`, - `Makefile`). -- A `Makefile` that doubles as a Kbuild fragment and as a standalone wrapper — - the conventional shape for kernel modules. The lab's outer build invokes it - the Kbuild way; running `make` in this directory directly invokes the wrapper. -- A module init that sleeps for `debug_delay_ms` (default 5000 ms, exposed as - a module parameter) before doing anything observable. The delay gives the - host time to attach GDB and set breakpoints before init runs. -- A `LINUX_VERSION_CODE` shim for the 6.4 `class_create()` signature change, - as a real-world example of one of the things out-of-tree modules have to - carry. +## Replacing the example with your own module + +Run `make clean-module`, then drop your project in. Edit your `Makefile` +to set `obj-m`, `-y`, and `ccflags-y`. Rebuild. + +## One caveat: no symlinks inside `module/` + +The lab stages your sources into `build/intermediate///` +as a tree of absolute symlinks (`cp -as`) and then runs Kbuild from +there. If `module/` contains its OWN symlinks, the staged copy ends up +with symlinks pointing at the original symlinks — Kbuild will still find +the files, but the DWARF prefix-map remap (which assumes source files +live under `module/`) may not produce the paths your IDE expects. Keep +`module/` symlink-free, or copy in real files for any external sources +you depend on. diff --git a/scripts/00-check-target.sh b/scripts/00-check-target.sh new file mode 100755 index 0000000..a286470 --- /dev/null +++ b/scripts/00-check-target.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Non-mutating health check for a target. Run this before scripts/01 to +# catch configuration mistakes before any state on the target changes. +# +# Reports, one line each: +# ssh is the target reachable over ssh? +# sudo does the configured (or fallback) sudo work? +# os distro and codename (must be ubuntu for now) +# kernel running kernel release +# headers /lib/modules/$kernel/build present? +# vmlinux debug-symbol vmlinux available? +# kernel-source linux-source-* package extracted? +# debug endpoint TCP port for the picked debug method reachable from the dev host? +# +# A "debug method" argument is optional; if omitted, both kgdb and qemu +# endpoints are checked when configured. +set -euo pipefail + +usage() { + echo "usage: $0 [kgdb|qemu]" >&2 + exit 2 +} + +target="${1:-}" +debug_method="${2:-}" +case "$debug_method" in ""|kgdb|qemu) ;; *) usage ;; esac + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 [kgdb|qemu]" + +env_file="$(lab_env_file "$repo_root")" +# shellcheck source=/dev/null +source "$env_file" +lab_load_target "$target" + +# Colored OK / FAIL / WARN, falling back to plain text when stdout isn't a TTY. +if [[ -t 1 ]]; then + GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[0;33m'; RESET='\033[0m' +else + GREEN=''; RED=''; YELLOW=''; RESET='' +fi +ok() { printf " ${GREEN}OK${RESET} %s\n" "$1"; } +fail() { printf " ${RED}FAIL${RESET} %s\n" "$1"; failures=$((failures+1)); } +warn() { printf " ${YELLOW}WARN${RESET} %s\n" "$1"; } +failures=0 + +echo "checking $LAB_TARGET ($LAB_SSH_TARGET:$LAB_SSH_PORT)" + +# --- ssh --- +if lab_ssh 'true' 2>/dev/null; then + ok "ssh: reachable" +else + fail "ssh: cannot reach $LAB_SSH_TARGET on port $LAB_SSH_PORT" + echo + echo "fix one of:" + echo " - TARGET_SSH_HOST/PORT/USER for '$target' in lab.local.env" + echo " - the target VM is up and openssh-server is running" + echo " - if password auth: TARGET_SSH_PASS[$target] is set" + exit 1 +fi + +# --- sudo --- +if lab_ssh_sudo 'true' 2>/dev/null; then + ok "sudo: works" +else + fail "sudo: configured password rejected or sudo not installed" + echo + echo "fix one of:" + echo " - TARGET_SUDO_PASS[$target] in lab.local.env (defaults to TARGET_SSH_PASS)" + echo " - configure NOPASSWD for the lab user on the target" +fi + +# --- os --- +# shellcheck disable=SC2016 # $ID and $VERSION_CODENAME are intentionally evaluated on the remote. +remote_os="$(lab_ssh '. /etc/os-release 2>/dev/null && printf "%s %s" "$ID" "${VERSION_CODENAME:-unknown}"' || echo unknown)" +case "$remote_os" in + "ubuntu "*) ok "os: $remote_os" ;; + *) fail "os: $remote_os (only ubuntu targets are implemented)" ;; +esac + +# --- kernel --- +live_kernel="$(lab_ssh 'uname -r' 2>/dev/null || echo unknown)" +ok "kernel: $live_kernel" + +cache_dir="$repo_root/.kernel-cache/$target" +cached_kernel="$(cat "$cache_dir/kernel.release" 2>/dev/null || echo)" +if [[ -n "$cached_kernel" && "$cached_kernel" != "$live_kernel" ]]; then + warn "cached kernel ($cached_kernel) differs from live; re-run scripts/02-setup-host-build.sh $target" +fi + +# --- headers --- +if lab_ssh "test -d /lib/modules/$live_kernel/build" 2>/dev/null; then + ok "headers: /lib/modules/$live_kernel/build present" +else + warn "headers: missing on target; run scripts/01-provision-target.sh $target " +fi + +# --- vmlinux (debug image) --- +if lab_ssh_sudo "test -r /usr/lib/debug/boot/vmlinux-$live_kernel" 2>/dev/null; then + ok "vmlinux: /usr/lib/debug/boot/vmlinux-$live_kernel readable (debug symbols installed)" +else + warn "vmlinux: missing on target; for source debugging, re-run scripts/01-... with --debug-symbols" +fi + +# --- kernel source --- +src_found="$(lab_ssh 'ls -1d /usr/src/linux-source-*/ 2>/dev/null | head -1 || true' 2>/dev/null)" +if [[ -n "$src_found" ]]; then + ok "kernel-source: ${src_found%/} present (step-into-kernel will resolve source)" +else + warn "kernel-source: missing on target; for step-into-kernel, re-run scripts/01-... with --debug-symbols" +fi + +# --- debug endpoint(s) reachable from this host --- +check_endpoint() { + local label="$1" key="$2" + local endpoint + endpoint="$(target_cfg "$target" "$key")" + if [[ -z "$endpoint" ]]; then + warn "$label: TARGET_${key}[$target] is unset (skip if you don't use this method)" + return + fi + local host="${endpoint%:*}" port="${endpoint##*:}" + if command -v nc >/dev/null 2>&1; then + if nc -z -w 3 "$host" "$port" 2>/dev/null; then + ok "$label: $endpoint reachable from this host" + else + warn "$label: $endpoint NOT reachable from this host (start your GDB stub / serial bridge)" + fi + else + warn "$label: cannot test (nc not installed); endpoint configured as $endpoint" + fi +} + +case "$debug_method" in + kgdb) check_endpoint "debug-kgdb" DEBUG_ENDPOINT_KGDB ;; + qemu) check_endpoint "debug-qemu" DEBUG_ENDPOINT_QEMU ;; + "") + check_endpoint "debug-kgdb" DEBUG_ENDPOINT_KGDB + check_endpoint "debug-qemu" DEBUG_ENDPOINT_QEMU + ;; +esac + +echo +if [[ $failures -eq 0 ]]; then + echo "all checks passed." + echo "next: scripts/01-provision-target.sh $target [--debug-symbols]" +else + echo "$failures check(s) FAILED; resolve before running scripts/01." + exit 1 +fi diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index 0e7895a..c01de37 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -1,125 +1,206 @@ #!/usr/bin/env bash +# Provision a target VM for kernel module debugging. +# +# Target-side setup: install kernel headers (so the host build can sync them), +# optionally install matching vmlinux debug image + kernel source for +# step-into-kernel, and update the guest's GRUB command line with the boot +# args this lab needs (nokaslr always; kgdboc + sysrq for the kgdb method). +# +# With --uninstall, remove every boot arg the lab added (nokaslr / kgdboc / +# sysrq_always_enabled / maxcpus) — but leave installed packages alone. +# +# Reboot the target after this script completes so the new boot args take +# effect. The script itself does not reboot — staying out of the user's way +# matters for VMs that are reverted from snapshots and shouldn't be touched. set -euo pipefail usage() { echo "usage: $0 [--debug-symbols]" >&2 + echo " $0 --uninstall" >&2 exit 2 } target="${1:-}" -debug_method="${2:-}" -symbols="${3:-}" -case "$debug_method" in kgdb|qemu) ;; *) usage ;; esac -case "$symbols" in ""|--debug-symbols) ;; *) usage ;; esac +arg2="${2:-}" +arg3="${3:-}" + +# Two modes: normal provision ( [--debug-symbols]) and +# --uninstall (which doesn't need a debug method since it only undoes GRUB). +uninstall=0 +debug_method="" +symbols="" +case "$arg2" in + --uninstall) + uninstall=1 + [[ -z "$arg3" ]] || usage + ;; + kgdb|qemu) + debug_method="$arg2" + case "$arg3" in + ""|--debug-symbols) symbols="$arg3" ;; + *) usage ;; + esac + ;; + *) usage ;; +esac repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" validate_target "$target" "usage: $0 [--debug-symbols]" -env_file="$repo_root/lab.local.env" -[[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" + +env_file="$(lab_env_file "$repo_root")" # shellcheck source=/dev/null source "$env_file" -target_is_configured "$target" - -ssh_bin="${SSH_BIN:-ssh}" -sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(target_require_cfg "$target" SSH_HOST)" -ssh_port="$(target_cfg "$target" SSH_PORT)" -ssh_user="$(target_require_cfg "$target" SSH_USER)" -ssh_pass="$(target_cfg "$target" SSH_PASS)" -sudo_pass="$(target_cfg "$target" SUDO_PASS)" +lab_load_target "$target" + target_os="$(target_cfg "$target" OS)" kgdb_tty="$(target_cfg "$target" KGDB_TTY)" kgdb_baud="$(target_cfg "$target" KGDB_BAUD)" - -ssh_port="${ssh_port:-22}" target_os="${target_os:-ubuntu}" kgdb_tty="${kgdb_tty:-ttyS0}" kgdb_baud="${kgdb_baud:-115200}" -sudo_pass="${sudo_pass:-$ssh_pass}" -sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" case "$target_os" in ubuntu) ;; *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; esac -if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it on the build host: sudo apt-get install -y sshpass" >&2 - exit 1 -fi +# Preflight: ssh works. We don't insist sudo works yet because the user +# might be running 01 *to* configure sudo. The actual provisioning shell +# (a piped `sudo -S bash -s`) will exercise sudo and fail with apt's own +# diagnostics if the password is wrong. +lab_check_connection --no-sudo -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port") -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") +if (( uninstall )); then + echo "uninstalling lab GRUB args from $target ($LAB_SSH_TARGET:$LAB_SSH_PORT)" +else + symbols_blurb="${symbols:+, with debug symbols}" + echo "provisioning $target ($LAB_SSH_TARGET:$LAB_SSH_PORT) for debug method: $debug_method$symbols_blurb" fi -echo "provisioning $target at $ssh_user@$ssh_host:$ssh_port (debug method: $debug_method)" - -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" -t "$ssh_user@$ssh_host" \ - "TARGET_OS='$target_os' DEBUG_METHOD='$debug_method' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$sudo_pass_b64' DEBUG_MAXCPUS='${DEBUG_MAXCPUS:-}' bash -s" <<'REMOTE' -set -euo pipefail - -sudo_run() { - sudo "$@" -} - -sudo_auth() { - if [[ -n "${SUDO_PASS_B64:-}" ]]; then - printf '%s\n' "$(printf '%s' "$SUDO_PASS_B64" | base64 -d)" | sudo -S -v - else - sudo -v - fi +# Run the remote script as root via SUDO_ASKPASS so stdin stays a clean +# pipe of "env-vars then script body" for bash -s. +# +# Why NOT `sudo -S` + piped password on stdin: sudo only reads stdin when +# policy requires it. With NOPASSWD configured, or cached creds, sudo +# skips the stdin read entirely and the password line we piped leaks into +# bash -s as the first script line — producing +# `bash: line 1: : command not found`. `sudo -k` doesn't help +# because NOPASSWD is a policy override, not a cache thing. +# +# SUDO_ASKPASS works regardless: sudo invokes the askpass helper to fetch +# the password (or doesn't, under NOPASSWD), and stdin is purely the +# bash -s script. We scp the helper + a chmod-600 password file to the +# target up front; a trap cleans both up on script exit. +askpass="$LAB_REMOTE_DIR/.kmod-askpass-$$" +pwfile="$LAB_REMOTE_DIR/.kmod-sudo-pw-$$" +cleanup_sudo_helpers() { + lab_ssh "rm -f '$askpass' '$pwfile' 2>/dev/null" || true } +trap cleanup_sudo_helpers EXIT + +lab_ssh "mkdir -p '$LAB_REMOTE_DIR'; umask 077; cat > '$pwfile'" <<<"$LAB_SUDO_PASS" +lab_ssh "umask 077; cat > '$askpass'; chmod 700 '$askpass'" <&2; exit 1; } -[[ "${ID:-}" == "ubuntu" ]] || { echo "target is not Ubuntu" >&2; exit 1; } -sudo_auth +[[ "${TARGET_OS:-ubuntu}" == "ubuntu" ]] || { + echo "unsupported target OS: ${TARGET_OS:-}" >&2; exit 1; } +[[ "${ID:-}" == "ubuntu" ]] || { + echo "target is not Ubuntu (os-release ID=$ID); only ubuntu targets are implemented" >&2; exit 1; } -echo "installing packages for running kernel $kernel" -apt_retry update -apt_retry install -y linux-headers-"$kernel" rsync -[[ -d "/lib/modules/$kernel/build" ]] || { - echo "missing /lib/modules/$kernel/build after installing headers" >&2 - exit 1 +grub_file=/etc/default/grub +grub_managed_keys='kgdboc=|maxcpus=|sysrq_always_enabled=|nokaslr' + +# Read current GRUB_CMDLINE_LINUX_DEFAULT, stripping the lab's managed keys +# so reruns replace rather than append. `nokaslr` is a bare token (no =), +# matched as a standalone word. +read_current_cmdline() { + local raw stripped + raw="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" + # Strip key=value forms and the bare "nokaslr" token; collapse spaces. + stripped="$(printf '%s' "$raw" | sed -E "s/(^| )($grub_managed_keys)([^ ]*)?/ /g; s/ */ /g; s/^ +//; s/ +$//")" + printf '%s' "$stripped" } -vmlinux="/usr/lib/debug/boot/vmlinux-${kernel}" +write_cmdline() { + local newline="$1" + cp "$grub_file" "$grub_file.kmod-debug-lab.$(date +%Y%m%d%H%M%S).bak" + if grep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' "$grub_file"; then + sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$newline\"|" "$grub_file" + else + printf 'GRUB_CMDLINE_LINUX_DEFAULT="%s"\n' "$newline" >> "$grub_file" + fi + update-grub +} + +if [[ "${UNINSTALL:-0}" == "1" ]]; then + echo "stripping lab boot args from $grub_file" + current="$(read_current_cmdline)" + write_cmdline "$current" + echo + echo "done. reboot to drop the lab boot args." + echo " boot args now: $current" + exit 0 +fi + +echo "[1/3] installing kernel headers for running kernel $kernel" +apt_retry update +apt_retry install -y "linux-headers-$kernel" rsync +[[ -d "/lib/modules/$kernel/build" ]] || + { echo "missing /lib/modules/$kernel/build after installing headers; check apt output above" >&2; exit 1; } + if [[ "$INSTALL_DEBUG_SYMBOLS" == "--debug-symbols" ]]; then + echo "[2/3] installing vmlinux debug image + kernel source for step-into-kernel" + vmlinux="/usr/lib/debug/boot/vmlinux-$kernel" if [[ ! -r "$vmlinux" ]]; then apt_retry install -y ubuntu-dbgsym-keyring - cat </dev/null -deb http://ddebs.ubuntu.com ${codename} main restricted universe multiverse + cat > /etc/apt/sources.list.d/ddebs.list <&2 - echo "retry later if ddebs.ubuntu.com is returning 503" >&2 - exit 1 - } + apt_retry install -y "linux-image-unsigned-${kernel}-dbgsym" || + { echo "failed to install debug symbols for $kernel; retry later if ddebs.ubuntu.com is returning 503" >&2; exit 1; } fi + [[ -r "$vmlinux" ]] || echo "warning: $vmlinux still missing after install; kernel source debugging will be unavailable" - # Kernel source. Ubuntu's linux-source- package drops a - # tarball under /usr/src/. We don't extract on the target — script 02 - # rsyncs the tarball back to the host and extracts there, so the target - # stays unmodified beyond the package install. With the source available - # locally, GDB can step into kernel functions instead of just - # disassembling them. + # Try the major.minor.patch-suffixed package first (e.g. linux-source-6.8.0); + # fall back to the unversioned meta-package. Extraction is deferred to + # script 02 — the host has more incentive to extract (it's where GDB lives) + # and doing it there avoids needing tar / sudo on the target. short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" src_pkg="" - for candidate in "linux-source-${short_kver}" "linux-source"; do + for candidate in "linux-source-$short_kver" "linux-source"; do [[ -z "$candidate" || "$candidate" == "linux-source-" ]] && continue if apt_retry install -y "$candidate"; then src_pkg="$candidate" @@ -127,39 +208,28 @@ EOF fi done if [[ -z "$src_pkg" ]]; then - echo "warning: could not install a linux-source package; step-into-kernel won't have source files" >&2 + echo "warning: could not install a linux-source package; step-into-kernel will only show disassembly" >&2 + elif ls -1 /usr/src/linux-source-*.tar.* /usr/src/linux-source-*/linux-source-*.tar.* 2>/dev/null | head -1 >/dev/null; then + echo " kernel source tarball is in /usr/src on the target; script 02 will sync and extract it" else - # Confirm the tarball is somewhere on disk for script 02 to find. - # Ubuntu has shipped both /usr/src/linux-source-X.tar.bz2 and - # /usr/src/linux-source-X/linux-source-X.tar.bz2 layouts over time — - # we accept either. - if ls -1 /usr/src/linux-source-*.tar.* /usr/src/linux-source-*/linux-source-*.tar.* 2>/dev/null | head -1 >/dev/null; then - echo "kernel source tarball is in /usr/src on the target; script 02 will sync and extract on the host" - else - echo "warning: $src_pkg installed but no linux-source tarball found under /usr/src/" >&2 - fi + echo "warning: $src_pkg installed but no linux-source tarball found under /usr/src/" >&2 fi +else + echo "[2/3] skipping debug symbols (pass --debug-symbols to enable step-into-kernel)" fi -[[ -r "$vmlinux" ]] || echo "note: $vmlinux is missing; VS Code source debugging may need it" -grub_file=/etc/default/grub -current="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" -# Strip any prior values for the keys we manage so reruns with changed values -# replace rather than append. kgdboc and sysrq_always_enabled are also stripped -# so switching from kgdb to qemu mode removes them. -current="$(printf '%s' "$current" | sed -E 's/(^| )(kgdboc=|maxcpus=|sysrq_always_enabled=)[^ ]*//g; s/ */ /g; s/^ +//; s/ +$//')" +echo "[3/3] updating GRUB command line" +current="$(read_current_cmdline)" -# nokaslr is required in both modes so vmlinux symbols line up with running addresses. +# nokaslr is required in both modes so vmlinux symbols line up with running +# addresses. kgdb mode also needs an in-kernel debugger channel (kgdboc) and +# sysrq enabled so users can trigger a halt with `echo g > /proc/sysrq-trigger` +# — there is no kgdb_breakpoint() call in any user module to halt on insmod. +# qemu mode skips both: the hypervisor stub halts the vCPU directly. args=("nokaslr") - -# kgdb mode also needs an in-kernel debugger channel (kgdboc) and sysrq enabled -# so users can trigger a halt with "echo g > /proc/sysrq-trigger" — there is no -# longer a kgdb_breakpoint() in the module to halt on insmod. qemu mode skips -# both: the hypervisor stub halts the vCPU directly. if [[ "$DEBUG_METHOD" == "kgdb" ]]; then args+=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "sysrq_always_enabled=1") fi - if [[ -n "${DEBUG_MAXCPUS:-}" ]]; then args+=("maxcpus=${DEBUG_MAXCPUS}") fi @@ -167,13 +237,18 @@ for arg in "${args[@]}"; do case " $current " in *" $arg "*) ;; *) current="${current:+$current }$arg" ;; esac done -sudo_run cp "$grub_file" "$grub_file.small-ko.$(date +%Y%m%d%H%M%S).bak" -if grep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' "$grub_file"; then - sudo_run sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$current\"|" "$grub_file" +write_cmdline "$current" + +echo +echo "done. reboot the target VM before debugging." +echo " boot args: $current" +REMOTE_SCRIPT +} | SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "SUDO_ASKPASS='$askpass' sudo -A bash -s" + +echo +if (( uninstall )); then + echo "uninstall complete. reboot the $target VM to drop the lab boot args." else - echo "GRUB_CMDLINE_LINUX_DEFAULT=\"$current\"" | sudo_run tee -a "$grub_file" >/dev/null + echo "provisioning complete. reboot the $target VM, then:" + echo "next: scripts/02-setup-host-build.sh $target" fi -sudo_run update-grub - -echo "done. reboot the VM before debugging. boot args: $current" -REMOTE diff --git a/scripts/02-setup-host-build.sh b/scripts/02-setup-host-build.sh index 8a7ea71..bb63dbd 100755 --- a/scripts/02-setup-host-build.sh +++ b/scripts/02-setup-host-build.sh @@ -1,4 +1,17 @@ #!/usr/bin/env bash +# Sync the target's kernel build tree to the host so an out-of-tree module +# can be cross-built against the target's exact kernel without copying the +# whole source tree per build. +# +# Result: .kernel-cache// populated with: +# build/ -> symlink into the synced linux-headers tree +# source/ -> symlink into the synced linux-source tree (if --debug-symbols was used in 01) +# vmlinux -> debug-symbol vmlinux from the target (if available) +# kernel.release -> running kernel release (e.g. 6.8.0-117-generic) +# remote.build.path -> where headers came from on the target +# remote.header.paths -> list of all paths synced from /usr/src/ +# Also updates .kernel-cache/current -> so plain `make` and the +# IntelliSense config track the most recently synced target. set -euo pipefail target="${1:-}" @@ -7,73 +20,33 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" validate_target "$target" "usage: $0 " -env_file="$repo_root/lab.local.env" -[[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" +require_debian_host + +env_file="$(lab_env_file "$repo_root")" # shellcheck source=/dev/null source "$env_file" -require_debian_host -target_is_configured "$target" - -ssh_bin="${SSH_BIN:-ssh}" -rsync_bin="${RSYNC_BIN:-rsync}" -sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(target_require_cfg "$target" SSH_HOST)" -ssh_port="$(target_cfg "$target" SSH_PORT)" -ssh_user="$(target_require_cfg "$target" SSH_USER)" -ssh_pass="$(target_cfg "$target" SSH_PASS)" -sudo_pass="$(target_cfg "$target" SUDO_PASS)" +lab_load_target "$target" + target_os="$(target_cfg "$target" OS)" -ssh_port="${ssh_port:-22}" -sudo_pass="${sudo_pass:-$ssh_pass}" -sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" target_os="${target_os:-ubuntu}" - case "$target_os" in ubuntu) ;; *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; esac -if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it on the build host: sudo apt-get install -y sshpass" >&2 - exit 1 -fi - -ssh_target="$ssh_user@$ssh_host" -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -rsync_rsh="$ssh_bin -o StrictHostKeyChecking=accept-new -p $ssh_port" -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") - rsync_rsh="$sshpass_bin -e $ssh_bin -o StrictHostKeyChecking=accept-new -p $ssh_port" -fi - -remote_sudo_prefix() { - if [[ -n "$sudo_pass_b64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$sudo_pass_b64" - else - printf "sudo " - fi -} - -apt_update() { - sudo apt-get \ - -o Acquire::Retries=3 \ - -o Acquire::http::Timeout=20 \ - -o Acquire::https::Timeout=20 \ - update -} +# --- Install dev-host build prerequisites --------------------------------- -apt_install() { +apt_get() { sudo apt-get \ -o Acquire::Retries=3 \ -o Acquire::http::Timeout=20 \ -o Acquire::https::Timeout=20 \ - install -y "$@" + "$@" } -echo "installing/validating host kernel-module build packages" -apt_update -apt_install \ +echo "[1/5] installing host build prerequisites" +apt_get update +apt_get install -y \ bc \ bison \ build-essential \ @@ -86,25 +59,28 @@ apt_install \ rsync \ sshpass -kernel="$(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "uname -r")" +# --- Discover the target's kernel + header layout ------------------------- + +echo "[2/5] discovering target kernel" +lab_check_connection +kernel="$(lab_ssh 'uname -r')" +echo " target $target is running $kernel" + cache_dir="$repo_root/.kernel-cache/$target" -build_dir="$cache_dir/build" usr_src_dir="$cache_dir/usr-src" +build_dir="$cache_dir/build" remote_vmlinux="/usr/lib/debug/boot/vmlinux-$kernel" mkdir -p "$cache_dir" "$usr_src_dir" -echo "target $target is currently running kernel $kernel" -echo "discovering target kernel header layout" -build_real="$(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "readlink -f '/lib/modules/$kernel/build'")" -if ! SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "test -d '$build_real'"; then - echo "missing target build directory: $build_real" >&2 - echo "run scripts/01-provision-target.sh $target first" >&2 - exit 1 +build_real="$(lab_ssh "readlink -f '/lib/modules/$kernel/build'")" +if ! lab_ssh "test -d '$build_real'"; then + die "missing target build directory $build_real; +- on the target, check: ls -la /lib/modules/$kernel/build +- if it's missing, re-run: scripts/01-provision-target.sh $target " fi -mapfile -t remote_header_dirs < <(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "bash -s" <\t in a single round-trip so the sync -# loop below can stay dumb. -mapfile -t remote_source_entries < <(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "bash -s" <<'REMOTE' +# Kernel source is discovered separately: it can be a tarball (a single file +# at /usr/src/linux-source-X.tar.bz2) OR a directory (possibly containing +# the tarball inside it, on some Ubuntu releases). One round-trip classifies +# each path as file|dir so the sync loop stays trivial. +mapfile -t remote_source_entries < <(lab_ssh "bash -s" <<'REMOTE' set -euo pipefail shopt -s nullglob seen=() @@ -145,45 +120,37 @@ done REMOTE ) +# --- Sync ---------------------------------------------------------------- + sync_header_dir() { local remote_dir="$1" - local name local dest - - name="$(basename "$remote_dir")" - dest="$usr_src_dir/$name" - - echo "syncing $remote_dir -> $dest" + dest="$usr_src_dir/$(basename "$remote_dir")" + echo " $remote_dir -> ${dest#"$repo_root"/}" # Preserve symlinks inside Ubuntu's kernel header trees. Some optional - # symlinks, such as rust support links, may be dangling and are harmless for + # symlinks (e.g. rust support) may be dangling and are harmless for # external C module builds. - SSHPASS="$ssh_pass" "$rsync_bin" -a --delete -e "$rsync_rsh" \ - "$ssh_target:$remote_dir/" "$dest/" + lab_rsync_from "$remote_dir/" "$dest/" --delete } -sync_remote_path() { - # Sync either a remote dir (contents synced into $dest/) or a remote file - # (file synced as $dest). Used for kernel-source paths where the remote - # layout can be either. +sync_source_path() { local remote_path="$1" local kind="$2" - local name local dest - - name="$(basename "$remote_path")" - dest="$usr_src_dir/$name" - + dest="$usr_src_dir/$(basename "$remote_path")" + echo " $remote_path -> ${dest#"$repo_root"/}" if [[ "$kind" == "dir" ]]; then - echo "syncing $remote_path/ -> $dest/" - SSHPASS="$ssh_pass" "$rsync_bin" -a --delete -e "$rsync_rsh" \ - "$ssh_target:$remote_path/" "$dest/" + lab_rsync_from "$remote_path/" "$dest/" --delete else - echo "syncing $remote_path -> $dest" - SSHPASS="$ssh_pass" "$rsync_bin" -a -e "$rsync_rsh" \ - "$ssh_target:$remote_path" "$dest" + lab_rsync_from "$remote_path" "$dest" fi } +echo "[3/5] syncing kernel headers and source" +# Dedup the discovered header paths; the discovery on the target can +# legitimately list the same canonical path twice (e.g. when +# /lib/modules/$kernel/source and /usr/src/linux-headers-$base resolve to +# the same dir via symlink chains). printf '%s\n' "${remote_header_dirs[@]}" | awk 'NF && !seen[$0]++' | while IFS= read -r remote_dir; do sync_header_dir "$remote_dir" @@ -191,36 +158,48 @@ done for entry in "${remote_source_entries[@]}"; do [[ -z "$entry" ]] && continue - path="${entry%$'\t'*}" - kind="${entry##*$'\t'}" - sync_remote_path "$path" "$kind" + sync_source_path "${entry%$'\t'*}" "${entry##*$'\t'}" done build_name="$(basename "$build_real")" rm -rf "$build_dir" ln -s "usr-src/$build_name" "$build_dir" - -if [[ ! -f "$build_dir/Makefile" ]]; then - echo "synced build tree is missing Makefile: $build_dir" >&2 - exit 1 +[[ -f "$build_dir/Makefile" ]] || die "synced build tree is missing Makefile: $build_dir" + +# --- vmlinux ------------------------------------------------------------- + +echo "[4/5] checking vmlinux debug image" +if lab_ssh_sudo "test -r '$remote_vmlinux'"; then + # Use rsync over sudo: skips when unchanged, restartable, checksummed. + # Without this the cat-based fetch would silently corrupt vmlinux any + # time sudo printed anything unexpected to stdout or the SSH connection + # dropped mid-transfer (the file is ~415MB so the window is non-trivial). + echo " rsync $remote_vmlinux -> ${cache_dir#"$repo_root"/}/vmlinux" + lab_rsync_from "$remote_vmlinux" "$cache_dir/vmlinux" --rsync-path='sudo rsync' +else + rm -f "$cache_dir/vmlinux" + echo " no vmlinux on target ($remote_vmlinux is missing)" + echo " module debugging will work; kernel source debugging will not" + echo " to enable, run: scripts/01-provision-target.sh $target --debug-symbols" fi -# Kernel source: find a usable source root under usr-src/ and point the -# stable `source` symlink at it. Ubuntu has shipped three layouts: +# --- Kernel source resolution -------------------------------------------- +# +# Find a usable kernel-source root under usr-src/. Ubuntu has shipped three +# layouts over time: # 1. /usr/src/linux-source-X.tar.bz2 (tarball, no enclosing dir) # 2. /usr/src/linux-source-X/ (pre-extracted, files at top) # 3. /usr/src/linux-source-X/linux-source-X/ (pre-extracted, nested one deep) # Layout 1 also occurs nested as /usr/src/linux-source-X/linux-source-X.tar.bz2. # -# Strategy: look for a Makefile (the kernel's top-level marker) at depth ≤ 3 -# under usr-src/. If found, that's our source root — symlink directly to it, -# no extraction needed. Otherwise look for a tarball and extract on the host -# into $cache_dir/source-tree/. -# -# Extraction happens on the host, not the target, for two reasons: it avoids -# sudo + tar on the target for what's purely a host-side debug resource, and -# it's self-healing — re-running 02 fixes broken/partial trees without -# needing to touch the target. +# Strategy: look for a Makefile + init/main.c (the kernel's top-level +# markers) at depth ≤ 3 under usr-src/. If found, symlink directly to it; +# otherwise look for a tarball and extract on the host into source-tree/. +# Extraction on the host (not the target) avoids a sudo+tar dependency on +# the target for what's purely a host-side debug resource, and makes the +# step self-healing: re-running 02 fixes partial/broken trees. + +echo "[5/5] resolving kernel source for step-into-kernel" source_link="$cache_dir/source" source_tree="$cache_dir/source-tree" rm -f "$source_link" @@ -229,60 +208,64 @@ is_kernel_source_root() { [[ -f "$1/Makefile" && -d "$1/init" && -f "$1/init/main.c" ]] } +# Pick the source dir matching the live kernel's major.minor.patch when +# possible (handles the case where multiple linux-source-X packages are +# installed on the target). Fall back to the highest-versioned root. +short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" found_root="" +matching_root="" while IFS= read -r mf; do candidate="$(dirname "$mf")" if is_kernel_source_root "$candidate"; then - found_root="$candidate" - break + # `sort -V` below means "first match" is also the highest version; + # remember the first hit, then prefer a kver-matching one if seen. + [[ -z "$found_root" ]] && found_root="$candidate" + if [[ -n "$short_kver" && "$candidate" == *"linux-source-$short_kver"* ]]; then + matching_root="$candidate" + break + fi fi -done < <(find "$usr_src_dir" -mindepth 1 -maxdepth 3 -name Makefile -path '*linux-source-*' 2>/dev/null | sort) +done < <(find "$usr_src_dir" -mindepth 1 -maxdepth 3 -name Makefile -path '*linux-source-*' 2>/dev/null | sort -Vr) +found_root="${matching_root:-$found_root}" if [[ -n "$found_root" ]]; then rm -rf "$source_tree" - rel="${found_root#$cache_dir/}" + rel="${found_root#"$cache_dir"/}" ln -s "$rel" "$source_link" - echo "kernel source: $source_link -> $rel" + echo " kernel source: $source_link -> $rel" else shopt -s nullglob - tarballs=( "$usr_src_dir"/linux-source-*.tar.* "$usr_src_dir"/linux-source-*/linux-source-*.tar.* ) + tarballs=("$usr_src_dir"/linux-source-*.tar.* "$usr_src_dir"/linux-source-*/linux-source-*.tar.*) shopt -u nullglob if [[ ${#tarballs[@]} -gt 0 ]]; then tarball="${tarballs[0]}" if is_kernel_source_root "$source_tree"; then - echo "kernel source already extracted at $source_tree" + echo " kernel source already extracted at $source_tree" else - echo "extracting $tarball -> $source_tree" + echo " extracting $(basename "$tarball") -> source-tree/ (one-time, ~30s)" rm -rf "$source_tree" mkdir -p "$source_tree" tar -C "$source_tree" --strip-components=1 -xf "$tarball" fi ln -s source-tree "$source_link" - echo "kernel source: $source_link -> source-tree (extracted from $(basename "$tarball"))" + echo " kernel source: $source_link -> source-tree" else rm -rf "$source_tree" - echo "no kernel source tarball or extracted tree on target; step-into-kernel will only show disassembly" - echo "run scripts/01-provision-target.sh $target --debug-symbols to install it" + echo " no kernel source on target; kernel step-into will show disassembly only" + echo " to enable, run: scripts/01-provision-target.sh $target --debug-symbols" fi fi -echo "checking vmlinux debug image" -remote_sudo="$(remote_sudo_prefix)" -if SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}test -r '$remote_vmlinux'"; then - echo "copying $remote_vmlinux" - SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}cat '$remote_vmlinux'" > "$cache_dir/vmlinux" -else - rm -f "$cache_dir/vmlinux" - echo "warning: missing readable $remote_vmlinux on target" >&2 - echo "module builds can still work, but VS Code source debugging needs it" >&2 - echo "run scripts/01-provision-target.sh $target --debug-symbols to install it" >&2 -fi +# --- Bookkeeping --------------------------------------------------------- printf '%s\n' "$kernel" > "$cache_dir/kernel.release" printf '%s\n' "$build_real" > "$cache_dir/remote.build.path" printf '%s\n' "${remote_header_dirs[@]}" > "$cache_dir/remote.header.paths" ln -sfn "$target" "$repo_root/.kernel-cache/current" -echo "cached $target kernel $kernel in $cache_dir" -echo "local KDIR: $build_dir" -echo "current IntelliSense target: $target" +echo +echo "synced $target kernel $kernel" +echo " build -> $build_dir" +[[ -L "$source_link" ]] && echo " source -> $(readlink -f "$source_link")" +[[ -f "$cache_dir/vmlinux" ]] && echo " vmlinux: $cache_dir/vmlinux" +echo "next: scripts/03-build-module.sh $target" diff --git a/scripts/03-build-module.sh b/scripts/03-build-module.sh index 9d01b3b..2404f6b 100755 --- a/scripts/03-build-module.sh +++ b/scripts/03-build-module.sh @@ -1,8 +1,16 @@ #!/usr/bin/env bash +# Build module/ against the synced headers for . +# +# Driven by the top-level Makefile, which stages module/ into +# build/intermediate/// and runs Kbuild from there with +# -ffile-prefix-map injected via KCFLAGS. Result is one .ko under +# build/artifacts///. The module name is whatever the user's +# obj-m declares; the lab does not need to know it. set -euo pipefail target="${1:-}" repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" validate_target "$target" "usage: $0 " @@ -10,24 +18,18 @@ cache_dir="$repo_root/.kernel-cache/$target" kernel_file="$cache_dir/kernel.release" kdir="$cache_dir/build" -if [[ ! -f "$kernel_file" || ! -d "$kdir" ]]; then - echo "missing kernel cache for $target" >&2 - echo "run scripts/02-setup-host-build.sh $target first" >&2 - exit 1 -fi +[[ -f "$kernel_file" && -d "$kdir" ]] || + die "missing kernel cache for $target; run: scripts/02-setup-host-build.sh $target" module_dir="$repo_root/module" -if [[ ! -e "$module_dir/Makefile" && ! -e "$module_dir/Kbuild" ]]; then - echo "no Makefile or Kbuild under $module_dir" >&2 - echo "drop your module project under module/ (see module/README.md)" >&2 - exit 1 -fi +[[ -e "$module_dir/Makefile" || -e "$module_dir/Kbuild" ]] || + die "no Makefile or Kbuild under $module_dir; drop your module project there (see $module_dir/README.md), or try the example: make use-example NAME=chuck_norise" kernel="$(<"$kernel_file")" intermediate_dir="$repo_root/build/intermediate/$target/$kernel" artifact_dir="$repo_root/build/artifacts/$target/$kernel" -echo "building module under $module_dir for $target kernel $kernel" +echo "building $module_dir for $target kernel $kernel" make -C "$repo_root" \ KDIR="$kdir" \ BUILD_ID="$target/$kernel" \ @@ -37,10 +39,9 @@ make -C "$repo_root" \ clean modules mapfile -t kos < <(find "$artifact_dir" -maxdepth 1 -name '*.ko' -type f | sort) -if [[ ${#kos[@]} -eq 0 ]]; then - echo "build produced no .ko under $artifact_dir" >&2 - exit 1 -fi +[[ ${#kos[@]} -gt 0 ]] || die "build produced no .ko under $artifact_dir; check the make output above" + for ko in "${kos[@]}"; do echo "built $ko" done +echo "next: scripts/04-deploy-debug-vscode.sh $target " diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index 26f879c..dcf1c81 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -1,4 +1,26 @@ #!/usr/bin/env bash +# Deploy the built module to a target, load it, and write the GDB init files +# VS Code (or `gdb -tui`) needs to attach and source-debug it. +# +# Steps: +# 1. Free the debug-endpoint TCP port from any stale GDB clients. Both +# KGDB's serial bridge and QEMU's gdbstub serve one client at a time, +# so any leftover gdb owning that socket blocks fresh attaches. This +# script kills any *local* gdb process holding the endpoint — be aware +# if you have unrelated gdb sessions on the same host:port. +# 2. Upload module/.ko to the target. +# 3. Insmod it (with debug_delay_ms if the module declares the param, +# falling back to a plain insmod otherwise). +# 4. Poll /sys/module//sections/ for the module's runtime load +# addresses; convert to an `add-symbol-file ... -s .name addr ...` +# line saved as the per-(target,method) symbols file. +# 5. Emit .gdb/-.gdb and -attached.gdb variants, plus +# current-debug{,attached}.gdb and current-vmlinux symlinks the IDE +# launch configs point at. +# +# Env knobs (default values in parens): +# DEBUG_LOAD_DELAY_MS (5000) passed to the module if it declares debug_delay_ms +# INSMOD_WAIT_SECS (30) how long to wait for /sys/module/.../sections/.text set -euo pipefail usage() { @@ -14,157 +36,231 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" validate_target "$target" "usage: $0 " -env_file="$repo_root/lab.local.env" -[[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" + +env_file="$(lab_env_file "$repo_root")" # shellcheck source=/dev/null source "$env_file" -target_is_configured "$target" - -ssh_bin="${SSH_BIN:-ssh}" -scp_bin="${SCP_BIN:-scp}" -sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(target_require_cfg "$target" SSH_HOST)" -ssh_port="$(target_cfg "$target" SSH_PORT)" -ssh_user="$(target_require_cfg "$target" SSH_USER)" -ssh_pass="$(target_cfg "$target" SSH_PASS)" -sudo_pass="$(target_cfg "$target" SUDO_PASS)" -remote_dir="$(target_cfg "$target" REMOTE_DIR)" +lab_load_target "$target" + endpoint_key="DEBUG_ENDPOINT_${debug_method^^}" debug_endpoint="$(target_require_cfg "$target" "$endpoint_key")" debug_delay_ms="${DEBUG_LOAD_DELAY_MS:-5000}" +insmod_wait_secs="${INSMOD_WAIT_SECS:-30}" -ssh_port="${ssh_port:-22}" -remote_dir="${remote_dir:-/tmp/kmod-debug-lab}" -sudo_pass="${sudo_pass:-$ssh_pass}" -sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" - -if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it on the build host: sudo apt-get install -y sshpass" >&2 - exit 1 -fi - -ssh_target="$ssh_user@$ssh_host" -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -scp_cmd=("$scp_bin" -o StrictHostKeyChecking=accept-new -P "$ssh_port") -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") - scp_cmd=("$sshpass_bin" -e "${scp_cmd[@]}") -fi - -remote_sudo_prefix() { - if [[ -n "$sudo_pass_b64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$sudo_pass_b64" - else - printf "sudo " - fi -} +# --- Locate built artifact + vmlinux -------------------------------------- cache_dir="$repo_root/.kernel-cache/$target" kernel_file="$cache_dir/kernel.release" vmlinux="$cache_dir/vmlinux" -if [[ ! -f "$kernel_file" ]]; then - echo "missing kernel cache for $target" >&2 - echo "run scripts/02-setup-host-build.sh $target first" >&2 - exit 1 -fi +[[ -f "$kernel_file" ]] || + die "missing kernel cache for $target; run: scripts/02-setup-host-build.sh $target" -if [[ ! -f "$vmlinux" ]]; then - echo "missing $vmlinux" >&2 - echo "module builds may work, but VS Code source debugging needs the matching vmlinux" >&2 - echo "install the target's linux-image-*-dbgsym package, then rerun:" >&2 - echo " scripts/02-setup-host-build.sh $target" >&2 - exit 1 -fi +[[ -f "$vmlinux" ]] || + die "missing $vmlinux — source debugging needs the matching vmlinux from the target. +- install + sync it with: + scripts/01-provision-target.sh $target $debug_method --debug-symbols + scripts/02-setup-host-build.sh $target" kernel="$(<"$kernel_file")" + +# Preflight: connection + sudo work, and the target is still running the +# kernel we cached. If the target rebooted into a different kernel since +# scripts/02 ran, vmlinux symbols won't match runtime addresses and GDB +# would silently show wrong source lines. +lab_check_connection +live_kernel="$(lab_ssh 'uname -r')" +if [[ "$live_kernel" != "$kernel" ]]; then + die "kernel mismatch: target is now running '$live_kernel' but cache has '$kernel'. +- if the target was rebooted into a new kernel, re-sync: + scripts/01-provision-target.sh $target $debug_method --debug-symbols # if you also need the new vmlinux + scripts/02-setup-host-build.sh $target +- if you booted the wrong kernel, reboot back into $kernel" +fi + artifact_dir="$repo_root/build/artifacts/$target/$kernel" intermediate_dir="$repo_root/build/intermediate/$target/$kernel" module_dir="$repo_root/module" mapfile -t artifacts < <(find "$artifact_dir" -maxdepth 1 -name '*.ko' -type f | sort) -if [[ ${#artifacts[@]} -eq 0 ]]; then - echo "no .ko under $artifact_dir" >&2 - echo "run scripts/03-build-module.sh $target first" >&2 - exit 1 -fi -if [[ ${#artifacts[@]} -gt 1 ]]; then - echo "expected exactly one .ko under $artifact_dir; found:" >&2 - printf ' %s\n' "${artifacts[@]}" >&2 - echo "the lab assumes a single obj-m per build" >&2 - exit 1 -fi +case ${#artifacts[@]} in + 0) die "no .ko under $artifact_dir; run: scripts/03-build-module.sh $target" ;; + 1) ;; + *) + echo "expected one .ko under $artifact_dir; found:" >&2 + printf ' %s\n' "${artifacts[@]}" >&2 + die "the lab assumes a single obj-m per build. Merge sources into one module (obj-m += foo.o; foo-y := a.o b.o ...) or split into separate module/ trees." + ;; +esac artifact="${artifacts[0]}" module_name="$(basename "$artifact" .ko)" +# --- GDB output paths ----------------------------------------------------- + gdb_dir="$repo_root/.gdb" mkdir -p "$gdb_dir" symbols_file="$gdb_dir/$target-$debug_method-module-symbols.gdb" gdb_file="$gdb_dir/$target-$debug_method.gdb" +gdb_attached_file="$gdb_dir/$target-$debug_method-attached.gdb" loader_log="$gdb_dir/$target-$debug_method-loader.log" -loader_script="$gdb_dir/$target-$debug_method-loader.sh" - -rm -f "$symbols_file" "$loader_log" "$loader_script" - -remote_module="$remote_dir/$module_name.ko" -remote_sudo="$(remote_sudo_prefix)" - -# Kill any local gdb processes that still hold a TCP connection to the debug -# endpoint. Both KGDB's serial bridge and QEMU's gdbstub serve one client at a -# time; leftover gdb processes from previous sessions (zombie bare-gdb -# invocations, half-torn-down VS Code launches, etc.) sit in the endpoint's -# accept queue and block fresh attaches. -debug_host="${debug_endpoint%:*}" -debug_port="${debug_endpoint##*:}" -if command -v ss >/dev/null 2>&1; then - stale_pid_list=$({ ss -ntp 2>/dev/null \ - | grep -F "$debug_host:$debug_port" \ - | grep -oE 'pid=[0-9]+' \ - | cut -d= -f2 \ - | sort -u; } || true) - stale_gdb_pids="" - for pid in $stale_pid_list; do - if [[ "$(ps -p "$pid" -o comm= 2>/dev/null || true)" == "gdb" ]]; then - stale_gdb_pids="$stale_gdb_pids $pid" - fi + +rm -f "$symbols_file" "$loader_log" + +# --- Free the debug endpoint from stale GDB clients ----------------------- +# +# Both KGDB's serial bridge and QEMU's gdbstub serve one client at a time. +# A zombie gdb left over from a previous session sits in the endpoint's +# accept queue and blocks fresh attaches. We best-effort kill any local gdb +# process that holds a connection to the endpoint. Errors here never fail +# the deploy — at worst the user has to `killall gdb` manually. +# +# `lsof -ti` is more robust than parsing `ss -ntp` output: it gives one PID +# per line and won't break across iproute2 versions. +free_debug_endpoint() { + command -v lsof >/dev/null 2>&1 || return 0 + local host="${debug_endpoint%:*}" port="${debug_endpoint##*:}" + local pids + pids="$(lsof -ti "@$host:$port" 2>/dev/null || true)" + local gdb_pids=() + local pid + for pid in $pids; do + [[ "$(ps -p "$pid" -o comm= 2>/dev/null || true)" == "gdb" ]] && gdb_pids+=("$pid") done - if [[ -n "$stale_gdb_pids" ]]; then + if [[ ${#gdb_pids[@]} -gt 0 ]]; then echo "killing leftover gdb clients on $debug_endpoint:" - # shellcheck disable=SC2086 - ps -p $stale_gdb_pids -o pid,cmd 2>/dev/null | tail -n +2 | sed 's/^/ /' || true - # shellcheck disable=SC2086 - kill -9 $stale_gdb_pids 2>/dev/null || true + ps -p "${gdb_pids[@]}" -o pid,cmd 2>/dev/null | tail -n +2 | sed 's/^/ /' || true + kill -9 "${gdb_pids[@]}" 2>/dev/null || true sleep 0.3 fi -fi +} +free_debug_endpoint || true + +# --- Upload + load on target ---------------------------------------------- + +remote_module="$LAB_REMOTE_DIR/$module_name.ko" + +echo "uploading $artifact -> $LAB_SSH_TARGET:$remote_module" +lab_ssh "mkdir -p '$LAB_REMOTE_DIR'" +lab_scp_to "$artifact" "$remote_module" +# Verify the upload — a truncated .ko would silently insmod-fail with cryptic +# "invalid module format" errors. +local_size="$(stat -c %s "$artifact")" +remote_size="$(lab_ssh "stat -c %s '$remote_module' 2>/dev/null" | tr -d '[:space:]')" +[[ "$remote_size" == "$local_size" ]] || + die "upload size mismatch: local $local_size bytes, remote ${remote_size:-missing} bytes; retry scripts/04 or check disk space on the target" + +# Load the module and discover the addresses /sys/module//sections/ +# exposes once it's live. The insmod runs inside a single nohup'd sh -c so +# the debug_delay_ms-then-plain-insmod fallback chains correctly: if we +# instead ran two `nohup ... &` invocations the shell would background them +# independently and the `||` between them would be a no-op (it'd just check +# the success of the backgrounding, not the insmod itself). +# +# Output is captured to $loader_log; we surface either the success tail +# line or the full log on failure. +load_and_discover_symbols() { + echo "loading $module_name on $target (debug_delay_ms=$debug_delay_ms)" + lab_ssh_sudo "rmmod '$module_name' >/dev/null 2>&1 || true" + # Background insmod on the target so we can poll /sys/module/.../sections + # in parallel with the module's debug_delay_ms sleep. + # + # Three subtleties: + # - `<&0` keeps sudo's stdin attached to the SSH-inherited pipe (where + # the password arrives). Non-job-control bash automatically redirects + # backgrounded processes' stdin to /dev/null UNLESS the command has + # its own stdin redirection — without `<&0`, sudo would see EOF + # instead of the password and silently fail to authenticate. + # - `& sleep 1` keeps the SSH session open long enough for sudo to + # read the password and exec into nohup. The sleep is short because + # sudo authentication is fast; if SSH closed earlier the in-flight + # password write could be lost. + # - The orphaned nohup+sh+insmod chain survives session exit because + # non-interactive bash doesn't huponexit. + # + # The `|| insmod ...` falls back when the module doesn't declare a + # debug_delay_ms parameter (the kv arg would otherwise reject with EINVAL). + lab_ssh_sudo "nohup sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$LAB_REMOTE_DIR/insmod.log' 2>&1 <&0 & sleep 1" + + echo "polling /sys/module/$module_name/sections/ (timeout ${insmod_wait_secs}s)" + + # Dump every readable file under /sys/module//sections/. Discovery + # is dynamic so modules with non-standard sections (.text.hot, custom + # __ksymtab subsections, ...) get their addresses picked up too. + # + # The sh -c body MUST be single-quoted in the SSH command so the outer + # remote bash doesn't expand $(ls -A) and $f before sh -c sees them. + # With double quotes, the outer bash would evaluate the substitution in + # its own CWD; the for loop would iterate over the wrong filenames and + # we'd never find .text — even though the module had already loaded. + local dump_body + # shellcheck disable=SC2016 # $(...) is intentionally not expanded locally — see comment above. + dump_body='cd "/sys/module/'"$module_name"'/sections" 2>/dev/null && for f in $(ls -A 2>/dev/null); do [ -r "$f" ] && printf "%s %s\n" "$f" "$(cat "$f")"; done' + + local deadline=$((SECONDS + insmod_wait_secs)) + local tmp + tmp="$(mktemp)" + while ((SECONDS < deadline)); do + if lab_ssh_sudo "sh -c '$dump_body'" > "$tmp" 2>/dev/null; then + local text_addr + text_addr="$(awk '$1 == ".text" { print $2 }' "$tmp")" + if [[ -n "$text_addr" ]]; then + local cmd="add-symbol-file $artifact $text_addr" + while read -r sec addr; do + [[ "$sec" != ".text" && -n "$addr" ]] && cmd+=" -s $sec $addr" + done < "$tmp" + { + echo "$cmd" + printf 'echo loaded %s module symbols for %s\\n\n' "$module_name" "$target" + } > "$symbols_file" + rm -f "$tmp" + echo "wrote $symbols_file" + return 0 + fi + fi + # No explicit sleep — each SSH round-trip already takes 0.3-1s, which + # is the right polling cadence. + done -echo "uploading $artifact to $ssh_target:$remote_module" -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "mkdir -p '$remote_dir'" -SSHPASS="$ssh_pass" "${scp_cmd[@]}" "$artifact" "$ssh_target:$remote_module" + rm -f "$tmp" + echo "timeout: /sys/module/$module_name/sections/.text did not appear within ${insmod_wait_secs}s" + echo "remote insmod.log (empty = insmod produced no output; module may still have failed to load):" + lab_ssh "cat '$LAB_REMOTE_DIR/insmod.log' 2>/dev/null || true" | sed 's/^/ /' + echo "hint: ssh into the target and try the load by hand:" + echo " ssh $LAB_SSH_TARGET sudo insmod $remote_module" + echo " ssh $LAB_SSH_TARGET sudo dmesg | tail -40" + echo " ssh $LAB_SSH_TARGET ls -la /sys/module/$module_name/sections/ 2>/dev/null" + return 1 +} -# If we have a synced kernel source tree (from scripts/02 + provision with -# --debug-symbols), try to map Ubuntu's build-time source prefix to our local -# copy so GDB can show kernel source on step-into. We discover the prefix -# dynamically by asking addr2line for the file/line of `start_kernel`, which -# always lives at init/main.c in any Linux source tree — the prefix is -# whatever comes before that suffix. -kernel_src_link="$cache_dir/source" +if load_and_discover_symbols > "$loader_log" 2>&1; then + tail -1 "$loader_log" +else + echo "module load / symbol discovery failed (full log: $loader_log):" + sed 's/^/ /' "$loader_log" + exit 1 +fi + +# --- Build the GDB init files -------------------------------------------- +# +# If the kernel source has been synced (scripts/01 --debug-symbols + scripts/02), +# discover Ubuntu's build-time source prefix from vmlinux so GDB can remap +# DWARF references to our local copy. We ask addr2line where `start_kernel` +# lives — the path is always /init/main.c, so stripping the +# known suffix yields the prefix. kernel_substitute_line="" kernel_directory_line="" +kernel_src_link="$cache_dir/source" if [[ -L "$kernel_src_link" || -d "$kernel_src_link" ]]; then kernel_src_root="$(readlink -f "$kernel_src_link")" kernel_directory_line="directory $kernel_src_root" if command -v nm >/dev/null 2>&1 && command -v addr2line >/dev/null 2>&1; then - # `|| true` because pipefail + SIGPIPE: nm dumps every symbol in - # vmlinux (millions), awk's `exit` after the first match closes the - # pipe, nm gets SIGPIPE on its next write, and the pipeline exits - # 141. We still capture the address awk printed before exiting. + # `|| true` because pipefail + SIGPIPE: nm dumps every vmlinux + # symbol, awk's `exit` after the first match closes the pipe and nm + # gets SIGPIPE on its next write. We still capture awk's output. sym_addr="$(nm "$vmlinux" 2>/dev/null | awk 'NF==3 && $3=="start_kernel" {print $1; exit}' || true)" if [[ -n "$sym_addr" ]]; then sym_loc="$(addr2line -e "$vmlinux" "$sym_addr" 2>/dev/null | head -1 | cut -d: -f1 || true)" - # Expected shape: /init/main.c build_prefix="${sym_loc%/init/main.c}" if [[ -n "$build_prefix" && "$build_prefix" != "$sym_loc" ]]; then kernel_substitute_line="set substitute-path $build_prefix $kernel_src_root" @@ -173,12 +269,12 @@ if [[ -L "$kernel_src_link" || -d "$kernel_src_link" ]]; then echo "warning: addr2line returned unexpected location for start_kernel: $sym_loc" fi else - echo "warning: could not find start_kernel in vmlinux; skipping kernel source remap" + echo "warning: could not find start_kernel in vmlinux; kernel step-into will show disassembly" fi fi else - echo "note: $kernel_src_link missing; GDB will show kernel disassembly instead of source" - echo " run scripts/01 with --debug-symbols then re-run scripts/02 to enable step-into-kernel" + echo "note: no kernel source synced; kernel step-into will show disassembly" + echo " to enable: scripts/01-... --debug-symbols && scripts/02-... $target" fi { @@ -205,96 +301,19 @@ source $symbols_file EOF } > "$gdb_file" -# Trimmed-down variant for IDEs that handle the gdb-side attach themselves -# (Native Debug, cppdbg with miDebuggerServerAddress). The IDE has already -# called `target remote` and loaded the executable, so commands that touch -# global gdb state (mi-async, remote timeouts, architecture override) error -# with "Cannot change this setting while the inferior is running" — drop them. -gdb_attached_file="$gdb_dir/$target-$debug_method-attached.gdb" -grep -vE '^(target remote |set mi-async |set target-async |set tcp connect-timeout |set remotetimeout |set architecture |symbol-file )' "$gdb_file" > "$gdb_attached_file" +# Native Debug ("type": "gdb") and cppdbg with miDebuggerServerAddress have +# already called `target remote` and loaded the executable by the time they +# source our script, so commands that touch global gdb state error with +# "Cannot change this setting while the inferior is running". Strip them. +grep -vE '^(target remote |set mi-async |set target-async |set tcp connect-timeout |set remotetimeout |set architecture |symbol-file )' \ + "$gdb_file" > "$gdb_attached_file" ln -sfn "$target-$debug_method.gdb" "$gdb_dir/current-debug.gdb" ln -sfn "$target-$debug_method-attached.gdb" "$gdb_dir/current-debug-attached.gdb" ln -sfn "../.kernel-cache/$target/vmlinux" "$gdb_dir/current-vmlinux" -{ - printf '#!/usr/bin/env bash\n' - printf 'set -euo pipefail\n' - printf 'target=%q\n' "$target" - printf 'module_name=%q\n' "$module_name" - printf 'symbols_file=%q\n' "$symbols_file" - printf 'artifact=%q\n' "$artifact" - printf 'ssh_bin=%q\n' "$ssh_bin" - printf 'sshpass_bin=%q\n' "$sshpass_bin" - printf 'ssh_pass=%q\n' "$ssh_pass" - printf 'ssh_port=%q\n' "$ssh_port" - printf 'ssh_target=%q\n' "$ssh_target" - printf 'sudo_pass_b64=%q\n' "$sudo_pass_b64" - printf 'remote_module=%q\n' "$remote_module" - printf 'remote_dir=%q\n' "$remote_dir" - printf 'debug_delay_ms=%q\n' "$debug_delay_ms" - cat <<'LOADER' - -remote_sudo_prefix() { - if [[ -n "$sudo_pass_b64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$sudo_pass_b64" - else - printf "sudo " - fi -} - -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") -fi -remote_sudo="$(remote_sudo_prefix)" - -echo "loading $module_name on $target and discovering symbols" -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}rmmod '$module_name' >/dev/null 2>&1 || true" - -# debug_delay_ms is the lab's convention for "sleep this long in module_init so -# the host has time to attach GDB and discover sections". Modules that declare -# it as a module_param accept the kv argument; modules that don't will reject -# insmod with -EINVAL on the unknown parameter. We try with the parameter and -# fall back to a plain insmod, inside a single sh -c so the || chain runs -# inside one backgrounded process. -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "nohup ${remote_sudo}sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$remote_dir/insmod.log' 2>&1 &" - -tmp_sections="$(mktemp)" -trap 'rm -f "$tmp_sections"' EXIT - -section_list=".text .data .bss .rodata .init.text .exit.text .text.unlikely .rodata.str1.1 .rodata.str1.8 .init.data .exit.data" -deadline=$((SECONDS + 30)) -while ((SECONDS < deadline)); do - if SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "${remote_sudo}sh -c 'for sec in $section_list; do path=/sys/module/$module_name/sections/\$sec; if [ -r \"\$path\" ]; then printf \"%s %s\n\" \"\$sec\" \"\$(cat \"\$path\")\"; fi; done'" \ - > "$tmp_sections"; then - text_addr="$(awk '$1 == ".text" { print $2 }' "$tmp_sections")" - if [[ -n "$text_addr" ]]; then - cmd="add-symbol-file $artifact $text_addr" - while read -r sec addr; do - if [[ "$sec" != ".text" && -n "$addr" ]]; then - cmd="$cmd -s $sec $addr" - fi - done < "$tmp_sections" - { - echo "$cmd" - echo "echo loaded $module_name module symbols for $target\\n" - } > "$symbols_file" - echo "wrote module symbols to $symbols_file" - exit 0 - fi - fi - sleep 0.2 -done - -echo "failed to discover /sys/module/$module_name/sections/.text within 30s" -exit 1 -LOADER -} > "$loader_script" -chmod +x "$loader_script" -"$loader_script" > "$loader_log" 2>&1 - -echo "generated $gdb_file" -echo "loaded $module_name and wrote symbols; log: $loader_log" +echo +echo "ready. GDB endpoint: $debug_endpoint" +echo " gdb script: $gdb_file" +echo " loader log: $loader_log" +echo "next: F5 in VS Code (Kernel: cppdbg) — or: gdb -tui $vmlinux -x $gdb_file" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 0de37be..bb987d4 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -1,10 +1,26 @@ #!/usr/bin/env bash +# scripts/lib/common.sh — shared helpers sourced by every numbered script. +# +# Naming: +# die, validate_target, target_* bare functions; no global state required. +# lab_* higher-level helpers; rely on globals +# populated by `lab_load_target`. +# LAB_* globals populated by `lab_load_target`. +# Scripts should call lab_ssh / lab_ssh_sudo +# / lab_scp_to / lab_rsync_from rather than +# touching the underlying ssh/scp/sshpass +# binaries directly. +# +# Sourcing this file is side-effect-free. State is created when a script +# calls `lab_load_target ` after sourcing `lab.local.env`. die() { echo "$*" >&2 exit 1 } +# --- Target validation ---------------------------------------------------- + validate_target() { local target="$1" local usage="$2" @@ -50,5 +66,169 @@ target_require_cfg() { require_debian_host() { command -v apt-get >/dev/null 2>&1 || - die "missing apt-get; the development host must be Debian-based" + die "missing apt-get; the development host must be Debian-based (Debian, Ubuntu, WSL Ubuntu, ...)" +} + +# --- Lab env loading ------------------------------------------------------ + +# Validate that lab.local.env exists under repo_root and return its path on +# stdout. Caller is expected to `source` the returned path directly so the +# `declare -A TARGET_*=...` entries land in the script's global scope (a +# function-internal source would scope them to the function). +lab_env_file() { + local repo_root="$1" + local env_file="$repo_root/lab.local.env" + [[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it for your machines" + printf '%s' "$env_file" +} + +# --- Per-target connection setup ------------------------------------------ + +# Read TARGET_* config for the given target and populate LAB_* globals used +# by every remote helper below. Validates that ssh password tooling is +# available if a password is configured. +# +# After this returns, the following globals are set: +# LAB_TARGET target name +# LAB_SSH_HOST/PORT/USER raw connection details +# LAB_SSH_PASS ssh password ("" when using keys) +# LAB_SUDO_PASS sudo password ("" when sudo is passwordless / NOPASSWD) +# LAB_REMOTE_DIR target staging dir (with per-user default) +# LAB_SSH_TARGET "user@host" form for use in commands +# LAB_SSH_CMD array, ready to invoke (includes sshpass + timeouts) +# LAB_SCP_CMD array, ready to invoke (includes sshpass + timeouts) +# LAB_RSYNC_RSH rsync -e value (includes sshpass + timeouts) +lab_load_target() { + local target="$1" + target_is_configured "$target" + + # shellcheck disable=SC2034 # LAB_TARGET is consumed by sourcing scripts. + LAB_TARGET="$target" + LAB_SSH_HOST="$(target_require_cfg "$target" SSH_HOST)" + LAB_SSH_PORT="$(target_cfg "$target" SSH_PORT)" + LAB_SSH_USER="$(target_require_cfg "$target" SSH_USER)" + LAB_SSH_PASS="$(target_cfg "$target" SSH_PASS)" + LAB_SUDO_PASS="$(target_cfg "$target" SUDO_PASS)" + LAB_REMOTE_DIR="$(target_cfg "$target" REMOTE_DIR)" + + LAB_SSH_PORT="${LAB_SSH_PORT:-22}" + # Sudo password falls back to ssh password — common single-user dev case. + LAB_SUDO_PASS="${LAB_SUDO_PASS:-$LAB_SSH_PASS}" + # Per-user default so two devs on the same host don't collide on /tmp. + LAB_REMOTE_DIR="${LAB_REMOTE_DIR:-/tmp/kmod-debug-lab-$LAB_SSH_USER}" + + local ssh_bin scp_bin sshpass_bin + ssh_bin="${SSH_BIN:-ssh}" + scp_bin="${SCP_BIN:-scp}" + sshpass_bin="${SSHPASS_BIN:-sshpass}" + + if [[ -n "$LAB_SSH_PASS" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then + die "TARGET_SSH_PASS[$target] is set but '$sshpass_bin' is not installed; install it with: sudo apt-get install -y sshpass" + fi + + LAB_SSH_TARGET="$LAB_SSH_USER@$LAB_SSH_HOST" + + # LAB_SSH_CMD and LAB_SCP_CMD are flag-only — they do NOT include the + # target. Helpers and direct callers append the target themselves so + # `ssh ... -t user@host cmd` works (anything after the host is treated + # as the remote command by ssh). + # + # ConnectTimeout fails fast (10s) if the target is unreachable instead + # of blocking for minutes. ServerAliveInterval keeps long-lived + # connections (e.g. the 30s symbol-discovery loop) from being dropped + # by NAT idle timers. + local ssh_opts=( + -o StrictHostKeyChecking=accept-new + -o ConnectTimeout=10 + -o ServerAliveInterval=15 + ) + LAB_SSH_CMD=("$ssh_bin" "${ssh_opts[@]}" -p "$LAB_SSH_PORT") + LAB_SCP_CMD=("$scp_bin" "${ssh_opts[@]}" -P "$LAB_SSH_PORT") + LAB_RSYNC_RSH="$ssh_bin ${ssh_opts[*]} -p $LAB_SSH_PORT" + if [[ -n "$LAB_SSH_PASS" ]]; then + LAB_SSH_CMD=("$sshpass_bin" -e "${LAB_SSH_CMD[@]}") + LAB_SCP_CMD=("$sshpass_bin" -e "${LAB_SCP_CMD[@]}") + LAB_RSYNC_RSH="$sshpass_bin -e $LAB_RSYNC_RSH" + fi +} + +# --- Remote command helpers ----------------------------------------------- +# +# All of these require a prior `lab_load_target` call. + +# Run a command on the target. Arguments are passed through to ssh as the +# remote command (typically one quoted shell string). Inherits stdin from +# the caller — handy when callers want to pipe data in (e.g. lab_ssh_sudo +# pipes the sudo password). +lab_ssh() { + SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "$@" +} + +# Run a command on the target as root. The sudo password reaches the remote +# `sudo -S` via ssh's stdin, which means it never appears in any process's +# argv on the target — `ps auxww` is clean. When LAB_SUDO_PASS is empty +# (NOPASSWD sudo or sudo configured for passwordless), the empty string is +# piped and sudo proceeds without prompting. +# +# Why `-k`: without it, if sudo has cached credentials from an earlier +# invocation (e.g. `scripts/00-check-target.sh` just ran, or the user did +# `sudo` on the target within the last 5 min), `sudo -S` skips the stdin +# read entirely. The piped password then leaks into whatever inherits +# stdin from sudo — for `sudo bash -s`, the password line becomes bash's +# first script line and you get `bash: line 1: a: command not found`. +# `-k` ignores the cache for THIS invocation only; it does NOT invalidate +# the user's existing sudo timestamp. +# +# Callers that need to feed their own stdin to the remote command (e.g. +# rsync) should use `lab_rsync_from` / scp helpers instead — those take a +# different path that doesn't compete for stdin. +lab_ssh_sudo() { + printf '%s\n' "$LAB_SUDO_PASS" | + SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "sudo -k -S -p '' $*" +} + +# Copy a local file to the target. The remote path is absolute. +lab_scp_to() { + local local_path="$1" + local remote_path="$2" + SSHPASS="$LAB_SSH_PASS" "${LAB_SCP_CMD[@]}" "$local_path" "$LAB_SSH_TARGET:$remote_path" +} + +# Rsync a remote path (file or dir) into a local destination. Extra rsync +# args can be appended — notably `--rsync-path='sudo rsync'` for paths only +# root can read (vmlinux), and `--delete` for tree mirroring. +lab_rsync_from() { + local remote_src="$1" + local local_dst="$2" + shift 2 + SSHPASS="$LAB_SSH_PASS" "${RSYNC_BIN:-rsync}" -a -e "$LAB_RSYNC_RSH" "$@" \ + "$LAB_SSH_TARGET:$remote_src" "$local_dst" +} + +# --- Preflight ------------------------------------------------------------ + +# Fast, non-mutating connection check. Verifies SSH reachable, and (by +# default) that sudo works. Pass `--no-sudo` to skip the sudo test — useful +# in `scripts/01-provision-target.sh`, which a fresh user might be running +# *to* configure sudo for the first time. +# +# Called at the top of 01/02/04 so failures surface before any real work. +# `scripts/00-check-target.sh` runs a longer-form report on top of this. +# shellcheck disable=SC2120 # arg is optional (--no-sudo); callers without it are intentional. +lab_check_connection() { + local check_sudo=1 + [[ "${1:-}" == "--no-sudo" ]] && check_sudo=0 + + if ! lab_ssh 'true' 2>/dev/null; then + die "cannot reach $LAB_SSH_TARGET over ssh (port $LAB_SSH_PORT). +- check TARGET_SSH_HOST/PORT/USER for '$LAB_TARGET' in lab.local.env +- if the VM is up, try by hand: ssh -p $LAB_SSH_PORT $LAB_SSH_TARGET +- if you're using passwords, confirm TARGET_SSH_PASS[$LAB_TARGET] is set" + fi + if (( check_sudo )) && ! lab_ssh_sudo 'true' 2>/dev/null; then + die "ssh reaches $LAB_SSH_TARGET but sudo doesn't work there. +- check TARGET_SUDO_PASS[$LAB_TARGET] in lab.local.env (defaults to TARGET_SSH_PASS) +- on the target, confirm: sudo -n -v (or run sudo by hand once to cache creds) +- consider configuring NOPASSWD for the lab user on long-lived dev targets" + fi }