From 3cd0008a4bbbef092ab8cb4ad75158196f8bc8d2 Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 20:41:46 +0300 Subject: [PATCH] cleaner readmes and remove large comments --- .github/workflows/lint.yml | 3 +- .vscode/c_cpp_properties.json | 5 - CONTRIBUTING.md | 111 +++---- Makefile | 59 ++-- README.md | 366 ++++++++--------------- examples/chuck_norise/Makefile | 21 +- examples/chuck_norise/README.md | 59 ++-- examples/chuck_norise/src/chuck_device.c | 11 +- examples/chuck_norise/src/hello.c | 4 +- module/README.md | 78 +---- scripts/00-check-target.sh | 21 +- scripts/01-provision-target.sh | 102 +++---- scripts/02-setup-host-build.sh | 89 ++---- scripts/03-build-module.sh | 7 +- scripts/04-deploy-debug-vscode.sh | 151 ++++------ scripts/lib/common.sh | 130 +++----- 16 files changed, 428 insertions(+), 789 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8c84bff..75dcc27 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -54,8 +54,7 @@ jobs: strategy: fail-fast: false matrix: - # Surface drift across LTS kernels. ubuntu-22.04 ships 5.15; 24.04 ships 6.8. - os: [ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 248ab5d..9065b7f 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -41,10 +41,5 @@ "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler_types.h" ] } - ], - "_notes": [ - "KBUILD_MODNAME is a placeholder; the actual value at compile time is your module's real name. IntelliSense only needs the macro to resolve, not match exactly.", - "ubuntu/include is Ubuntu-kernel-specific (where the distro's extra headers live); on non-Ubuntu kernels VS Code silently ignores it.", - "All paths under .kernel-cache/current/ resolve only after scripts/02-setup-host-build.sh has run. Before that, expect include squiggles." ] } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e518573..d7a590b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,83 +1,68 @@ # Contributing -Bug reports, fixes, and improvements are welcome. This file describes what -that looks like for this repo. +Bug reports, fixes, and improvements welcome. -## What this repo is (and isn't) +## What belongs here -This is a **template** for building and source-debugging out-of-tree Linux -kernel modules. The lab tooling lives in `Makefile`, `scripts/`, -`.vscode/`, `host/`, and the docs. The example module under -`examples/chuck_norise/` exists to demonstrate the workflow end-to-end. +This is a **template** for building and source-debugging out-of-tree +Linux kernel modules. Lab tooling lives in `Makefile`, `scripts/`, +`.vscode/`, `host/`. The example under `examples/chuck_norise/` +demonstrates the workflow end-to-end. -PRs that **belong** here: +**Yes:** -- Bugs in the lab tooling — broken scripts, wrong assumptions about target - state, scripts that fail on re-run, unclear error messages. -- Support for a new target OS family (currently only Ubuntu). See the - `TARGET_OS` checks in scripts 01 and 02 for where to add a backend. -- New debug methods that fit the ` -> GDB` shape (anything - speaking the GDB remote serial protocol). -- Documentation that fixes misleading or missing information. -- Quality-of-life improvements: better error messages, more idempotent - re-runs, faster sync, smaller artifact dirs. +- Bugs in the lab tooling: broken scripts, wrong assumptions, scripts + that fail on re-run, unclear errors. +- New target-OS backends (currently only Ubuntu — see the `TARGET_OS` + checks in scripts 01 and 02). +- New debug methods that fit the ` → GDB` shape. +- Docs fixes. +- Quality-of-life: better errors, idempotent re-runs, faster sync. -PRs that **do not** belong here: +**No:** -- Changes to `module/` (that's the user's slot; we keep it as a placeholder - with just a README). -- New example modules under `examples/`. We keep one example focused and - small so it stays useful as a walkthrough. -- Project-management features (work logs, task tracking, etc.). The lab - stays focused on build + debug. +- Changes to `module/` (user's slot; stays as just a placeholder README). +- New examples under `examples/` (one focused example beats many). +- Project-management features (work logs, task tracking). ## Reporting bugs Open an issue with: -- Your **dev host** (distro, version, kernel) and **target VM** (distro, - version, kernel, hypervisor). -- The debug method you tried (`kgdb` or `qemu`) and the endpoint you - pointed it at. -- The command you ran and the full output. For deploy failures, attach +- Dev host (distro, version, kernel) and target VM (distro, version, + kernel, hypervisor). +- Debug method tried (`kgdb` or `qemu`) and endpoint configured. +- Command run and full output. For deploy failures, attach `.gdb/--loader.log`. -- What you expected, what happened instead. +- What you expected vs what happened. ## Submitting changes -1. Fork and branch from `main`. -2. Make focused commits — one logical change per commit. The git history - should read top-to-bottom as a small set of intentional steps. -3. Keep scripts shellcheck-clean (`shellcheck -x scripts/lib/common.sh - scripts/0*.sh`) and bash-syntax-clean (`bash -n`). -4. Update docs in the same PR. If you touched a script, check `README.md` - and `module/README.md` for anything the change makes inaccurate. -5. Open a PR with a description that explains *why* the change is needed, - not just what it does. The diff already says what. +1. Fork, branch from `main`. +2. Focused commits — one logical change per commit. +3. Keep scripts shellcheck-clean and `bash -n`-clean: + `shellcheck -x scripts/lib/common.sh scripts/0*.sh` and + `for f in scripts/lib/*.sh scripts/0*.sh; do bash -n "$f"; done`. +4. Update docs in the same PR — if you touched a script, check + `README.md` and `module/README.md` for anything the change made + inaccurate. +5. PR description explains *why*, not just what. ## Style -- **Shell:** tabs for indentation. `set -euo pipefail` at the top of every - script. Errors via the `die` helper. Use `lab_*` helpers from - `scripts/lib/common.sh` for SSH/SCP/rsync — do not call those binaries - directly from numbered scripts. -- **Makefile:** tabs for recipes. Explicit `.PHONY` declarations. -- **Markdown:** wrap prose at ~80 columns. Use fenced code blocks with - language tags. -- **JSON (VS Code config):** 2-space indent, trailing newline. - -The repo includes an `.editorconfig` that captures these — most editors -will apply it automatically. - -## Testing changes locally - -There is no automated test suite for the build/debug flow (it requires real -hardware-or-VM targets). Before sending a PR: - -- Run `bash -n scripts/0*.sh scripts/lib/*.sh` to catch syntax errors. -- If you have `shellcheck`, run it too: `shellcheck -x scripts/lib/common.sh - scripts/0*.sh`. -- Smoke-test the full chain against at least one target VM you have access - to: `01 --debug-symbols` → reboot → `02` → `03` → `04` → F5 in VS Code - → set breakpoints in both your module and a kernel function and confirm - both bind. +- **Shell:** tabs for indent. `set -euo pipefail` at the top. Errors via + the `die` helper. Use `lab_*` helpers from `scripts/lib/common.sh` — + don't call ssh/scp/sshpass binaries directly from numbered scripts. +- **Makefile:** tabs for recipes; explicit `.PHONY`. +- **Markdown:** wrap at ~80; fenced code blocks with language tags. +- **JSON / YAML:** 2-space indent, trailing newline. + +`.editorconfig` captures these — most editors apply it automatically. + +## Testing locally + +No automated test suite for the build/debug flow (it needs real VMs). +Before sending a PR, smoke-test the full chain against at least one +target VM: `01 --debug-symbols` → reboot → `02` → `03` → `04` → F5 in +VS Code → set breakpoints in both your module and a kernel function, +confirm both bind. diff --git a/Makefile b/Makefile index 7560971..7c48b92 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,15 @@ -# Top-level wrapper that drives the user's kernel module under module/. +# Top-level wrapper for the user's module under module/. # -# The user's module is built out-of-tree against a target-specific kernel -# header tree cached under .kernel-cache//. Build artifacts go under -# build/intermediate/// (Kbuild's M=) and the final .ko is -# moved to build/artifacts///. +# Builds out-of-tree against a target-specific kernel header tree under +# .kernel-cache//. Sources are staged into +# build/intermediate/// before Kbuild runs from there, +# and `-ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR)` is injected +# via KCFLAGS so DWARF paths and `__FILE__` resolve back to module/ — +# IDE breakpoints keyed by absolute path bind to the real sources. # -# Staging exists for two reasons: -# 1. Per-target out-of-tree artifacts. Building the same module/ tree against -# multiple target kernels needs separate intermediate dirs. -# 2. DWARF path rewriting. -ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR) -# rewrites paths in DWARF debug info (and __FILE__) from the staged -# intermediate dir back to module/, so IDEs that bind breakpoints by -# absolute path (VS Code, CLion) match the symtab. -# -# The user's module/Makefile sees only a normal Kbuild invocation rooted at -# the staging directory. KCFLAGS is the kernel build's official escape hatch -# for adding C flags from the outside, so the user's Makefile stays generic. +# Per-target staging also keeps each (target, kernel) pair's object +# files isolated. The user's module/Makefile stays a normal Kbuild file +# with no knowledge of the staging. HOST_KERNEL := $(shell uname -r) CURRENT_CACHE := $(CURDIR)/.kernel-cache/current @@ -40,16 +34,16 @@ help: @echo "Top-level lab Makefile." @echo "" @echo "Normally invoked by scripts/03-build-module.sh , which sets" - @echo "KDIR, BUILD_ID, INTERMEDIATE_DIR, and ARTIFACT_DIR to per-target paths." + @echo "KDIR, BUILD_ID, INTERMEDIATE_DIR, and ARTIFACT_DIR per-target." @echo "" @echo "Build targets:" - @echo " modules build module/ against KDIR (default)" - @echo " clean remove intermediate and artifact dirs for BUILD_ID" - @echo " prepare-build stage module/ into INTERMEDIATE_DIR (internal)" + @echo " modules build module/ against KDIR (default)" + @echo " clean remove BUILD_ID's intermediate + artifact dirs" + @echo " prepare-build stage module/ into INTERMEDIATE_DIR (internal)" @echo "" @echo "module/ ergonomics:" - @echo " use-example NAME=chuck_norise copy an example into module/" - @echo " clean-module reset module/ to just README.md" + @echo " use-example NAME=chuck_norise copy an example into module/" + @echo " clean-module reset module/ to just README.md" @echo "" @echo "Vars (auto-resolved when .kernel-cache/current points at a target):" @echo " MODULE_DIR $(MODULE_DIR)" @@ -59,8 +53,9 @@ help: @echo " ARTIFACT_DIR $(ARTIFACT_DIR)" @echo " EXTRA_CCFLAGS $(EXTRA_CCFLAGS) (appended to KCFLAGS)" -# Copy examples/$(NAME)/ into module/. Refuses if module/ already contains a -# Makefile/Kbuild (use clean-module first). NAME=chuck_norise unless overridden. +# Copy examples/$(NAME)/ into module/. Refuses if module/ already has a +# Makefile/Kbuild (run clean-module first). Preserves the placeholder +# module/README.md so the example's own README doesn't overwrite it. NAME ?= chuck_norise use-example: @if [ ! -d "$(CURDIR)/examples/$(NAME)" ]; then \ @@ -80,14 +75,8 @@ use-example: @echo "done. next: scripts/03-build-module.sh " @echo "(the example's docs stay at examples/$(NAME)/README.md)" -# Remove everything from module/ except its placeholder README, so you can -# drop a fresh module project in. Refuses to touch tracked files outside -# module/. clean-module: - @if [ ! -d "$(MODULE_DIR)" ]; then \ - echo "Makefile: $(MODULE_DIR) is missing — nothing to clean." >&2; \ - exit 1; \ - fi + @[ -d "$(MODULE_DIR)" ] || { echo "Makefile: $(MODULE_DIR) is missing." >&2; exit 1; } @find "$(MODULE_DIR)" -mindepth 1 -maxdepth 1 -not -name README.md -print0 \ | xargs -0 -r rm -rf @echo "module/ reset (kept only README.md)" @@ -108,11 +97,9 @@ modules: prepare-build echo "built $(ARTIFACT_DIR)/$$name"; \ done -# Mirror $(MODULE_DIR) into $(INTERMEDIATE_DIR) as a tree of absolute -# symlinks so Kbuild's `M=` sees the user's sources in a writeable scratch -# dir without polluting module/. `cp -as` is GNU coreutils; it recursively -# creates dirs and symlinks each file. Re-running is idempotent because we -# wipe the symlink scaffolding first. +# `cp -as` recursively creates dirs and absolute symlinks for each file. +# Re-runs are idempotent: we wipe the symlink scaffolding first so a +# removed source file doesn't linger as a stale symlink. prepare-build: @if [ ! -e "$(MODULE_DIR)/Makefile" ] && [ ! -e "$(MODULE_DIR)/Kbuild" ]; then \ echo "Makefile: no Makefile or Kbuild under $(MODULE_DIR);" >&2; \ diff --git a/README.md b/README.md index 538ee56..cdfdd3b 100644 --- a/README.md +++ b/README.md @@ -2,52 +2,49 @@ A template for building and **source-debugging** out-of-tree Linux kernel modules against one or more target VMs. Drop your module under `module/`, -point the lab at a target, press F5 — step through your code (and into -kernel code) in VS Code or `gdb -tui`. +point the lab at a target, press F5 — step through your code and into +kernel code from VS Code or `gdb -tui`. -What you get out of the box: +What you get: - **One slot, any module.** `module/` is your project; the lab is generic. - **Per-target builds.** Cross-build the same source against multiple target kernels without polluting your tree. -- **Breakpoints just work.** DWARF `-ffile-prefix-map` is injected for - you so IDE breakpoints by absolute path bind to your real source files. -- **Step into the kernel.** With one flag (`--debug-symbols`), the lab - pulls down `vmlinux` + kernel source and wires GDB `substitute-path` - so you can step from your module's read handler into `vfs_read`. +- **Breakpoints just work.** `-ffile-prefix-map` is injected via `KCFLAGS` + so IDE breakpoints by absolute path bind to your real source files. +- **Step into the kernel.** With `--debug-symbols`, the lab pulls down + `vmlinux` + kernel source and wires GDB `substitute-path` so you can + step from your module's read handler into `vfs_read`. - **Two debug methods.** In-kernel KGDB over serial (`kgdb`) and any hypervisor GDB stub (`qemu`, also VMware's `debugStub`). -- **One-command flow.** Four numbered scripts + a VS Code F5. -The repo ships with a small worked example (`examples/chuck_norise/`) so -you can run the whole flow end-to-end before writing a line of module -code. +The repo ships with a worked example (`examples/chuck_norise/`) so you +can run the full flow end-to-end before writing a line of module code. --- ## Quickstart -You need a **Debian-based dev host** (Debian, Ubuntu, WSL Ubuntu, …) and -at least one **Ubuntu target VM** reachable over SSH. From fresh clone -to a stopped breakpoint: +You need a **Debian-based dev host** (Debian, Ubuntu, WSL Ubuntu) and at +least one **Ubuntu target VM** reachable over SSH. ```bash # 0. clone + configure git clone kmod-debug-lab && cd kmod-debug-lab cp lab.example.env lab.local.env -$EDITOR lab.local.env # set TARGET_SSH_*, endpoints, etc. +$EDITOR lab.local.env # TARGET_SSH_*, endpoints, etc. -# 0.5 (optional) sanity-check the target before changing anything on it +# optional preflight (non-mutating) ./scripts/00-check-target.sh server qemu -# 1. one-time target setup (installs headers, vmlinux dbg, kernel source) +# 1. target setup: headers, vmlinux dbg, kernel source (one-time) ./scripts/01-provision-target.sh server qemu --debug-symbols # ...reboot the target VM so new GRUB args take effect... -# 2. one-time host setup (syncs everything to .kernel-cache/server/) +# 2. host setup: sync everything to .kernel-cache/server/ (one-time) ./scripts/02-setup-host-build.sh server -# 3. try the example module (replace with your own when ready) +# 3. try the example (or skip and drop your own module under module/) make use-example NAME=chuck_norise ./scripts/03-build-module.sh server @@ -61,49 +58,45 @@ code . Set a breakpoint in `module/src/hello.c` and one in `vfs_read` — both bind. You're stepping through kernel code from your module. +After the one-time `01 + 02`, the inner loop is **03 → 04 → F5** (or +just F5 — the launch's pre-task runs 04, and 03 if needed). + --- -## How the pieces fit +## Repo layout ``` module/ your module project (Makefile + sources) -examples/chuck_norise/ worked example; copy into module/ with `make use-example` +examples/chuck_norise/ worked example; `make use-example` copies it into module/ -scripts/00-check-target.sh non-mutating preflight: ssh / sudo / headers / vmlinux / endpoint -scripts/01-provision-target.sh target-side: install headers, vmlinux dbg, source, GRUB args - (also `--uninstall` to undo the GRUB args) -scripts/02-setup-host-build.sh dev-host: sync headers + vmlinux + kernel source into .kernel-cache/ -scripts/03-build-module.sh dev-host: build module/ via Kbuild against the synced headers +scripts/00-check-target.sh preflight: ssh / sudo / headers / vmlinux / endpoint +scripts/01-provision-target.sh target-side: headers, vmlinux dbg, kernel source, GRUB args + (`--uninstall` to undo the GRUB args) +scripts/02-setup-host-build.sh dev-host: sync headers + vmlinux + source into .kernel-cache/ +scripts/03-build-module.sh dev-host: build module/ against the synced headers scripts/04-deploy-debug-vscode.sh dev-host: upload, insmod, write .gdb/-.gdb -scripts/lib/common.sh shared bash helpers (lab_ssh, lab_ssh_sudo, lab_rsync_from, …) +scripts/lib/common.sh shared helpers (lab_ssh, lab_ssh_sudo, lab_rsync_from, ...) -.kernel-cache// per-target build/source/vmlinux cache (gitignored) +.kernel-cache// synced build/source/vmlinux (gitignored) build/ per-target intermediate + final .ko (gitignored) -.gdb/ generated GDB init files (gitignored, regenerated on F5) +.gdb/ generated GDB init files (gitignored, regenerated each F5) -host/ optional helpers for VMware-on-Windows users +host/ optional VMware-on-Windows helpers lab.example.env -> lab.local.env per-machine config (gitignored copy) ``` -Top-level `Makefile` exposes a few non-build targets too: -`make use-example NAME=` (copy `examples//` into `module/`), -`make clean-module` (reset `module/` to just its placeholder README), -`make help` (full var/target listing). - -The numbered scripts are designed to be run in order. After the one-time -01 + 02 against a target, the inner loop is **03 → 04 → F5** (or just F5, -which runs 04 as a prelaunch task and rebuilds via 03 if you ran it -manually first). +Useful `make` targets: `make use-example NAME=`, `make clean-module`, +`make help`. --- ## The `module/` contract -`module/` is the lab's only slot for your kernel module project. The lab -does not know your module's name, source layout, or behavior; it expects -exactly: +`module/` is the lab's only slot for your module. The lab does not know +its name, source layout, or behavior. Expectations: -- **A standard Kbuild file** at `module/Makefile` (or `module/Kbuild`): +- **`module/Makefile`** (or `Kbuild`) following standard out-of-tree + conventions: ```makefile obj-m += my_module.o @@ -111,54 +104,41 @@ exactly: ccflags-y := -I$(src)/include -g -DDEBUG ``` - See `examples/chuck_norise/Makefile` for the canonical shape, including - the optional `ifndef KERNELRELEASE` wrapper that lets you also run plain - `make` directly in `module/`. + `examples/chuck_norise/Makefile` shows the canonical shape, including + the optional `ifndef KERNELRELEASE` wrapper that lets `make` work + directly in `module/`. - **Exactly one `obj-m` entry per build.** The lab discovers the module - name from the produced `.ko` — you don't declare it anywhere - else. + name from the produced `.ko`. -- **Optional:** a `debug_delay_ms` module parameter: +- **Optional `debug_delay_ms` module parameter** so the host has time to + attach GDB before init runs: ```c static unsigned int debug_delay_ms = 5000; module_param(debug_delay_ms, uint, 0644); ``` - When present, scripts/04 passes the value from `DEBUG_LOAD_DELAY_MS` - (in `lab.local.env`, default 5000 ms) so your `module_init` sleeps - long enough for the host to read `/sys/module//sections/*` and - for you to break in GDB before init does anything. Modules without - the parameter would reject `insmod foo.ko debug_delay_ms=...` with - `-EINVAL`; the loader catches that and retries with a plain `insmod`, - so undeclared modules still load. + Script 04 passes `DEBUG_LOAD_DELAY_MS` (from `lab.local.env`) on + `insmod`. Modules that don't declare the parameter fall back to a + plain `insmod` automatically. -Source layout, headers, license — those are yours. The lab passes -through whatever your Kbuild file declares. +Source layout is yours. The lab passes through whatever your Kbuild file +declares. --- ## Configuring the lab -Copy the example env and edit it for your machines: - -```bash -cp lab.example.env lab.local.env -``` - Targets are **data, not variable prefixes**. Add a profile name to `TARGETS`, then add entries to the `TARGET_*` maps under that name: ```bash TARGETS=(desktop server) -declare -A TARGET_SSH_HOST=( - [desktop]=ubuntu-desktop.local - [server]=ubuntu-server.local -) +declare -A TARGET_SSH_HOST=([desktop]=ubuntu-desktop.local [server]=ubuntu-server.local) declare -A TARGET_SSH_USER=([desktop]=user [server]=user) -declare -A TARGET_SSH_PASS=([desktop]= [server]=) # empty when using SSH keys +declare -A TARGET_SSH_PASS=([desktop]= [server]=) # empty when using SSH keys declare -A TARGET_DEBUG_ENDPOINT_QEMU=( [desktop]=127.0.0.1:1234 @@ -166,233 +146,141 @@ declare -A TARGET_DEBUG_ENDPOINT_QEMU=( ) ``` -Endpoints are split per debug method -(`TARGET_DEBUG_ENDPOINT_KGDB` and `TARGET_DEBUG_ENDPOINT_QEMU`). Either -map may be left empty per target if you don't use that method. Script 04 -errors clearly if you pick a method without an endpoint. - -Target names accept `[A-Za-z][A-Za-z0-9_-]*`. The VS Code task picker -accepts any name you type — no editing of `.vscode/` needed when adding -a target. +Endpoints are split per debug method (`TARGET_DEBUG_ENDPOINT_KGDB`, +`TARGET_DEBUG_ENDPOINT_QEMU`); either map can be empty per target. The +VS Code task picker accepts any target name you type — no edits to +`.vscode/` when adding targets. --- ## Debug methods -Provisioning (script 01) and deploy (script 04) both take a -`` argument: - -- **`kgdb`** — in-kernel KGDB over the guest's serial port. Script 01 - adds `kgdboc=ttyS0,115200` and `sysrq_always_enabled=1` to the guest's - GRUB command line. Break with `echo g | sudo tee /proc/sysrq-trigger` - on the target. Useful when you cannot change the hypervisor's command - line (VMware Workstation is the common case). - - **`qemu`** — the hypervisor's built-in GDB stub - (QEMU's `-gdb tcp::PORT` / `-s`, VMware's `debugStub.listen.guest64`). + (QEMU's `-gdb tcp::PORT` or `-s`, VMware's `debugStub.listen.guest64`). No KGDB in the guest; the hypervisor halts the vCPU directly. +- **`kgdb`** — in-kernel KGDB over the guest's serial port. Script 01 + adds `kgdboc=ttyS0,115200` and `sysrq_always_enabled=1` to GRUB. Break + with `echo g | sudo tee /proc/sysrq-trigger` on the target. Useful + when you can't change the hypervisor's command line (VMware + Workstation is the common case). Neither method assumes any in-module `kgdb_breakpoint()` call. In both, -you break manually after script 04 finishes. +you break manually after 04 finishes. -### QEMU gdbstub +**QEMU stub:** `qemu-system-x86_64 ... -gdb tcp::1234` (`-S` to pause +at boot). -```text -qemu-system-x86_64 ... -gdb tcp::1234 -``` +**VMware debug stub:** in the VM's `.vmx`: -Use `-S` to pause the vCPU at boot. Without `-S`, the guest runs until -GDB connects. - -### VMware debug stub - -Power off the VM, edit the VM's `.vmx`: - -```text +``` debugStub.listen.guest64 = "TRUE" debugStub.port.guest64 = "8864" debugStub.listen.guest64.remote = "TRUE" debugStub.hideBreakpoints = "FALSE" ``` -Same shape as the QEMU stub — set -`TARGET_DEBUG_ENDPOINT_QEMU[server]=127.0.0.1:8864`. - -### KGDB over serial → TCP - -Set up a serial bridge from the guest's `/dev/ttyS0` to a TCP listener -GDB can connect to. How depends on the hypervisor: - -- **QEMU:** `-serial tcp:127.0.0.1:5520,server,nowait` -- **VMware Workstation on Windows:** named pipe in the VM's serial port - + `host/bridge-kgdb.ps1`: - - ```powershell - .\host\bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520 - ``` - -Provision the target with `kgdb`: - -```bash -./scripts/01-provision-target.sh server kgdb -``` - -`TARGET_KGDB_TTY` and `TARGET_KGDB_BAUD` in `lab.local.env` must match -the GRUB args added by provisioning (defaults `ttyS0` / `115200`). +Then set `TARGET_DEBUG_ENDPOINT_QEMU[server]=127.0.0.1:8864`. -Useful sanity check before F5: +**KGDB over serial → TCP:** bridge the guest's `/dev/ttyS0` to a TCP +listener. For QEMU: `-serial tcp:127.0.0.1:5520,server,nowait`. For +VMware Workstation on Windows: named-pipe serial port + +`host/bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520`. `TARGET_KGDB_TTY` +and `TARGET_KGDB_BAUD` in `lab.local.env` must match the GRUB args +script 01 adds. -```bash -nc -vz 127.0.0.1 5520 -``` +Sanity check before F5: `nc -vz 127.0.0.1 `. --- ## Stepping into kernel code -Pass `--debug-symbols` to script 01 to install both the matching -`vmlinux` debug image and the `linux-source-X` package on the target. -Script 02 syncs both, extracts the source on the host, and creates a -stable `.kernel-cache//source` symlink. - -Script 04 then asks `addr2line` where `start_kernel` lives in `vmlinux` -(always `/init/main.c`) and emits the corresponding GDB -remap into the generated `.gdb` file: - -```text -set substitute-path -directory -``` +Pass `--debug-symbols` to script 01 (installs the matching `vmlinux` +debug image and the `linux-source-X` package). Script 02 syncs and +extracts both. Script 04 then asks `addr2line` where `start_kernel` +lives in `vmlinux` (always `/init/main.c`) and emits the +corresponding `set substitute-path` into the generated `.gdb` file. That's it — set a breakpoint in `vfs_read` and step through it. -This step gracefully degrades. If `vmlinux` is missing, script 04 errors -clearly. If the kernel source wasn't synced, script 04 prints a one-line -note and continues — module debugging still works; you just see -disassembly when stepping into kernel functions. Check -`.gdb/--loader.log` for details when something is off. +Gracefully degrades: missing `vmlinux` is fatal (script 04 errors with +remediation); missing kernel source prints a one-line note and module +debugging still works (kernel step-into shows disassembly instead of +source). --- ## Undoing target-side changes -Script 01 modifies the target's GRUB command line. To remove every boot -arg the lab added (`nokaslr`, `kgdboc=...`, `sysrq_always_enabled=1`, -`maxcpus=...`) and restore a clean kernel command line: - -```bash -./scripts/01-provision-target.sh server --uninstall -``` - -This leaves installed packages alone — only the GRUB args are reverted. -A timestamped backup of `/etc/default/grub` is left on the target -(`/etc/default/grub.kmod-debug-lab..bak`) for paranoid recovery. - ---- - -## Build outputs - -``` -build/intermediate/// staged Kbuild tree (recursive symlinks back to module/) -build/artifacts///.ko final module -.kernel-cache//build -> synced linux-headers -.kernel-cache//source -> synced linux-source (when --debug-symbols) -.kernel-cache//vmlinux debug-symbol vmlinux from the target -.kernel-cache/current -> drives bare `make` and IntelliSense -.gdb/-.gdb full GDB init: arch, vmlinux, target remote, sourced symbols -.gdb/--attached.gdb trimmed variant for IDEs that already attach themselves -.gdb/--loader.log output of remote insmod + section discovery -.gdb/current-debug{.gdb,-attached.gdb} symlinks the VS Code launch configs read (re-pointed each F5) -``` - -`build/`, `.kernel-cache/`, `.gdb/`, `lab.local.env`, and -`compile_commands.json` are gitignored. +`./scripts/01-provision-target.sh --uninstall` removes every +boot arg the lab added (`nokaslr`, `kgdboc=`, `sysrq_always_enabled=1`, +`maxcpus=`) and re-runs `update-grub`. Installed packages are left in +place. A timestamped GRUB backup is left on the target under +`/etc/default/grub.kmod-debug-lab..bak`. --- ## Troubleshooting -**`sshpass: command not found`.** You configured `TARGET_SSH_PASS`. On -the dev host: `sudo apt-get install -y sshpass`. Leave -`TARGET_SUDO_PASS` empty when the sudo password matches the SSH -password. - -**F5 fails with `target remote ... Connection timed out`.** The endpoint -in `.gdb/current-debug.gdb` isn't reachable from the dev host. Verify -with `nc -vz `. On WSL, the Windows host address is often -in `/etc/resolv.conf`: - -```bash -grep nameserver /etc/resolv.conf -``` +**`sshpass: command not found`.** `TARGET_SSH_PASS` is set. On the dev +host: `sudo apt-get install -y sshpass`. -After editing `lab.local.env`, re-run F5 so script 04 regenerates the +**F5 fails with `target remote ... Connection timed out`.** Endpoint +not reachable from the dev host. `nc -vz ` to confirm. On +WSL, the Windows-host address is in `/etc/resolv.conf`'s `nameserver` +line. Re-run F5 after fixing `lab.local.env` so 04 regenerates the `.gdb/` files. -**GDB connects but the module never loads.** Check -`.gdb/--loader.log` for the insmod error. You can also -SSH in and try the insmod by hand: +**GDB connects but the module never loads.** Inspect +`.gdb/--loader.log`. Reproduce by hand: +`ssh sudo insmod /tmp/kmod-debug-lab-/.ko` then +`ssh sudo dmesg | tail -40`. -```bash -ssh sudo insmod /tmp/kmod-debug-lab/.ko -ssh sudo dmesg | tail -40 -``` - -**Breakpoints bind to files under `build/intermediate/...`.** Re-run F5 -so `.gdb/current-debug.gdb` is regenerated with the correct -`substitute-path`. Don't edit `.gdb/` by hand — it's regenerated every -F5. +**Breakpoints bind under `build/intermediate/...`.** Re-run F5; 04 +regenerates `.gdb/current-debug.gdb` with the correct +`substitute-path`. Don't edit `.gdb/` by hand. **VS Code shows include squiggles for `linux/module.h`.** Script 02 -hasn't synced the headers yet, or you switched targets and IntelliSense -is caching old paths. Re-run `scripts/02-setup-host-build.sh ` -and `C/C++: Reset IntelliSense Database` from the command palette. +hasn't synced the headers yet, or you switched targets and +IntelliSense is caching old paths. Re-run script 02 and run +`C/C++: Reset IntelliSense Database`. **`/sys/module//sections/.text` didn't appear within 30s.** Your -module either failed to load, or it's slow to init on the target. The -loader log will show `insmod` output. Bump the timeout with -`INSMOD_WAIT_SECS=60 ./scripts/04-... ...` if needed. - -**GDB shows `Cannot access memory at address …` when stepping into the -kernel.** Either `vmlinux` is missing (re-run `scripts/01-... --debug-symbols` -then `scripts/02-...`) or the cached `vmlinux` belongs to a different -kernel than the one running on the target. The latter is caught by -script 04's stale-kernel guard, but if you bypassed it, re-sync. - -**Spaces in your repo path.** Clone to a path WITHOUT spaces. GDB's -`set substitute-path FROM TO` splits on whitespace, so a `repo_root` like -`/home/me/path with spaces/...` produces a substitute-path that gets -parsed as four arguments and silently fails to remap module sources. -Kbuild is also notoriously fragile with spaces in paths. +module failed to load, or it's slow to init. Loader log shows insmod +output. Bump with `INSMOD_WAIT_SECS=60 ./scripts/04-... ...` if needed. + +**`Cannot access memory at address …` when stepping into the kernel.** +Either vmlinux is missing (re-run `01 --debug-symbols && 02`) or the +cached vmlinux is for a different kernel than the target now runs +(script 04 normally catches this — if you bypassed the guard, re-sync). + +**Spaces in your repo path.** Clone to a path without spaces. GDB's +`set substitute-path FROM TO` splits on whitespace; Kbuild is also +fragile with spaces. --- ## Tool versions -Tested with: - -- **Dev host:** Ubuntu 24.04, bash 5+, GNU coreutils, GDB ≥ 10, gcc 13. -- **Target VM:** Ubuntu 22.04 / 24.04, kernel 5.15 / 6.8. -- The `cp -as` recursive-symlink staging assumes GNU coreutils on the dev - host; macOS would need `brew install coreutils` and a `CP=gcp` override. -- `set substitute-path` for kernel-source remapping needs GDB ≥ 7.5 - (every supported distro ships much newer). -- `-ffile-prefix-map` (DWARF rewriting) requires GCC ≥ 8 or Clang ≥ 7. +Dev host: bash 5+, GNU coreutils, GDB ≥ 10, gcc ≥ 8 (for +`-ffile-prefix-map`). Target VM: Ubuntu 22.04 / 24.04 (kernel 5.15 / +6.8 tested; CI also exercises ubuntu-24.04 runners with 6.17). macOS +dev hosts need `brew install coreutils` and `make CP=gcp` for the +recursive-symlink staging. --- ## Adding a new target OS -Currently only `TARGET_OS[*]=ubuntu` is implemented in scripts 01 and 02. -Adding (say) Fedora means teaching script 01 how to install headers / -dbgsym / source via `dnf`, and teaching script 02 how to discover the -header tree paths under `/usr/src/`. Both scripts check `target_os` near -the top and fail fast on unknown values — that's where to add a backend. -PRs welcome. +Only `TARGET_OS[*]=ubuntu` is implemented in scripts 01 and 02. Adding +(say) Fedora means teaching script 01 to install headers / dbgsym / +source via `dnf`, and teaching script 02 to discover the header tree +paths under `/usr/src/`. Both scripts check `target_os` near the top +and fail fast on unknown values — that's where to add a backend. PRs +welcome. --- ## License -MIT — see [LICENSE](LICENSE). Your module under `module/` is yours; the -lab's license does not impose terms on what you build. +MIT — see [LICENSE](LICENSE). The lab's license does not impose terms +on your module under `module/`. diff --git a/examples/chuck_norise/Makefile b/examples/chuck_norise/Makefile index 5aa9583..c196f49 100644 --- a/examples/chuck_norise/Makefile +++ b/examples/chuck_norise/Makefile @@ -1,18 +1,13 @@ # Standard out-of-tree Linux kernel module Makefile. # -# This file is read in two different modes: -# -# 1. As a Kbuild fragment, when the kernel build system pulls it in via -# `make -C $KDIR M=$THIS_DIR modules`. In that mode KERNELRELEASE is set, -# so the Kbuild assignments below run and the wrapper recipe is skipped. -# -# 2. As a standalone Makefile, when a developer runs `make` in this directory -# directly. In that mode KERNELRELEASE is unset, so the wrapper recipe runs -# and re-invokes mode 1 against the host kernel's build tree. -# -# The lab's outer build (scripts/03-build-module.sh) always drives mode 1 -# against a target-specific kernel cache, but mode 2 is useful for ad-hoc -# builds against the host kernel. +# Read in two modes: +# 1. As a Kbuild fragment when the kernel build system pulls it in via +# `make -C $KDIR M=$THIS_DIR modules` (KERNELRELEASE is set; Kbuild +# assignments run, wrapper recipe is skipped). This is the mode the +# lab's outer build uses. +# 2. As a standalone Makefile when you run `make` in this dir directly +# (KERNELRELEASE is unset; the wrapper recipe re-invokes mode 1 +# against the host kernel). obj-m += chuck_norise.o chuck_norise-y := src/hello.o src/chuck_device.o src/chuck_message.o diff --git a/examples/chuck_norise/README.md b/examples/chuck_norise/README.md index 92b2a7d..c3f8e7b 100644 --- a/examples/chuck_norise/README.md +++ b/examples/chuck_norise/README.md @@ -1,53 +1,38 @@ # chuck_norise — example module -A small multi-file char device module used as the worked example for this lab. -Builds as `chuck_norise.ko` and exposes `/dev/chuck_norise`. Reads return the -exact string `chuck norise!` repeated indefinitely, preserving the file offset -for each open file descriptor. +Worked example for the lab. Builds as `chuck_norise.ko` and exposes +`/dev/chuck_norise`. Reads return `chuck norise!` repeated, preserving +the file offset. -It is intentionally tiny so it stays useful as a template walk-through: an init -that sleeps long enough to give you time to attach GDB, a cdev/class lifecycle -split into its own file, and a separately-testable "produce bytes into a -userspace buffer" routine. +## Run it through the lab -## Use it as a template - -The repo's build pipeline operates on whatever lives under `module/` at the -repo root. To try this example end-to-end, copy it into place from the -repo root: +From the repo root: ```bash make use-example NAME=chuck_norise ``` -Then run the normal lab flow (`scripts/00-...` through `scripts/04-...`, -or press F5 in VS Code — see the top-level [README](../../README.md) for -the full quickstart). After the module loads on the target VM: +Then follow the [top-level Quickstart](../../README.md#quickstart). After +the module loads on the target VM: ```bash -head -c 10 /dev/chuck_norise # prints "chuck nori" -``` - -To see offset-preserving reads from a single open fd: +head -c 10 /dev/chuck_norise # chuck nori -```bash +# offset-preserving reads from a single open fd exec 9/dev/null # chu -dd bs=1 count=5 <&9 2>/dev/null # ck no -dd bs=1 count=7 <&9 2>/dev/null # rise!ch +dd bs=1 count=3 <&9 2>/dev/null # chu +dd bs=1 count=5 <&9 2>/dev/null # ck no +dd bs=1 count=7 <&9 2>/dev/null # rise!ch exec 9<&- ``` -## What this example demonstrates - -- A standard out-of-tree Linux kernel module project layout (`src/`, `include/`, - `Makefile`). -- A `Makefile` that doubles as a Kbuild fragment and as a standalone wrapper — - the conventional shape for kernel modules. The lab's outer build invokes it - the Kbuild way; running `make` in this directory directly invokes the wrapper. -- A module init that sleeps for `debug_delay_ms` (default 5000 ms, exposed as - a module parameter) before doing anything observable. The delay gives the - host time to attach GDB and set breakpoints before init runs. -- A `LINUX_VERSION_CODE` shim for the 6.4 `class_create()` signature change, - as a real-world example of one of the things out-of-tree modules have to - carry. +## What this example shows + +- The conventional out-of-tree layout (`src/`, `include/`, `Makefile`). +- A `Makefile` that doubles as a Kbuild fragment AND a standalone + wrapper (`ifndef KERNELRELEASE` clause). +- A `debug_delay_ms` module parameter so the host has time to attach + GDB before init runs. +- A `LINUX_VERSION_CODE` shim for the 6.4 `class_create()` signature + change — a real-world example of the compat carrying out-of-tree + modules have to do. diff --git a/examples/chuck_norise/src/chuck_device.c b/examples/chuck_norise/src/chuck_device.c index cb6758d..d0d3dcf 100644 --- a/examples/chuck_norise/src/chuck_device.c +++ b/examples/chuck_norise/src/chuck_device.c @@ -29,17 +29,14 @@ static ssize_t chuck_read(struct file *file, char __user *buffer, size_t count, return chuck_message_read(buffer, count, position); } +/* .llseek deliberately unset — kernel installs default_llseek for char + * devices, which matches chuck_read's behavior (it respects *position). + * Don't reach for no_llseek; it was removed in 6.12 (commit 868941b14441). + */ static const struct file_operations chuck_fops = { .owner = THIS_MODULE, .open = chuck_open, .read = chuck_read, - /* - * .llseek deliberately unset. The kernel installs default_llseek for - * char devices, which matches chuck_read's behavior (it advances and - * respects *position). Earlier versions of this file pinned - * .llseek = no_llseek, but that symbol was removed in Linux 6.12 - * (mainline commit 868941b14441 "fs: remove no_llseek"). - */ }; static struct class *chuck_class_create(void) diff --git a/examples/chuck_norise/src/hello.c b/examples/chuck_norise/src/hello.c index 406b46d..d29c98e 100644 --- a/examples/chuck_norise/src/hello.c +++ b/examples/chuck_norise/src/hello.c @@ -38,5 +38,5 @@ module_init(hello_init); module_exit(hello_exit); MODULE_LICENSE("GPL"); -MODULE_AUTHOR("small-ko lab"); -MODULE_DESCRIPTION("Multi-file char device module for kernel source debugging"); +MODULE_AUTHOR("Linux Kernel Module Debug Lab"); +MODULE_DESCRIPTION("Multi-file char device example for kernel source debugging"); diff --git a/module/README.md b/module/README.md index 3026dcd..397d41e 100644 --- a/module/README.md +++ b/module/README.md @@ -1,75 +1,27 @@ # module/ — your kernel module goes here -This directory is the lab's only slot for the kernel module being built and -debugged. The repo is otherwise generic — it does not know your module's -name, source layout, or what it does. +The lab's only slot for the module being built and debugged. The repo is +otherwise generic — it doesn't know your module's name, source layout, +or what it does. -## Contract - -Drop a standard out-of-tree Linux kernel module project here. The lab -expects: - -- **`module/Makefile`** (or `module/Kbuild`) following standard Kbuild - conventions for out-of-tree modules: - - ```makefile - obj-m += my_module.o - my_module-y := src/main.o src/util.o - ccflags-y := -I$(src)/include -g -DDEBUG - ``` - - See `examples/chuck_norise/Makefile` for the canonical shape, including - the optional `ifndef KERNELRELEASE` wrapper that lets the same file be - driven by `make` directly. - -- **Sources anywhere you like.** The lab does not impose a `src/` or - `include/` layout — your Makefile's `*-y` line picks the source files - and `ccflags-y` picks the include paths. - -- **Exactly one `obj-m` entry per build.** The lab discovers the module - name from the produced `.ko`, so you don't declare it anywhere - else. - -- **Optionally a `debug_delay_ms` module parameter** that sleeps in - `module_init` so the host has time to attach GDB before init runs. - Declared as: - - ```c - static unsigned int debug_delay_ms = 5000; - module_param(debug_delay_ms, uint, 0644); - ``` - - When declared, scripts/04 passes `DEBUG_LOAD_DELAY_MS` from - `lab.local.env` to `insmod`. When not declared, the module loads - without it. - -That's the entire contract. The lab handles per-target staging, DWARF -path rewriting, uploading, loading, and symbol discovery on the target. +**Contract:** see the [top-level README](../README.md#the-module-contract). +TL;DR: a standard out-of-tree `Makefile` (or `Kbuild`) with one `obj-m` +entry; sources anywhere you like; an optional `debug_delay_ms` module +parameter so the host has time to attach GDB. ## Try the example ```bash -make use-example NAME=chuck_norise -``` - -Then build and debug as in the top-level `README.md`. To reset: - -```bash -make clean-module +make use-example NAME=chuck_norise # copy examples/chuck_norise/ into module/ +make clean-module # reset module/ to just this README ``` -## Replacing the example with your own module - -Run `make clean-module`, then drop your project in. Edit your `Makefile` -to set `obj-m`, `-y`, and `ccflags-y`. Rebuild. - ## One caveat: no symlinks inside `module/` The lab stages your sources into `build/intermediate///` -as a tree of absolute symlinks (`cp -as`) and then runs Kbuild from -there. If `module/` contains its OWN symlinks, the staged copy ends up -with symlinks pointing at the original symlinks — Kbuild will still find -the files, but the DWARF prefix-map remap (which assumes source files -live under `module/`) may not produce the paths your IDE expects. Keep -`module/` symlink-free, or copy in real files for any external sources -you depend on. +as a tree of absolute symlinks (`cp -as`) and runs Kbuild there. If +`module/` contains its OWN symlinks, the staged copy ends up with +symlinks-to-symlinks; the DWARF prefix-map remap (which assumes sources +live under `module/`) won't produce the paths your IDE expects. Keep +`module/` symlink-free, or copy in real files for external sources you +need. diff --git a/scripts/00-check-target.sh b/scripts/00-check-target.sh index a286470..23551d5 100755 --- a/scripts/00-check-target.sh +++ b/scripts/00-check-target.sh @@ -1,19 +1,10 @@ #!/usr/bin/env bash -# Non-mutating health check for a target. Run this before scripts/01 to -# catch configuration mistakes before any state on the target changes. +# Non-mutating health check for a target. Run before scripts/01 to catch +# configuration mistakes before any state on the target changes. # -# Reports, one line each: -# ssh is the target reachable over ssh? -# sudo does the configured (or fallback) sudo work? -# os distro and codename (must be ubuntu for now) -# kernel running kernel release -# headers /lib/modules/$kernel/build present? -# vmlinux debug-symbol vmlinux available? -# kernel-source linux-source-* package extracted? -# debug endpoint TCP port for the picked debug method reachable from the dev host? -# -# A "debug method" argument is optional; if omitted, both kgdb and qemu -# endpoints are checked when configured. +# Reports ssh / sudo / os / kernel / headers / vmlinux / kernel-source / +# debug endpoint. The debug-method arg is optional — both kgdb and qemu +# endpoints are checked when omitted. set -euo pipefail usage() { @@ -35,7 +26,7 @@ env_file="$(lab_env_file "$repo_root")" source "$env_file" lab_load_target "$target" -# Colored OK / FAIL / WARN, falling back to plain text when stdout isn't a TTY. +# Colored OK / WARN / FAIL on a TTY; plain text when piped. if [[ -t 1 ]]; then GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[0;33m'; RESET='\033[0m' else diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index c01de37..6802ed7 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -1,17 +1,16 @@ #!/usr/bin/env bash # Provision a target VM for kernel module debugging. # -# Target-side setup: install kernel headers (so the host build can sync them), -# optionally install matching vmlinux debug image + kernel source for -# step-into-kernel, and update the guest's GRUB command line with the boot -# args this lab needs (nokaslr always; kgdboc + sysrq for the kgdb method). +# Installs kernel headers (so the host build can sync them), optionally the +# matching vmlinux debug image + kernel source for step-into-kernel, and +# adds the boot args this lab needs to the guest's GRUB command line: +# - nokaslr (both methods; lets vmlinux symbols line up) +# - kgdboc + sysrq (kgdb method only) +# - maxcpus (only if DEBUG_MAXCPUS is set in the env) # -# With --uninstall, remove every boot arg the lab added (nokaslr / kgdboc / -# sysrq_always_enabled / maxcpus) — but leave installed packages alone. -# -# Reboot the target after this script completes so the new boot args take -# effect. The script itself does not reboot — staying out of the user's way -# matters for VMs that are reverted from snapshots and shouldn't be touched. +# `--uninstall` removes every boot arg the lab added and leaves installed +# packages alone. Reboot the target after either mode for boot args to +# take effect — this script never reboots on its own. set -euo pipefail usage() { @@ -24,8 +23,6 @@ target="${1:-}" arg2="${2:-}" arg3="${3:-}" -# Two modes: normal provision ( [--debug-symbols]) and -# --uninstall (which doesn't need a debug method since it only undoes GRUB). uninstall=0 debug_method="" symbols="" @@ -66,9 +63,8 @@ case "$target_os" in *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; esac -# Preflight: ssh works. We don't insist sudo works yet because the user -# might be running 01 *to* configure sudo. The actual provisioning shell -# (a piped `sudo -S bash -s`) will exercise sudo and fail with apt's own +# Skip the sudo check — the user might be running 01 *to* configure sudo. +# The provisioning shell below will exercise it and fail with apt's own # diagnostics if the password is wrong. lab_check_connection --no-sudo @@ -79,26 +75,16 @@ else echo "provisioning $target ($LAB_SSH_TARGET:$LAB_SSH_PORT) for debug method: $debug_method$symbols_blurb" fi -# Run the remote script as root via SUDO_ASKPASS so stdin stays a clean -# pipe of "env-vars then script body" for bash -s. -# -# Why NOT `sudo -S` + piped password on stdin: sudo only reads stdin when -# policy requires it. With NOPASSWD configured, or cached creds, sudo -# skips the stdin read entirely and the password line we piped leaks into -# bash -s as the first script line — producing -# `bash: line 1: : command not found`. `sudo -k` doesn't help -# because NOPASSWD is a policy override, not a cache thing. -# -# SUDO_ASKPASS works regardless: sudo invokes the askpass helper to fetch -# the password (or doesn't, under NOPASSWD), and stdin is purely the -# bash -s script. We scp the helper + a chmod-600 password file to the -# target up front; a trap cleans both up on script exit. +# We run the remote script as root via `sudo -A bash -s` so stdin stays a +# clean pipe of env-var-assignments + script body for bash to read. The +# straightforward "pipe password to sudo -S" pattern doesn't work here +# because sudo skips the stdin read whenever policy doesn't require auth +# (NOPASSWD, cached creds), and the password line then leaks into bash as +# its first command. SUDO_ASKPASS routes the password through a helper +# script so stdin is never contended. askpass="$LAB_REMOTE_DIR/.kmod-askpass-$$" pwfile="$LAB_REMOTE_DIR/.kmod-sudo-pw-$$" -cleanup_sudo_helpers() { - lab_ssh "rm -f '$askpass' '$pwfile' 2>/dev/null" || true -} -trap cleanup_sudo_helpers EXIT +trap 'lab_ssh "rm -f $askpass $pwfile 2>/dev/null" || true' EXIT lab_ssh "mkdir -p '$LAB_REMOTE_DIR'; umask 077; cat > '$pwfile'" <<<"$LAB_SUDO_PASS" lab_ssh "umask 077; cat > '$askpass'; chmod 700 '$askpass'" <&2; exit 1; } -[[ "${ID:-}" == "ubuntu" ]] || { - echo "target is not Ubuntu (os-release ID=$ID); only ubuntu targets are implemented" >&2; exit 1; } +[[ "${TARGET_OS:-ubuntu}" == "ubuntu" ]] || + { echo "unsupported target OS: ${TARGET_OS:-}" >&2; exit 1; } +[[ "${ID:-}" == "ubuntu" ]] || + { echo "target is not Ubuntu (os-release ID=$ID); only ubuntu targets are implemented" >&2; exit 1; } grub_file=/etc/default/grub +# Lab-managed boot args; stripped on every run so reruns replace rather +# than append, and `--uninstall` cleans them all out. grub_managed_keys='kgdboc=|maxcpus=|sysrq_always_enabled=|nokaslr' -# Read current GRUB_CMDLINE_LINUX_DEFAULT, stripping the lab's managed keys -# so reruns replace rather than append. `nokaslr` is a bare token (no =), -# matched as a standalone word. read_current_cmdline() { local raw stripped raw="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" - # Strip key=value forms and the bare "nokaslr" token; collapse spaces. stripped="$(printf '%s' "$raw" | sed -E "s/(^| )($grub_managed_keys)([^ ]*)?/ /g; s/ */ /g; s/^ +//; s/ +$//")" printf '%s' "$stripped" } write_cmdline() { - local newline="$1" cp "$grub_file" "$grub_file.kmod-debug-lab.$(date +%Y%m%d%H%M%S).bak" if grep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' "$grub_file"; then - sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$newline\"|" "$grub_file" + sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$1\"|" "$grub_file" else - printf 'GRUB_CMDLINE_LINUX_DEFAULT="%s"\n' "$newline" >> "$grub_file" + printf 'GRUB_CMDLINE_LINUX_DEFAULT="%s"\n' "$1" >> "$grub_file" fi update-grub } @@ -190,27 +173,23 @@ DDEBS apt_retry update apt_retry install -y "linux-image-${kernel}-dbgsym" || apt_retry install -y "linux-image-unsigned-${kernel}-dbgsym" || - { echo "failed to install debug symbols for $kernel; retry later if ddebs.ubuntu.com is returning 503" >&2; exit 1; } + { echo "failed to install dbgsym for $kernel; retry later if ddebs.ubuntu.com is returning 503" >&2; exit 1; } fi [[ -r "$vmlinux" ]] || echo "warning: $vmlinux still missing after install; kernel source debugging will be unavailable" - # Try the major.minor.patch-suffixed package first (e.g. linux-source-6.8.0); - # fall back to the unversioned meta-package. Extraction is deferred to - # script 02 — the host has more incentive to extract (it's where GDB lives) - # and doing it there avoids needing tar / sudo on the target. + # Try the major.minor.patch-suffixed package first (linux-source-6.8.0), + # falling back to the meta-package. We don't extract here — script 02 + # does it host-side to avoid needing tar on the target. short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" src_pkg="" for candidate in "linux-source-$short_kver" "linux-source"; do [[ -z "$candidate" || "$candidate" == "linux-source-" ]] && continue - if apt_retry install -y "$candidate"; then - src_pkg="$candidate" - break - fi + apt_retry install -y "$candidate" && { src_pkg="$candidate"; break; } done if [[ -z "$src_pkg" ]]; then echo "warning: could not install a linux-source package; step-into-kernel will only show disassembly" >&2 elif ls -1 /usr/src/linux-source-*.tar.* /usr/src/linux-source-*/linux-source-*.tar.* 2>/dev/null | head -1 >/dev/null; then - echo " kernel source tarball is in /usr/src on the target; script 02 will sync and extract it" + echo " kernel source tarball is in /usr/src; script 02 will sync and extract it" else echo "warning: $src_pkg installed but no linux-source tarball found under /usr/src/" >&2 fi @@ -221,18 +200,9 @@ fi echo "[3/3] updating GRUB command line" current="$(read_current_cmdline)" -# nokaslr is required in both modes so vmlinux symbols line up with running -# addresses. kgdb mode also needs an in-kernel debugger channel (kgdboc) and -# sysrq enabled so users can trigger a halt with `echo g > /proc/sysrq-trigger` -# — there is no kgdb_breakpoint() call in any user module to halt on insmod. -# qemu mode skips both: the hypervisor stub halts the vCPU directly. args=("nokaslr") -if [[ "$DEBUG_METHOD" == "kgdb" ]]; then - args+=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "sysrq_always_enabled=1") -fi -if [[ -n "${DEBUG_MAXCPUS:-}" ]]; then - args+=("maxcpus=${DEBUG_MAXCPUS}") -fi +[[ "$DEBUG_METHOD" == "kgdb" ]] && args+=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "sysrq_always_enabled=1") +[[ -n "${DEBUG_MAXCPUS:-}" ]] && args+=("maxcpus=${DEBUG_MAXCPUS}") for arg in "${args[@]}"; do case " $current " in *" $arg "*) ;; *) current="${current:+$current }$arg" ;; esac done diff --git a/scripts/02-setup-host-build.sh b/scripts/02-setup-host-build.sh index bb63dbd..c064a41 100755 --- a/scripts/02-setup-host-build.sh +++ b/scripts/02-setup-host-build.sh @@ -1,15 +1,13 @@ #!/usr/bin/env bash # Sync the target's kernel build tree to the host so an out-of-tree module -# can be cross-built against the target's exact kernel without copying the -# whole source tree per build. +# can be cross-built against the target's exact kernel. # -# Result: .kernel-cache// populated with: -# build/ -> symlink into the synced linux-headers tree -# source/ -> symlink into the synced linux-source tree (if --debug-symbols was used in 01) +# Populates .kernel-cache// with: +# build/ -> synced linux-headers tree +# source/ -> synced linux-source tree (if --debug-symbols was used in 01) # vmlinux -> debug-symbol vmlinux from the target (if available) -# kernel.release -> running kernel release (e.g. 6.8.0-117-generic) -# remote.build.path -> where headers came from on the target -# remote.header.paths -> list of all paths synced from /usr/src/ +# kernel.release -> running kernel release +# remote.{build,header}.* -> bookkeeping # Also updates .kernel-cache/current -> so plain `make` and the # IntelliSense config track the most recently synced target. set -euo pipefail @@ -47,17 +45,8 @@ apt_get() { echo "[1/5] installing host build prerequisites" apt_get update apt_get install -y \ - bc \ - bison \ - build-essential \ - dwarves \ - flex \ - gdb \ - libelf-dev \ - libssl-dev \ - openssh-client \ - rsync \ - sshpass + bc bison build-essential dwarves flex gdb \ + libelf-dev libssl-dev openssh-client rsync sshpass # --- Discover the target's kernel + header layout ------------------------- @@ -99,10 +88,9 @@ done REMOTE ) -# Kernel source is discovered separately: it can be a tarball (a single file -# at /usr/src/linux-source-X.tar.bz2) OR a directory (possibly containing -# the tarball inside it, on some Ubuntu releases). One round-trip classifies -# each path as file|dir so the sync loop stays trivial. +# Kernel source is found separately: it can be a tarball-file or a dir +# (sometimes with the tarball inside it). Classify each path as file|dir so +# the sync loop below stays trivial. mapfile -t remote_source_entries < <(lab_ssh "bash -s" <<'REMOTE' set -euo pipefail shopt -s nullglob @@ -123,20 +111,16 @@ REMOTE # --- Sync ---------------------------------------------------------------- sync_header_dir() { - local remote_dir="$1" - local dest + local remote_dir="$1" dest dest="$usr_src_dir/$(basename "$remote_dir")" echo " $remote_dir -> ${dest#"$repo_root"/}" - # Preserve symlinks inside Ubuntu's kernel header trees. Some optional - # symlinks (e.g. rust support) may be dangling and are harmless for - # external C module builds. + # Preserve symlinks inside Ubuntu's header trees (e.g. dangling rust + # links — harmless for external C builds). lab_rsync_from "$remote_dir/" "$dest/" --delete } sync_source_path() { - local remote_path="$1" - local kind="$2" - local dest + local remote_path="$1" kind="$2" dest dest="$usr_src_dir/$(basename "$remote_path")" echo " $remote_path -> ${dest#"$repo_root"/}" if [[ "$kind" == "dir" ]]; then @@ -147,10 +131,8 @@ sync_source_path() { } echo "[3/5] syncing kernel headers and source" -# Dedup the discovered header paths; the discovery on the target can -# legitimately list the same canonical path twice (e.g. when -# /lib/modules/$kernel/source and /usr/src/linux-headers-$base resolve to -# the same dir via symlink chains). +# Dedup discovered paths — two map entries can canonicalize to the same +# dir via symlink chains. printf '%s\n' "${remote_header_dirs[@]}" | awk 'NF && !seen[$0]++' | while IFS= read -r remote_dir; do sync_header_dir "$remote_dir" @@ -170,34 +152,30 @@ ln -s "usr-src/$build_name" "$build_dir" echo "[4/5] checking vmlinux debug image" if lab_ssh_sudo "test -r '$remote_vmlinux'"; then - # Use rsync over sudo: skips when unchanged, restartable, checksummed. - # Without this the cat-based fetch would silently corrupt vmlinux any - # time sudo printed anything unexpected to stdout or the SSH connection - # dropped mid-transfer (the file is ~415MB so the window is non-trivial). + # rsync-over-sudo: skips unchanged, restartable, checksummed. Replaces + # an earlier cat-over-ssh approach that silently corrupted vmlinux on + # any sudo banner or connection drop (file is ~415MB). echo " rsync $remote_vmlinux -> ${cache_dir#"$repo_root"/}/vmlinux" lab_rsync_from "$remote_vmlinux" "$cache_dir/vmlinux" --rsync-path='sudo rsync' else rm -f "$cache_dir/vmlinux" echo " no vmlinux on target ($remote_vmlinux is missing)" echo " module debugging will work; kernel source debugging will not" - echo " to enable, run: scripts/01-provision-target.sh $target --debug-symbols" + echo " to enable: scripts/01-provision-target.sh $target --debug-symbols" fi # --- Kernel source resolution -------------------------------------------- # -# Find a usable kernel-source root under usr-src/. Ubuntu has shipped three -# layouts over time: +# Ubuntu has shipped three linux-source layouts: # 1. /usr/src/linux-source-X.tar.bz2 (tarball, no enclosing dir) -# 2. /usr/src/linux-source-X/ (pre-extracted, files at top) -# 3. /usr/src/linux-source-X/linux-source-X/ (pre-extracted, nested one deep) -# Layout 1 also occurs nested as /usr/src/linux-source-X/linux-source-X.tar.bz2. +# 2. /usr/src/linux-source-X/ (extracted, files at top) +# 3. /usr/src/linux-source-X/linux-source-X/ (extracted, nested one deep) +# Plus the variant where the tarball lives inside the same-named dir. # -# Strategy: look for a Makefile + init/main.c (the kernel's top-level -# markers) at depth ≤ 3 under usr-src/. If found, symlink directly to it; -# otherwise look for a tarball and extract on the host into source-tree/. -# Extraction on the host (not the target) avoids a sudo+tar dependency on -# the target for what's purely a host-side debug resource, and makes the -# step self-healing: re-running 02 fixes partial/broken trees. +# Strategy: look for a Makefile + init/main.c (the kernel-source markers) +# at depth ≤ 3 under usr-src/. If found, symlink to it. Otherwise extract +# a tarball into source-tree/ on the host (self-healing on rerun, and +# avoids needing tar on the target). echo "[5/5] resolving kernel source for step-into-kernel" source_link="$cache_dir/source" @@ -208,17 +186,14 @@ is_kernel_source_root() { [[ -f "$1/Makefile" && -d "$1/init" && -f "$1/init/main.c" ]] } -# Pick the source dir matching the live kernel's major.minor.patch when -# possible (handles the case where multiple linux-source-X packages are -# installed on the target). Fall back to the highest-versioned root. +# Prefer the linux-source-X dir whose X matches the live kernel; fall +# back to the highest version installed. short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" found_root="" matching_root="" while IFS= read -r mf; do candidate="$(dirname "$mf")" if is_kernel_source_root "$candidate"; then - # `sort -V` below means "first match" is also the highest version; - # remember the first hit, then prefer a kver-matching one if seen. [[ -z "$found_root" ]] && found_root="$candidate" if [[ -n "$short_kver" && "$candidate" == *"linux-source-$short_kver"* ]]; then matching_root="$candidate" @@ -252,7 +227,7 @@ else else rm -rf "$source_tree" echo " no kernel source on target; kernel step-into will show disassembly only" - echo " to enable, run: scripts/01-provision-target.sh $target --debug-symbols" + echo " to enable: scripts/01-provision-target.sh $target --debug-symbols" fi fi diff --git a/scripts/03-build-module.sh b/scripts/03-build-module.sh index 2404f6b..d34d096 100755 --- a/scripts/03-build-module.sh +++ b/scripts/03-build-module.sh @@ -1,11 +1,10 @@ #!/usr/bin/env bash # Build module/ against the synced headers for . # -# Driven by the top-level Makefile, which stages module/ into +# Delegates to the top-level Makefile, which stages module/ into # build/intermediate/// and runs Kbuild from there with -# -ffile-prefix-map injected via KCFLAGS. Result is one .ko under -# build/artifacts///. The module name is whatever the user's -# obj-m declares; the lab does not need to know it. +# -ffile-prefix-map injected via KCFLAGS. Result: one .ko under +# build/artifacts///. set -euo pipefail target="${1:-}" diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index dcf1c81..976f0b0 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -2,25 +2,18 @@ # Deploy the built module to a target, load it, and write the GDB init files # VS Code (or `gdb -tui`) needs to attach and source-debug it. # -# Steps: -# 1. Free the debug-endpoint TCP port from any stale GDB clients. Both -# KGDB's serial bridge and QEMU's gdbstub serve one client at a time, -# so any leftover gdb owning that socket blocks fresh attaches. This -# script kills any *local* gdb process holding the endpoint — be aware -# if you have unrelated gdb sessions on the same host:port. -# 2. Upload module/.ko to the target. -# 3. Insmod it (with debug_delay_ms if the module declares the param, -# falling back to a plain insmod otherwise). -# 4. Poll /sys/module//sections/ for the module's runtime load -# addresses; convert to an `add-symbol-file ... -s .name addr ...` -# line saved as the per-(target,method) symbols file. -# 5. Emit .gdb/-.gdb and -attached.gdb variants, plus -# current-debug{,attached}.gdb and current-vmlinux symlinks the IDE -# launch configs point at. +# Phases: +# 1. Best-effort kill of any local gdb holding the debug-endpoint TCP +# port (KGDB serial bridge and QEMU gdbstub each serve one client). +# 2. Upload module/.ko, verify size matches. +# 3. Insmod in the background; poll /sys/module//sections/ for the +# runtime load addresses and emit an `add-symbol-file ...` line. +# 4. Emit .gdb/-.gdb (+ -attached variant) and the +# current-* symlinks the IDE launch configs point at. # -# Env knobs (default values in parens): +# Env knobs: # DEBUG_LOAD_DELAY_MS (5000) passed to the module if it declares debug_delay_ms -# INSMOD_WAIT_SECS (30) how long to wait for /sys/module/.../sections/.text +# INSMOD_WAIT_SECS (30) poll deadline for /sys/module/.../sections/.text set -euo pipefail usage() { @@ -42,12 +35,11 @@ env_file="$(lab_env_file "$repo_root")" source "$env_file" lab_load_target "$target" -endpoint_key="DEBUG_ENDPOINT_${debug_method^^}" -debug_endpoint="$(target_require_cfg "$target" "$endpoint_key")" +debug_endpoint="$(target_require_cfg "$target" "DEBUG_ENDPOINT_${debug_method^^}")" debug_delay_ms="${DEBUG_LOAD_DELAY_MS:-5000}" insmod_wait_secs="${INSMOD_WAIT_SECS:-30}" -# --- Locate built artifact + vmlinux -------------------------------------- +# --- Locate artifact + vmlinux, verify target is on the cached kernel ---- cache_dir="$repo_root/.kernel-cache/$target" kernel_file="$cache_dir/kernel.release" @@ -55,7 +47,6 @@ vmlinux="$cache_dir/vmlinux" [[ -f "$kernel_file" ]] || die "missing kernel cache for $target; run: scripts/02-setup-host-build.sh $target" - [[ -f "$vmlinux" ]] || die "missing $vmlinux — source debugging needs the matching vmlinux from the target. - install + sync it with: @@ -64,18 +55,14 @@ vmlinux="$cache_dir/vmlinux" kernel="$(<"$kernel_file")" -# Preflight: connection + sudo work, and the target is still running the -# kernel we cached. If the target rebooted into a different kernel since -# scripts/02 ran, vmlinux symbols won't match runtime addresses and GDB -# would silently show wrong source lines. lab_check_connection live_kernel="$(lab_ssh 'uname -r')" if [[ "$live_kernel" != "$kernel" ]]; then - die "kernel mismatch: target is now running '$live_kernel' but cache has '$kernel'. -- if the target was rebooted into a new kernel, re-sync: - scripts/01-provision-target.sh $target $debug_method --debug-symbols # if you also need the new vmlinux + die "kernel mismatch: target now runs '$live_kernel' but cache has '$kernel'. +- if the target rebooted into a new kernel, re-sync: + scripts/01-provision-target.sh $target $debug_method --debug-symbols # for a new vmlinux scripts/02-setup-host-build.sh $target -- if you booted the wrong kernel, reboot back into $kernel" +- or reboot the target back into $kernel" fi artifact_dir="$repo_root/build/artifacts/$target/$kernel" @@ -89,13 +76,13 @@ case ${#artifacts[@]} in *) echo "expected one .ko under $artifact_dir; found:" >&2 printf ' %s\n' "${artifacts[@]}" >&2 - die "the lab assumes a single obj-m per build. Merge sources into one module (obj-m += foo.o; foo-y := a.o b.o ...) or split into separate module/ trees." + die "the lab assumes a single obj-m per build. Merge into one module (obj-m += foo.o; foo-y := a.o b.o ...) or split into separate module/ trees." ;; esac artifact="${artifacts[0]}" module_name="$(basename "$artifact" .ko)" -# --- GDB output paths ----------------------------------------------------- +# --- Output paths --------------------------------------------------------- gdb_dir="$repo_root/.gdb" mkdir -p "$gdb_dir" @@ -107,23 +94,16 @@ loader_log="$gdb_dir/$target-$debug_method-loader.log" rm -f "$symbols_file" "$loader_log" -# --- Free the debug endpoint from stale GDB clients ----------------------- -# -# Both KGDB's serial bridge and QEMU's gdbstub serve one client at a time. -# A zombie gdb left over from a previous session sits in the endpoint's -# accept queue and blocks fresh attaches. We best-effort kill any local gdb -# process that holds a connection to the endpoint. Errors here never fail -# the deploy — at worst the user has to `killall gdb` manually. +# --- Free the debug endpoint from stale GDB clients ---------------------- # -# `lsof -ti` is more robust than parsing `ss -ntp` output: it gives one PID -# per line and won't break across iproute2 versions. +# Best-effort: lsof to find sockets on the endpoint, ps to confirm comm=gdb, +# kill -9. Errors here never fail the deploy. free_debug_endpoint() { command -v lsof >/dev/null 2>&1 || return 0 local host="${debug_endpoint%:*}" port="${debug_endpoint##*:}" local pids pids="$(lsof -ti "@$host:$port" 2>/dev/null || true)" - local gdb_pids=() - local pid + local gdb_pids=() pid for pid in $pids; do [[ "$(ps -p "$pid" -o comm= 2>/dev/null || true)" == "gdb" ]] && gdb_pids+=("$pid") done @@ -136,71 +116,57 @@ free_debug_endpoint() { } free_debug_endpoint || true -# --- Upload + load on target ---------------------------------------------- +# --- Upload + load ------------------------------------------------------- remote_module="$LAB_REMOTE_DIR/$module_name.ko" echo "uploading $artifact -> $LAB_SSH_TARGET:$remote_module" lab_ssh "mkdir -p '$LAB_REMOTE_DIR'" lab_scp_to "$artifact" "$remote_module" -# Verify the upload — a truncated .ko would silently insmod-fail with cryptic -# "invalid module format" errors. +# Verify size — a truncated .ko produces cryptic "invalid module format" errors. local_size="$(stat -c %s "$artifact")" remote_size="$(lab_ssh "stat -c %s '$remote_module' 2>/dev/null" | tr -d '[:space:]')" [[ "$remote_size" == "$local_size" ]] || die "upload size mismatch: local $local_size bytes, remote ${remote_size:-missing} bytes; retry scripts/04 or check disk space on the target" -# Load the module and discover the addresses /sys/module//sections/ -# exposes once it's live. The insmod runs inside a single nohup'd sh -c so -# the debug_delay_ms-then-plain-insmod fallback chains correctly: if we -# instead ran two `nohup ... &` invocations the shell would background them -# independently and the `||` between them would be a no-op (it'd just check -# the success of the backgrounding, not the insmod itself). +# Background insmod on the target so we can poll for sections in parallel +# with the module's debug_delay_ms sleep. +# +# Three load-bearing details: +# - `<&0` keeps sudo's stdin attached to the SSH-inherited pipe (where +# the password arrives). Non-job-control bash auto-redirects async +# stdin to /dev/null without an explicit redirection — sudo would +# see EOF and silently fail to authenticate. +# - `& sleep 1` keeps the SSH session open long enough for sudo to +# authenticate and exec nohup before the channel closes. +# - `nohup` + non-interactive bash → the orphaned chain survives +# session exit (bash doesn't huponexit non-interactively). # -# Output is captured to $loader_log; we surface either the success tail -# line or the full log on failure. +# `|| insmod ...` falls back when the module doesn't declare a +# debug_delay_ms parameter (kv arg would otherwise reject with EINVAL). load_and_discover_symbols() { echo "loading $module_name on $target (debug_delay_ms=$debug_delay_ms)" lab_ssh_sudo "rmmod '$module_name' >/dev/null 2>&1 || true" - # Background insmod on the target so we can poll /sys/module/.../sections - # in parallel with the module's debug_delay_ms sleep. - # - # Three subtleties: - # - `<&0` keeps sudo's stdin attached to the SSH-inherited pipe (where - # the password arrives). Non-job-control bash automatically redirects - # backgrounded processes' stdin to /dev/null UNLESS the command has - # its own stdin redirection — without `<&0`, sudo would see EOF - # instead of the password and silently fail to authenticate. - # - `& sleep 1` keeps the SSH session open long enough for sudo to - # read the password and exec into nohup. The sleep is short because - # sudo authentication is fast; if SSH closed earlier the in-flight - # password write could be lost. - # - The orphaned nohup+sh+insmod chain survives session exit because - # non-interactive bash doesn't huponexit. - # - # The `|| insmod ...` falls back when the module doesn't declare a - # debug_delay_ms parameter (the kv arg would otherwise reject with EINVAL). lab_ssh_sudo "nohup sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$LAB_REMOTE_DIR/insmod.log' 2>&1 <&0 & sleep 1" echo "polling /sys/module/$module_name/sections/ (timeout ${insmod_wait_secs}s)" - # Dump every readable file under /sys/module//sections/. Discovery - # is dynamic so modules with non-standard sections (.text.hot, custom - # __ksymtab subsections, ...) get their addresses picked up too. + # Dump every readable file under sections/. Dynamic discovery handles + # non-standard sections (.text.hot, custom __ksymtab subsections, ...). # # The sh -c body MUST be single-quoted in the SSH command so the outer - # remote bash doesn't expand $(ls -A) and $f before sh -c sees them. - # With double quotes, the outer bash would evaluate the substitution in - # its own CWD; the for loop would iterate over the wrong filenames and - # we'd never find .text — even though the module had already loaded. + # remote bash doesn't expand $(ls -A) and $f before sh -c sees them — + # with double quotes the outer bash would expand in its own CWD and + # the for loop would iterate over the wrong filenames. local dump_body - # shellcheck disable=SC2016 # $(...) is intentionally not expanded locally — see comment above. + # shellcheck disable=SC2016 # $(...) is intentionally deferred to sh -c. dump_body='cd "/sys/module/'"$module_name"'/sections" 2>/dev/null && for f in $(ls -A 2>/dev/null); do [ -r "$f" ] && printf "%s %s\n" "$f" "$(cat "$f")"; done' - local deadline=$((SECONDS + insmod_wait_secs)) - local tmp + local deadline=$((SECONDS + insmod_wait_secs)) tmp tmp="$(mktemp)" while ((SECONDS < deadline)); do + # No sleep — each SSH round-trip already takes 0.3-1s, which is + # the right polling cadence. if lab_ssh_sudo "sh -c '$dump_body'" > "$tmp" 2>/dev/null; then local text_addr text_addr="$(awk '$1 == ".text" { print $2 }' "$tmp")" @@ -218,8 +184,6 @@ load_and_discover_symbols() { return 0 fi fi - # No explicit sleep — each SSH round-trip already takes 0.3-1s, which - # is the right polling cadence. done rm -f "$tmp" @@ -241,12 +205,12 @@ else exit 1 fi -# --- Build the GDB init files -------------------------------------------- +# --- GDB init files ------------------------------------------------------ # -# If the kernel source has been synced (scripts/01 --debug-symbols + scripts/02), -# discover Ubuntu's build-time source prefix from vmlinux so GDB can remap -# DWARF references to our local copy. We ask addr2line where `start_kernel` -# lives — the path is always /init/main.c, so stripping the +# If the kernel source has been synced (scripts/01 --debug-symbols + +# scripts/02), discover Ubuntu's build-time source prefix from vmlinux so +# GDB can remap DWARF references to our local copy. addr2line on +# `start_kernel` returns `/init/main.c`; stripping the # known suffix yields the prefix. kernel_substitute_line="" kernel_directory_line="" @@ -255,9 +219,8 @@ if [[ -L "$kernel_src_link" || -d "$kernel_src_link" ]]; then kernel_src_root="$(readlink -f "$kernel_src_link")" kernel_directory_line="directory $kernel_src_root" if command -v nm >/dev/null 2>&1 && command -v addr2line >/dev/null 2>&1; then - # `|| true` because pipefail + SIGPIPE: nm dumps every vmlinux - # symbol, awk's `exit` after the first match closes the pipe and nm - # gets SIGPIPE on its next write. We still capture awk's output. + # `|| true` because awk's `exit` SIGPIPEs nm, and pipefail would + # kill the script. We still capture awk's output before exit. sym_addr="$(nm "$vmlinux" 2>/dev/null | awk 'NF==3 && $3=="start_kernel" {print $1; exit}' || true)" if [[ -n "$sym_addr" ]]; then sym_loc="$(addr2line -e "$vmlinux" "$sym_addr" 2>/dev/null | head -1 | cut -d: -f1 || true)" @@ -301,9 +264,9 @@ source $symbols_file EOF } > "$gdb_file" -# Native Debug ("type": "gdb") and cppdbg with miDebuggerServerAddress have -# already called `target remote` and loaded the executable by the time they -# source our script, so commands that touch global gdb state error with +# Native Debug and cppdbg-with-miDebuggerServerAddress already called +# `target remote` and loaded the executable by the time they source this +# script — so commands that change global gdb state error with # "Cannot change this setting while the inferior is running". Strip them. grep -vE '^(target remote |set mi-async |set target-async |set tcp connect-timeout |set remotetimeout |set architecture |symbol-file )' \ "$gdb_file" > "$gdb_attached_file" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index bb987d4..220c1b0 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -1,18 +1,10 @@ #!/usr/bin/env bash -# scripts/lib/common.sh — shared helpers sourced by every numbered script. +# Shared helpers sourced by every numbered script. # -# Naming: -# die, validate_target, target_* bare functions; no global state required. -# lab_* higher-level helpers; rely on globals -# populated by `lab_load_target`. -# LAB_* globals populated by `lab_load_target`. -# Scripts should call lab_ssh / lab_ssh_sudo -# / lab_scp_to / lab_rsync_from rather than -# touching the underlying ssh/scp/sshpass -# binaries directly. -# -# Sourcing this file is side-effect-free. State is created when a script -# calls `lab_load_target ` after sourcing `lab.local.env`. +# `die`, `validate_target`, `target_*` are state-free. +# `lab_*` helpers rely on globals set by `lab_load_target`. +# `LAB_*` globals are set by `lab_load_target`. Use the helpers, not the +# raw ssh/scp/sshpass binaries. die() { echo "$*" >&2 @@ -66,38 +58,30 @@ target_require_cfg() { require_debian_host() { command -v apt-get >/dev/null 2>&1 || - die "missing apt-get; the development host must be Debian-based (Debian, Ubuntu, WSL Ubuntu, ...)" + die "the development host must be Debian-based (apt-get not found)" } # --- Lab env loading ------------------------------------------------------ -# Validate that lab.local.env exists under repo_root and return its path on -# stdout. Caller is expected to `source` the returned path directly so the -# `declare -A TARGET_*=...` entries land in the script's global scope (a -# function-internal source would scope them to the function). +# Callers `source` the returned path directly so TARGET_* assignments land +# in script-global scope (sourcing from inside a function would scope them +# to the function). lab_env_file() { - local repo_root="$1" - local env_file="$repo_root/lab.local.env" + local env_file="$1/lab.local.env" [[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it for your machines" printf '%s' "$env_file" } # --- Per-target connection setup ------------------------------------------ -# Read TARGET_* config for the given target and populate LAB_* globals used -# by every remote helper below. Validates that ssh password tooling is -# available if a password is configured. +# Read TARGET_* config and populate LAB_* globals: LAB_TARGET, LAB_SSH_HOST, +# LAB_SSH_PORT, LAB_SSH_USER, LAB_SSH_PASS, LAB_SUDO_PASS, LAB_REMOTE_DIR, +# LAB_SSH_TARGET, LAB_SSH_CMD[], LAB_SCP_CMD[], LAB_RSYNC_RSH. # -# After this returns, the following globals are set: -# LAB_TARGET target name -# LAB_SSH_HOST/PORT/USER raw connection details -# LAB_SSH_PASS ssh password ("" when using keys) -# LAB_SUDO_PASS sudo password ("" when sudo is passwordless / NOPASSWD) -# LAB_REMOTE_DIR target staging dir (with per-user default) -# LAB_SSH_TARGET "user@host" form for use in commands -# LAB_SSH_CMD array, ready to invoke (includes sshpass + timeouts) -# LAB_SCP_CMD array, ready to invoke (includes sshpass + timeouts) -# LAB_RSYNC_RSH rsync -e value (includes sshpass + timeouts) +# LAB_SSH_CMD / LAB_SCP_CMD are flag-only — they do NOT include the host. +# Helpers (and the few direct callers) append it themselves, so options +# like `-t` can be inserted before the host (ssh treats anything after +# the host as the remote command). lab_load_target() { local target="$1" target_is_configured "$target" @@ -112,9 +96,7 @@ lab_load_target() { LAB_REMOTE_DIR="$(target_cfg "$target" REMOTE_DIR)" LAB_SSH_PORT="${LAB_SSH_PORT:-22}" - # Sudo password falls back to ssh password — common single-user dev case. LAB_SUDO_PASS="${LAB_SUDO_PASS:-$LAB_SSH_PASS}" - # Per-user default so two devs on the same host don't collide on /tmp. LAB_REMOTE_DIR="${LAB_REMOTE_DIR:-/tmp/kmod-debug-lab-$LAB_SSH_USER}" local ssh_bin scp_bin sshpass_bin @@ -123,20 +105,14 @@ lab_load_target() { sshpass_bin="${SSHPASS_BIN:-sshpass}" if [[ -n "$LAB_SSH_PASS" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - die "TARGET_SSH_PASS[$target] is set but '$sshpass_bin' is not installed; install it with: sudo apt-get install -y sshpass" + die "TARGET_SSH_PASS[$target] is set but '$sshpass_bin' isn't installed; install with: sudo apt-get install -y sshpass" fi LAB_SSH_TARGET="$LAB_SSH_USER@$LAB_SSH_HOST" - # LAB_SSH_CMD and LAB_SCP_CMD are flag-only — they do NOT include the - # target. Helpers and direct callers append the target themselves so - # `ssh ... -t user@host cmd` works (anything after the host is treated - # as the remote command by ssh). - # - # ConnectTimeout fails fast (10s) if the target is unreachable instead - # of blocking for minutes. ServerAliveInterval keeps long-lived - # connections (e.g. the 30s symbol-discovery loop) from being dropped - # by NAT idle timers. + # ConnectTimeout: fail fast on dead targets instead of multi-minute TCP wait. + # ServerAliveInterval: keep long-lived connections (e.g. the polling loop) + # alive through NAT idle timers. local ssh_opts=( -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 @@ -153,50 +129,37 @@ lab_load_target() { } # --- Remote command helpers ----------------------------------------------- -# -# All of these require a prior `lab_load_target` call. +# All require a prior `lab_load_target` call. -# Run a command on the target. Arguments are passed through to ssh as the -# remote command (typically one quoted shell string). Inherits stdin from -# the caller — handy when callers want to pipe data in (e.g. lab_ssh_sudo -# pipes the sudo password). +# Inherits stdin from the caller (so a caller can pipe data into the remote +# command — see lab_ssh_sudo). lab_ssh() { SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "$@" } -# Run a command on the target as root. The sudo password reaches the remote -# `sudo -S` via ssh's stdin, which means it never appears in any process's -# argv on the target — `ps auxww` is clean. When LAB_SUDO_PASS is empty -# (NOPASSWD sudo or sudo configured for passwordless), the empty string is -# piped and sudo proceeds without prompting. +# Run a command on the target as root. The password reaches `sudo -S` via +# ssh's stdin, so it never appears in argv on either host. # -# Why `-k`: without it, if sudo has cached credentials from an earlier -# invocation (e.g. `scripts/00-check-target.sh` just ran, or the user did -# `sudo` on the target within the last 5 min), `sudo -S` skips the stdin -# read entirely. The piped password then leaks into whatever inherits -# stdin from sudo — for `sudo bash -s`, the password line becomes bash's -# first script line and you get `bash: line 1: a: command not found`. -# `-k` ignores the cache for THIS invocation only; it does NOT invalidate -# the user's existing sudo timestamp. +# `-k` ignores any cached sudo timestamp (e.g. left by `scripts/00-check-target.sh`) +# for THIS invocation only — without it, sudo would skip the stdin read and +# the password would leak into the next reader. It does NOT invalidate the +# user's existing sudo cache. # -# Callers that need to feed their own stdin to the remote command (e.g. -# rsync) should use `lab_rsync_from` / scp helpers instead — those take a -# different path that doesn't compete for stdin. +# For callers where the remote command itself reads stdin (e.g. `bash -s`), +# this helper is not enough — NOPASSWD policy also skips the stdin read, +# leaking the password. Use SUDO_ASKPASS instead (see scripts/01 for the +# pattern). lab_ssh_sudo() { printf '%s\n' "$LAB_SUDO_PASS" | SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "sudo -k -S -p '' $*" } -# Copy a local file to the target. The remote path is absolute. lab_scp_to() { - local local_path="$1" - local remote_path="$2" - SSHPASS="$LAB_SSH_PASS" "${LAB_SCP_CMD[@]}" "$local_path" "$LAB_SSH_TARGET:$remote_path" + SSHPASS="$LAB_SSH_PASS" "${LAB_SCP_CMD[@]}" "$1" "$LAB_SSH_TARGET:$2" } -# Rsync a remote path (file or dir) into a local destination. Extra rsync -# args can be appended — notably `--rsync-path='sudo rsync'` for paths only -# root can read (vmlinux), and `--delete` for tree mirroring. +# Extra rsync flags can be appended — notably `--rsync-path='sudo rsync'` +# for paths only root can read (vmlinux), and `--delete` for tree mirroring. lab_rsync_from() { local remote_src="$1" local local_dst="$2" @@ -207,14 +170,9 @@ lab_rsync_from() { # --- Preflight ------------------------------------------------------------ -# Fast, non-mutating connection check. Verifies SSH reachable, and (by -# default) that sudo works. Pass `--no-sudo` to skip the sudo test — useful -# in `scripts/01-provision-target.sh`, which a fresh user might be running -# *to* configure sudo for the first time. -# -# Called at the top of 01/02/04 so failures surface before any real work. -# `scripts/00-check-target.sh` runs a longer-form report on top of this. -# shellcheck disable=SC2120 # arg is optional (--no-sudo); callers without it are intentional. +# `--no-sudo` skips the sudo test, for scripts/01 which may be running TO +# configure sudo for the first time. +# shellcheck disable=SC2120 # --no-sudo arg is optional. lab_check_connection() { local check_sudo=1 [[ "${1:-}" == "--no-sudo" ]] && check_sudo=0 @@ -222,13 +180,13 @@ lab_check_connection() { if ! lab_ssh 'true' 2>/dev/null; then die "cannot reach $LAB_SSH_TARGET over ssh (port $LAB_SSH_PORT). - check TARGET_SSH_HOST/PORT/USER for '$LAB_TARGET' in lab.local.env -- if the VM is up, try by hand: ssh -p $LAB_SSH_PORT $LAB_SSH_TARGET -- if you're using passwords, confirm TARGET_SSH_PASS[$LAB_TARGET] is set" +- try by hand: ssh -p $LAB_SSH_PORT $LAB_SSH_TARGET +- if using passwords, confirm TARGET_SSH_PASS[$LAB_TARGET] is set" fi if (( check_sudo )) && ! lab_ssh_sudo 'true' 2>/dev/null; then die "ssh reaches $LAB_SSH_TARGET but sudo doesn't work there. - check TARGET_SUDO_PASS[$LAB_TARGET] in lab.local.env (defaults to TARGET_SSH_PASS) -- on the target, confirm: sudo -n -v (or run sudo by hand once to cache creds) -- consider configuring NOPASSWD for the lab user on long-lived dev targets" +- on the target, confirm: sudo -n -v +- consider NOPASSWD for the lab user on long-lived dev targets" fi }