From 30cacecca38c4c53896a152dc0021a183d6d8c9e Mon Sep 17 00:00:00 2001 From: dor Date: Thu, 21 May 2026 00:02:07 +0300 Subject: [PATCH 01/13] no dependencies on wsl --- .gitignore | 4 +- .vscode/launch.json | 30 +- .vscode/tasks.json | 18 +- README.md | 481 ++++++++---------- lab.example.env | 95 ++-- scripts/01-provision-target.sh | 53 +- ...up-wsl-build.sh => 02-setup-host-build.sh} | 57 +-- scripts/03-build-module.sh | 16 +- scripts/04-deploy-debug-vscode.sh | 64 +-- scripts/lib/common.sh | 54 ++ 10 files changed, 407 insertions(+), 465 deletions(-) rename scripts/{02-setup-wsl-build.sh => 02-setup-host-build.sh} (82%) create mode 100644 scripts/lib/common.sh diff --git a/.gitignore b/.gitignore index 00f934b..9fd6cc1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,7 @@ lab.local.env .kernel-cache/ -.gdb/*.gdb -.gdb/*.ready -.gdb/*.log +.gdb/ build/ *.ko diff --git a/.vscode/launch.json b/.vscode/launch.json index 9c396ab..cc0391d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,39 +2,19 @@ "version": "0.2.0", "configurations": [ { - "name": "Kernel: Debug server", + "name": "Kernel: Debug", "type": "cppdbg", "request": "launch", - "program": "${workspaceFolder}/.kernel-cache/server/vmlinux", + "program": "${workspaceFolder}/.gdb/current-vmlinux", "cwd": "${workspaceFolder}", "MIMode": "gdb", "miDebuggerPath": "gdb", "targetArchitecture": "x64", - "preLaunchTask": "Kernel: Deploy Debug server", + "preLaunchTask": "Kernel: Deploy Debug", "setupCommands": [ { - "description": "Attach KGDB and trigger deferred module load", - "text": "source ${workspaceFolder}/.gdb/server-kgdb.gdb", - "ignoreFailures": false - } - ], - "launchCompleteCommand": "exec-continue", - "externalConsole": false - }, - { - "name": "Kernel: Debug desktop", - "type": "cppdbg", - "request": "launch", - "program": "${workspaceFolder}/.kernel-cache/desktop/vmlinux", - "cwd": "${workspaceFolder}", - "MIMode": "gdb", - "miDebuggerPath": "gdb", - "targetArchitecture": "x64", - "preLaunchTask": "Kernel: Deploy Debug desktop", - "setupCommands": [ - { - "description": "Attach KGDB and trigger deferred module load", - "text": "source ${workspaceFolder}/.gdb/desktop-kgdb.gdb", + "description": "Attach GDB to the selected target", + "text": "source ${workspaceFolder}/.gdb/current-kgdb.gdb", "ignoreFailures": false } ], diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 4a3e933..2c4d58e 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -4,7 +4,7 @@ { "id": "kernelTarget", "type": "pickString", - "description": "Ubuntu 24.04 target VM", + "description": "Target profile", "options": [ "desktop", "server" @@ -20,9 +20,9 @@ "problemMatcher": [] }, { - "label": "Kernel: Setup WSL Build", + "label": "Kernel: Setup Host Build", "type": "shell", - "command": "./scripts/02-setup-wsl-build.sh ${input:kernelTarget}", + "command": "./scripts/02-setup-host-build.sh ${input:kernelTarget}", "problemMatcher": [] }, { @@ -40,18 +40,6 @@ "type": "shell", "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget}", "problemMatcher": [] - }, - { - "label": "Kernel: Deploy Debug server", - "type": "shell", - "command": "./scripts/04-deploy-debug-vscode.sh server", - "problemMatcher": [] - }, - { - "label": "Kernel: Deploy Debug desktop", - "type": "shell", - "command": "./scripts/04-deploy-debug-vscode.sh desktop", - "problemMatcher": [] } ] } diff --git a/README.md b/README.md index e8eac41..ee1de01 100644 --- a/README.md +++ b/README.md @@ -1,363 +1,321 @@ -# Ubuntu 24.04 Kernel Module Debug Lab +# Kernel Module Development Lab -This repo is a WSL-first lab for building a Linux kernel module and -experimenting with source debugging on VMware Ubuntu 24.04 LTS Desktop and -Server targets. +This repo is a development setup for building and debugging Linux kernel +modules against target VMs. The sample module builds as `hello.ko` and exposes `/dev/chuck_norise`. Reads repeat the exact string `chuck norise!`, preserving the file offset for each open file descriptor. -## Quick Start +## Machine Model -1. Copy `lab.example.env` to `lab.local.env`. -2. Fill in the SSH values for `desktop` and `server`. Fill in the debug - endpoint values too if you want to use F5 debugging. -3. Provision each VM once, then reboot it: +The lab uses two kinds of machines: -```bash -./scripts/01-provision-target.sh desktop -./scripts/01-provision-target.sh server -``` +1. A Debian-based development host. + This can be Debian, Ubuntu Desktop, Ubuntu under WSL, or another + Debian-based distro. The scripts assume `apt`/`apt-get`, `bash`, `ssh`, + `scp`, `rsync`, `make`, and standard GNU userland tools. -4. Sync each target's kernel headers into WSL: +2. A target VM. + The target is where the module is loaded and tested. Communication with the + target is over SSH. For now, the supported target profiles are Ubuntu 24.04 + Desktop (`desktop`) and Ubuntu 24.04 Server (`server`). Other Linux target + distros can be added later by teaching the provisioning and sync scripts how + to install headers, find kernel build trees, and configure debugging for that + distro. -```bash -./scripts/02-setup-wsl-build.sh desktop -./scripts/02-setup-wsl-build.sh server -``` +For debugging, GDB on the development host connects to an IP address and port. +That endpoint can be opened however you like. This repo documents two provider +options: VMware's built-in debug stub and a VMware serial port bridged to TCP. -5. Build from VS Code Remote-WSL with `Ctrl+Shift+B`. -6. To try source debugging, configure one of the debug endpoint options below, - then choose `Kernel: Debug desktop` or `Kernel: Debug server` and press F5. +## Configure The Lab -## Building - -In VS Code Remote-WSL, `Ctrl+Shift+B` runs the default `Kernel: Build` task. -The task prompts for `kernelTarget` and runs: +Copy the example environment file and edit it for your machines: ```bash -./scripts/03-build-module.sh +cp lab.example.env lab.local.env ``` -The built module is written to: +Targets are data, not variable prefixes. Add profile names to `TARGETS`, then +fill the `TARGET_*` maps with entries keyed by that profile name: -```text -build/artifacts///hello.ko +```bash +TARGETS=(desktop server) + +declare -A TARGET_OS=( + [desktop]=ubuntu + [server]=ubuntu +) + +declare -A TARGET_SSH_HOST=( + [desktop]=ubuntu-desktop.local + [server]=ubuntu-server.local +) ``` -Build intermediates are kept under `build/intermediate///`, so -Kbuild does not leave `.o`, `.cmd`, `Module.symvers`, or `modules.order` files -in the repo root or `module/src/`. +Target names may contain letters, numbers, `_`, or `-`. A target named +`ubuntu-server` is configured with map keys like `[ubuntu-server]=...`. -Running plain `make` uses `.kernel-cache/current` when it exists. The setup -script updates that link to the most recently synced target. +The debug endpoint is the host and port that GDB should connect to from the +development host: -## Source Debugging Status +```bash +declare -A TARGET_DEBUG_ENDPOINT=( + [server]=127.0.0.1:8864 +) +``` -Source debugging is work in progress. The pieces under active development are -`scripts/04-deploy-debug-vscode.sh` and the VS Code launch tasks. +For now, target provisioning and header sync implement `TARGET_OS[...]=ubuntu`. +Adding another target distro should happen by adding a new target OS backend to +the scripts instead of special-casing a profile name. -Today, F5 runs the matching deploy/debug task for the selected target. Script 04 -expects `hello.ko` to already exist, uploads it to the VM, loads it, reads the -module section addresses from `/sys/module/hello/sections/*`, writes a GDB -symbol file under `.gdb/`, and generates the GDB startup script that VS Code -uses. The current goal is to load module symbols and stop in -`module/src/hello.c`; expect rough edges while this flow is being tightened. +## Prepare A Target -The VS Code launch configs pin the debugger architecture to `x64` while the -generated GDB script sets GDB's kernel architecture to `i386:x86-64`. This -avoids debug-adapter architecture auto-detection failures and lets GDB use the -Linux kernel architecture name it expects. The deploy script also maps generated -Kbuild source paths back to `module/` with `set substitute-path`, so source -stepping opens the real module files instead of the staged build symlinks. +Provisioning is target-side setup. It currently supports Ubuntu targets. -If F5 fails at `target remote :` with a timeout, GDB could not reach -the endpoint configured by `_KGDB_ENDPOINT` in `lab.local.env`. Check -that the VMware debug stub or serial bridge is listening on the same host/port -and is reachable from WSL before starting the VS Code debug launch. +```bash +./scripts/01-provision-target.sh server +``` -## Debug Endpoint Options +This installs the running target kernel's headers and `rsync`. It also prepares +the target for the serial KGDB path by adding boot arguments such as: -GDB runs in WSL and needs a TCP endpoint for the target VM. Use one of these -options, then put that host and port in `_KGDB_ENDPOINT` in -`lab.local.env`. The variable name says `KGDB`, but it is also used for the -VMware debug stub endpoint. +```text +kgdboc=ttyS0,115200 nokaslr sysrq_always_enabled=1 +``` -### Option A: VMware Debug Stub (Recommended) +Reboot the target VM after provisioning. -Power off the VM, open the target VM's `.vmx` file, and add: +For source debugging with full kernel symbols, provision with debug symbols: -```text -debugStub.listen.guest64 = "TRUE" -debugStub.port.guest64 = "8864" -debugStub.listen.guest64.remote = "TRUE" -debugStub.hideBreakpoints = "FALSE" +```bash +./scripts/01-provision-target.sh server --debug-symbols ``` -Start the VM after saving the `.vmx` file. Set the matching endpoint in -`lab.local.env`, for example: +Then sync the target kernel headers, build tree, and optional `vmlinux` into the +development host: -```env -SERVER_KGDB_ENDPOINT=127.0.0.1:8864 +```bash +./scripts/02-setup-host-build.sh server ``` -If WSL cannot reach the Windows loopback address, use the Windows host address -visible from WSL instead. If both VMs may run at the same time, give each VM a -different `debugStub.port.guest64` value and match that port in -`lab.local.env`. +## Build -This option does not need a VMware serial port or the PowerShell bridge. +In VS Code, press `Ctrl+Shift+B`. The default `Kernel: Build` task prompts for +`kernelTarget` and runs: -### Option B: VMware Serial Port And Bridge +```bash +./scripts/03-build-module.sh +``` -Use this if you prefer Linux KGDB over a virtual serial port, or if the VMware -debug stub is not available in your VMware setup. The data path is: +The module is written to: ```text -VS Code in WSL -> GDB -> TCP port on Windows -> named pipe -> VMware serial port -> Ubuntu KGDB +build/artifacts///hello.ko ``` -The VM exposes a virtual serial port as a Windows named pipe. GDB cannot connect -to that pipe directly from WSL, so `host/bridge-kgdb.ps1` listens on a TCP port -and forwards bytes between TCP and the VMware named pipe. +Build intermediates are kept under: -#### Configure The VMware Serial Port +```text +build/intermediate/// +``` -Power off the VM before changing virtual hardware. In VMware Workstation: +Running plain `make` uses `.kernel-cache/current` when it exists. The setup +script updates that link to the most recently synced target. -1. Open the VM settings. -2. Add a `Serial Port` if the VM does not already have one. -3. Select `Use named pipe`. -4. Set the pipe name. -5. Select `This end is the server`. -6. Select `The other end is an application`. -7. Enable `Connect at power on`. +## Debug From VS Code -Use one pipe per VM. The repo defaults are: +The VS Code debug configuration uses the same target-picking mechanism as the +build tasks. The checked-in picker includes `desktop` and `server`; add new +profile names to `.vscode/tasks.json` if you add more target profiles. -```text -desktop VM: \\.\pipe\kgdb-desktop -server VM: \\.\pipe\kgdb-server -``` +1. Select `Kernel: Debug`. +2. Press F5. +3. VS Code runs the `Kernel: Deploy Debug` prelaunch task. +4. The task prompts for `kernelTarget`. +5. `scripts/04-deploy-debug-vscode.sh ` uploads and loads `hello.ko`, + prepares module symbols, and writes the current GDB files under `.gdb/`. +6. VS Code starts GDB and sources `.gdb/current-kgdb.gdb`. -The first serial port in the VM normally appears as `ttyS0` inside Ubuntu. If -you add more serial ports or change the VM hardware order, adjust -`_KGDB_TTY` in `lab.local.env`. +Source debugging through `scripts/04-deploy-debug-vscode.sh` and the VS Code +launch task is still work in progress. The current goal is to load module +symbols and stop in `module/src/hello.c`. -#### Configure The Lab Environment +The generated GDB script sets the kernel architecture to `i386:x86-64`, loads +the target `vmlinux`, adds module symbols from `/sys/module/hello/sections/*`, +and maps staged Kbuild paths back to the real files under `module/`. -`lab.local.env` connects the VM serial device to the TCP endpoint used by GDB. -The default server values are: +## Debug Endpoint Options + +GDB only needs an endpoint in this form: -```env -SERVER_KGDB_ENDPOINT=127.0.0.1:5520 -SERVER_KGDB_TTY=ttyS0 -SERVER_KGDB_BAUD=115200 +```text +: ``` -The default desktop values are: +Put that value in `TARGET_DEBUG_ENDPOINT` in `lab.local.env`. The endpoint can +come from VMware's debug stub, the serial bridge below, or any other bridge that +speaks GDB remote protocol. -```env -DESKTOP_KGDB_ENDPOINT=127.0.0.1:5510 -DESKTOP_KGDB_TTY=ttyS0 -DESKTOP_KGDB_BAUD=115200 -``` +### Option A: VMware Debug Stub (Recommended) -The endpoint port must match the TCP port passed to `host/bridge-kgdb.ps1`. -The TTY and baud rate must match the guest kernel boot argument configured by -the provisioning script. +Power off the VM, open the target VM's `.vmx` file, and add: -#### Provision The Guest For KGDB +```text +debugStub.listen.guest64 = "TRUE" +debugStub.port.guest64 = "8864" +debugStub.listen.guest64.remote = "TRUE" +debugStub.hideBreakpoints = "FALSE" +``` -From WSL, run provisioning for the target and reboot the VM: +Start the VM after saving the `.vmx` file. Then set the matching endpoint: ```bash -./scripts/01-provision-target.sh server +declare -A TARGET_DEBUG_ENDPOINT=( + [server]=127.0.0.1:8864 +) ``` -Provisioning installs the target kernel headers and updates GRUB with KGDB boot -arguments like: +If the development host cannot reach the VMware host's loopback address, use an +address for the VMware host that is reachable from the development host. If both +VMs may run at the same time, give each VM a different +`debugStub.port.guest64` value. -```text -kgdboc=ttyS0,115200 nokaslr sysrq_always_enabled=1 -``` +This option does not need a VMware serial port or the PowerShell bridge. -Reboot is required. Without the reboot, the target kernel is still running -without KGDB on the serial port. +### Option B: VMware Serial Port To TCP -For source debugging against full kernel symbols, provision with debug symbols: +Use this path if you want Linux KGDB over a virtual serial port: -```bash -./scripts/01-provision-target.sh server --debug-symbols +```text +VS Code -> GDB -> TCP port -> named pipe -> VMware serial port -> Ubuntu KGDB ``` -Then sync the target kernel headers and `vmlinux` into WSL: +In VMware Workstation, power off the VM and add a serial port: -```bash -./scripts/02-setup-wsl-build.sh server -``` +1. Select `Use named pipe`. +2. Set a unique pipe name. +3. Select `This end is the server`. +4. Select `The other end is an application`. +5. Enable `Connect at power on`. -#### Start The Windows Pipe-To-TCP Bridge +The repo defaults are: -Run the bridge from Windows PowerShell, not from WSL. Start it before pressing -F5 in VS Code. +```text +desktop VM: \\.\pipe\kgdb-desktop +server VM: \\.\pipe\kgdb-server +``` -For the server target: +Start the bridge from Windows PowerShell on the VMware host: ```powershell powershell -ExecutionPolicy Bypass -File .\host\bridge-kgdb.ps1 -Target server ``` -This forwards: +The default server bridge is: ```text \\.\pipe\kgdb-server <-> 127.0.0.1:5520 ``` -For the desktop target: +Set the matching endpoint: -```powershell -powershell -ExecutionPolicy Bypass -File .\host\bridge-kgdb.ps1 -Target desktop -``` - -This forwards: - -```text -\\.\pipe\kgdb-desktop <-> 127.0.0.1:5510 -``` +```bash +declare -A TARGET_DEBUG_ENDPOINT=( + [server]=127.0.0.1:5520 +) -If your VMware pipe or TCP port is different, override the defaults: +declare -A TARGET_KGDB_TTY=( + [server]=ttyS0 +) -```powershell -powershell -ExecutionPolicy Bypass -File .\host\bridge-kgdb.ps1 ` - -Target server ` - -PipeName "\\.\pipe\my-kgdb-pipe" ` - -Port 5520 +declare -A TARGET_KGDB_BAUD=( + [server]=115200 +) ``` -Leave the PowerShell window open while debugging. It should print that it is -waiting for a GDB TCP connection. When VS Code starts debugging, it should print -that GDB connected and that the named pipe connected. +The TTY and baud rate must match the boot arguments added by provisioning. ## Verify The Debug Endpoint -From Windows PowerShell: +From the development host, check the endpoint before launching GDB: -```powershell -Test-NetConnection 127.0.0.1 -Port 5520 +```bash +nc -vz 127.0.0.1 8864 ``` -From WSL, if `nc` is installed: +For the serial bridge, you can also check from Windows PowerShell: -```bash -nc -vz 127.0.0.1 5520 +```powershell +Test-NetConnection 127.0.0.1 -Port 5520 ``` -If Windows can connect but WSL cannot, use the Windows host address visible from -WSL instead of `127.0.0.1`: +If the VMware host can connect but the development host cannot, use an address +for the VMware host that the development host can reach instead of `127.0.0.1`. +On WSL, the Windows host address is often listed as the resolver: ```bash grep nameserver /etc/resolv.conf ``` -Then update `lab.local.env`: +After changing `lab.local.env`, run F5 again so script 04 regenerates the GDB +files under `.gdb/`. -```env -SERVER_KGDB_ENDPOINT=:5520 -``` - -Run F5 again after changing `lab.local.env` so -`scripts/04-deploy-debug-vscode.sh` regenerates `.gdb/server-kgdb.gdb` with the -new endpoint. +## Test The Module -## Start VS Code Debugging +After loading the module, run this on the target VM: -In VS Code Remote-WSL: - -1. Press `Ctrl+Shift+B` and choose the target when prompted. This runs - `Kernel: Build`. -2. Choose `Kernel: Debug server` or `Kernel: Debug desktop`. -3. Put a breakpoint in `module/src/hello.c` or let the module stop at - `kgdb_breakpoint()`. -4. Press F5. +```bash +head -c 10 /dev/chuck_norise +``` -The build task runs: +Expected output: -```bash -./scripts/03-build-module.sh server +```text +chuck nori ``` -The matching debug prelaunch task then runs: +Use one open file descriptor to see offset-preserving reads: ```bash -./scripts/04-deploy-debug-vscode.sh server +exec 9/dev/null +dd bs=1 count=5 <&9 2>/dev/null +dd bs=1 count=7 <&9 2>/dev/null +exec 9<&- ``` -Script 04 uploads `hello.ko`, removes any old `hello` module, inserts the new -module with a short `debug_delay_ms`, discovers the module section addresses -from `/sys/module/hello/sections/*`, writes `.gdb/server-module-symbols.gdb`, -and generates `.gdb/server-kgdb.gdb`. VS Code then starts GDB and sources that -generated script. - -The expected order is: - -1. VM is running with either the VMware debug stub enabled or the serial bridge - connected. -2. TCP endpoint is reachable from WSL. -3. `Ctrl+Shift+B` builds `hello.ko`. -4. F5 runs the matching deploy prelaunch task. -5. Script 04 loads `hello.ko` and prepares the generated GDB files. -6. VS Code starts GDB and sources the generated script. -7. GDB connects to the configured endpoint while the module is still inside - `debug_delay_ms`. -8. The target should stop at `kgdb_breakpoint()` in `module/src/hello.c`. +Expected output chunks are `chu`, `ck no`, and `rise!ch`. ## Troubleshooting -If F5 fails with `target remote ... Connection timed out`, the endpoint in -`.gdb/server-kgdb.gdb` is not reachable from WSL. Check that -`SERVER_KGDB_ENDPOINT` or `DESKTOP_KGDB_ENDPOINT` uses the reachable host and -port. For the recommended VMware debug stub path, confirm the `.vmx` debugStub -settings and port. For the serial bridge path, confirm `host/bridge-kgdb.ps1` -is still running. - -If the bridge says the TCP client connected but named-pipe connection fails, -check the VMware serial-port pipe name and make sure the VM is powered on with -the serial port connected at power on. - -If GDB connects but the module does not load, inspect: +If a script says `sshpass` is missing and you use SSH passwords, install it on +the development host: ```bash -cat .gdb/server-loader.log +sudo apt-get install -y sshpass ``` -If breakpoints bind to files under `build/intermediate/...`, the generated GDB -script should map them back to `module/` using `set substitute-path`. Regenerate -the script by pressing F5 again after any script changes. +Leave `TARGET_SUDO_PASS[target]` empty when the sudo password is the same as +the SSH password. -The module source is self-contained under `module/`: C files in `module/src/`, -headers in `module/include/`, and the module Kbuild fragment in -`module/Kbuild`. The top-level scripts and Makefile are lab/build tooling. +If F5 fails with `target remote ... Connection timed out`, the endpoint in +`.gdb/current-kgdb.gdb` is not reachable from the development host. Check +`TARGET_DEBUG_ENDPOINT[target]` and verify that your endpoint provider is +listening. -For full kernel symbols, run provisioning with `--debug-symbols`: +If GDB connects but the module does not load, inspect: ```bash -./scripts/01-provision-target.sh desktop --debug-symbols +cat .gdb/server-loader.log ``` -Without that flag, provisioning only installs target headers, `rsync`, and KGDB -boot arguments. Module builds still work. +Use the target-specific log name for the selected target, for example +`.gdb/desktop-loader.log`. -If you use SSH passwords instead of keys, set `DESKTOP_SSH_PASS` or -`SERVER_SSH_PASS` in `lab.local.env` and install `sshpass` in WSL: - -```bash -sudo apt-get install -y sshpass -``` - -Leave `_SUDO_PASS` empty when the sudo password is the same as the SSH -password. +If breakpoints bind to files under `build/intermediate/...`, regenerate the GDB +files by pressing F5 again. The generated script should map those staged paths +back to `module/` with `set substitute-path`. ## VS Code Include Errors @@ -365,50 +323,19 @@ The C/C++ extension reads `.vscode/c_cpp_properties.json`. It expects target kernel headers under `.kernel-cache//build`, which are created by: ```bash -./scripts/02-setup-wsl-build.sh desktop -./scripts/02-setup-wsl-build.sh server -``` - -Before that sync runs, VS Code can still show include squiggles for kernel -headers such as `linux/module.h` or `linux/fs.h`. If the squiggles remain after -syncing headers, run `C/C++: Reset IntelliSense Database` from the command -palette and select the matching configuration: `Linux kernel module - desktop` -or `Linux kernel module - server`. - -The IntelliSense configuration deliberately mirrors Kbuild's compile context: -GNU C mode, `__KERNEL__`, `MODULE`, `CC_USING_FENTRY`, Ubuntu's kernel include -directory, and the forced kernel headers `compiler-version.h`, `kconfig.h`, and -`compiler_types.h`. Without those forced headers, VS Code parses kernel headers -as ordinary C and reports false errors for `CONFIG_*`, `IS_ENABLED()`, ftrace, -and other kernel-only macros even when `make` builds successfully. - -## Reading The Device - -For a quick smoke test after loading the module: - -```bash -head -c 10 /dev/chuck_norise +./scripts/02-setup-host-build.sh desktop +./scripts/02-setup-host-build.sh server ``` -Expected output: +Before that sync runs, VS Code can show include squiggles for kernel headers +such as `linux/module.h` or `linux/fs.h`. If the squiggles remain after syncing +headers, run `C/C++: Reset IntelliSense Database` from the command palette and +select the matching configuration. -```text -chuck nori -``` - -Use one open file descriptor to see offset-preserving reads: - -```bash -exec 9/dev/null -dd bs=1 count=5 <&9 2>/dev/null -dd bs=1 count=7 <&9 2>/dev/null -exec 9<&- -``` - -Expected output chunks are `chu`, `ck no`, and `rise!ch`. +## Repository Layout -## Snapshot Restore +The module source is under `module/`: C files in `module/src/`, headers in +`module/include/`, and the Kbuild fragment in `module/Kbuild`. -VMware snapshot restore is host-only. Run `host/restore-snapshot.ps1` manually -from Windows PowerShell, not from WSL or VS Code Remote-WSL. +The top-level scripts and Makefile are development tooling. The optional +PowerShell scripts under `host/` are helpers for VMware Workstation on Windows. diff --git a/lab.example.env b/lab.example.env index 1e96ffc..a571c10 100644 --- a/lab.example.env +++ b/lab.example.env @@ -1,40 +1,75 @@ -# Copy this file to lab.local.env and edit the values for your machines. -# Supported targets are exactly: desktop, server. +# Copy this file to lab.local.env and edit it for your machines. -# Common local tools. +# Common local tools on the Debian-based development host. GDB_BIN=gdb SSH_BIN=ssh SCP_BIN=scp RSYNC_BIN=rsync SSHPASS_BIN=sshpass -# GDB connects to these endpoints from WSL. If VMware exposes KGDB through a -# Windows named pipe, bridge that pipe to TCP on the host and put the bridge -# endpoint here. -DESKTOP_KGDB_ENDPOINT=127.0.0.1:5510 -SERVER_KGDB_ENDPOINT=127.0.0.1:5520 - -# SSH details for the Ubuntu 24.04 Desktop VM. -DESKTOP_SSH_HOST=ubuntu-desktop.local -DESKTOP_SSH_PORT=22 -DESKTOP_SSH_USER=user -DESKTOP_SSH_PASS= -# Leave empty to reuse DESKTOP_SSH_PASS for sudo. -DESKTOP_SUDO_PASS= -DESKTOP_REMOTE_DIR=/tmp/small-ko -DESKTOP_KGDB_TTY=ttyS0 -DESKTOP_KGDB_BAUD=115200 - -# SSH details for the Ubuntu 24.04 Server VM. -SERVER_SSH_HOST=ubuntu-server.local -SERVER_SSH_PORT=22 -SERVER_SSH_USER=user -SERVER_SSH_PASS= -# Leave empty to reuse SERVER_SSH_PASS for sudo. -SERVER_SUDO_PASS= -SERVER_REMOTE_DIR=/tmp/small-ko -SERVER_KGDB_TTY=ttyS0 -SERVER_KGDB_BAUD=115200 +# Target profiles. The VS Code picker in .vscode/tasks.json should list the +# same names when you add or remove profiles. +TARGETS=(desktop server) + +# Only ubuntu targets are implemented for now. Add a new target OS backend in +# the scripts before using another value here. +declare -A TARGET_OS=( + [desktop]=ubuntu + [server]=ubuntu +) + +# SSH connection to each target VM. +declare -A TARGET_SSH_HOST=( + [desktop]=ubuntu-desktop.local + [server]=ubuntu-server.local +) + +declare -A TARGET_SSH_PORT=( + [desktop]=22 + [server]=22 +) + +declare -A TARGET_SSH_USER=( + [desktop]=user + [server]=user +) + +# Leave empty when using SSH keys. +declare -A TARGET_SSH_PASS=( + [desktop]= + [server]= +) + +# Leave empty to reuse TARGET_SSH_PASS for sudo. +declare -A TARGET_SUDO_PASS=( + [desktop]= + [server]= +) + +# Remote staging directory used for uploading hello.ko. +declare -A TARGET_REMOTE_DIR=( + [desktop]=/tmp/small-ko + [server]=/tmp/small-ko +) + +# GDB connects to these endpoints from the development host. The endpoint can +# come from VMware's debug stub, a serial-port bridge, or any other TCP bridge +# that speaks GDB remote protocol. +declare -A TARGET_DEBUG_ENDPOINT=( + [desktop]=127.0.0.1:5510 + [server]=127.0.0.1:5520 +) + +# Serial KGDB settings used only by the serial-port debug path. +declare -A TARGET_KGDB_TTY=( + [desktop]=ttyS0 + [server]=ttyS0 +) + +declare -A TARGET_KGDB_BAUD=( + [desktop]=115200 + [server]=115200 +) # The module waits briefly before kgdb_breakpoint so the deploy script can read # /sys/module/hello/sections/* and prepare module symbols for VS Code. diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index ff34a16..75f6bd2 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -1,47 +1,51 @@ #!/usr/bin/env bash set -euo pipefail +usage() { + echo "usage: $0 [--debug-symbols]" >&2 + exit 2 +} + target="${1:-}" symbols="${2:-}" -case "$target" in desktop|server) ;; *) echo "usage: $0 [--debug-symbols]" >&2; exit 2 ;; esac -case "$symbols" in ""|--debug-symbols) ;; *) echo "usage: $0 [--debug-symbols]" >&2; exit 2 ;; esac +case "$symbols" in ""|--debug-symbols) ;; *) usage ;; esac repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 [--debug-symbols]" env_file="$repo_root/lab.local.env" -[[ -f "$env_file" ]] || { echo "missing $env_file; copy lab.example.env first" >&2; exit 1; } - -set -a +[[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" # shellcheck source=/dev/null source "$env_file" -set +a - -prefix="${target^^}" -cfg() { local name="${prefix}_$1"; printf '%s' "${!name:-}"; } +target_is_configured "$target" ssh_bin="${SSH_BIN:-ssh}" sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(cfg SSH_HOST)" -ssh_port="$(cfg SSH_PORT)" -ssh_user="$(cfg SSH_USER)" -ssh_pass="$(cfg SSH_PASS)" -sudo_pass="$(cfg SUDO_PASS)" -kgdb_tty="$(cfg KGDB_TTY)" -kgdb_baud="$(cfg KGDB_BAUD)" - -[[ -n "$ssh_host" && -n "$ssh_user" ]] || { - echo "missing ${prefix}_SSH_HOST or ${prefix}_SSH_USER in lab.local.env" >&2 - exit 1 -} +ssh_host="$(target_require_cfg "$target" SSH_HOST)" +ssh_port="$(target_cfg "$target" SSH_PORT)" +ssh_user="$(target_require_cfg "$target" SSH_USER)" +ssh_pass="$(target_cfg "$target" SSH_PASS)" +sudo_pass="$(target_cfg "$target" SUDO_PASS)" +target_os="$(target_cfg "$target" OS)" +kgdb_tty="$(target_cfg "$target" KGDB_TTY)" +kgdb_baud="$(target_cfg "$target" KGDB_BAUD)" ssh_port="${ssh_port:-22}" +target_os="${target_os:-ubuntu}" kgdb_tty="${kgdb_tty:-ttyS0}" kgdb_baud="${kgdb_baud:-115200}" sudo_pass="${sudo_pass:-$ssh_pass}" sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" +case "$target_os" in + ubuntu) ;; + *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; +esac + if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it in WSL: sudo apt-get install -y sshpass" >&2 + echo "install it on the build host: sudo apt-get install -y sshpass" >&2 exit 1 fi @@ -53,7 +57,7 @@ fi echo "provisioning $target at $ssh_user@$ssh_host:$ssh_port" SSHPASS="$ssh_pass" "${ssh_cmd[@]}" -t "$ssh_user@$ssh_host" \ - "KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$sudo_pass_b64' bash -s" <<'REMOTE' + "TARGET_OS='$target_os' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$sudo_pass_b64' bash -s" <<'REMOTE' set -euo pipefail sudo_run() { @@ -75,6 +79,7 @@ apt_retry() { . /etc/os-release kernel="$(uname -r)" codename="${VERSION_CODENAME:-noble}" +[[ "${TARGET_OS:-ubuntu}" == "ubuntu" ]] || { echo "unsupported target OS: ${TARGET_OS:-}" >&2; exit 1; } [[ "${ID:-}" == "ubuntu" ]] || { echo "target is not Ubuntu" >&2; exit 1; } sudo_auth @@ -102,7 +107,7 @@ EOF exit 1 } fi -[[ -r "$vmlinux" ]] || echo "note: $vmlinux is missing; F5 KGDB source debugging may need it" +[[ -r "$vmlinux" ]] || echo "note: $vmlinux is missing; VS Code source debugging may need it" grub_file=/etc/default/grub current="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" diff --git a/scripts/02-setup-wsl-build.sh b/scripts/02-setup-host-build.sh similarity index 82% rename from scripts/02-setup-wsl-build.sh rename to scripts/02-setup-host-build.sh index e425107..1d92dd7 100755 --- a/scripts/02-setup-wsl-build.sh +++ b/scripts/02-setup-host-build.sh @@ -1,66 +1,49 @@ #!/usr/bin/env bash set -euo pipefail -usage() { - echo "usage: $0 " >&2 - exit 2 -} - target="${1:-}" -case "$target" in - desktop|server) ;; - *) usage ;; -esac repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 " env_file="$repo_root/lab.local.env" - -if [[ ! -f "$env_file" ]]; then - echo "missing $env_file; copy lab.example.env to lab.local.env and edit it" >&2 - exit 1 -fi - -set -a +[[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" # shellcheck source=/dev/null source "$env_file" -set +a - -prefix="${target^^}" -cfg() { - local name="${prefix}_$1" - printf '%s' "${!name:-}" -} +require_debian_host +target_is_configured "$target" ssh_bin="${SSH_BIN:-ssh}" rsync_bin="${RSYNC_BIN:-rsync}" sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(cfg SSH_HOST)" -ssh_port="$(cfg SSH_PORT)" -ssh_user="$(cfg SSH_USER)" -ssh_pass="$(cfg SSH_PASS)" -sudo_pass="$(cfg SUDO_PASS)" +ssh_host="$(target_require_cfg "$target" SSH_HOST)" +ssh_port="$(target_cfg "$target" SSH_PORT)" +ssh_user="$(target_require_cfg "$target" SSH_USER)" +ssh_pass="$(target_cfg "$target" SSH_PASS)" +sudo_pass="$(target_cfg "$target" SUDO_PASS)" +target_os="$(target_cfg "$target" OS)" ssh_port="${ssh_port:-22}" sudo_pass="${sudo_pass:-$ssh_pass}" sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" +target_os="${target_os:-ubuntu}" -[[ -n "$ssh_host" && -n "$ssh_user" ]] || { - echo "missing ${prefix}_SSH_HOST or ${prefix}_SSH_USER in lab.local.env" >&2 - exit 1 -} +case "$target_os" in + ubuntu) ;; + *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; +esac if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it in WSL: sudo apt-get install -y sshpass" >&2 + echo "install it on the build host: sudo apt-get install -y sshpass" >&2 exit 1 fi ssh_target="$ssh_user@$ssh_host" ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -ssh_transport=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port") rsync_rsh="$ssh_bin -o StrictHostKeyChecking=accept-new -p $ssh_port" if [[ -n "$ssh_pass" ]]; then ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") - ssh_transport=("$sshpass_bin" -e "${ssh_transport[@]}") rsync_rsh="$sshpass_bin -e $ssh_bin -o StrictHostKeyChecking=accept-new -p $ssh_port" fi @@ -88,7 +71,7 @@ apt_install() { install -y "$@" } -echo "installing/validating WSL kernel-module build packages" +echo "installing/validating host kernel-module build packages" apt_update apt_install \ bc \ @@ -178,7 +161,7 @@ if SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}test -r '$remote_vmlinux'" else rm -f "$cache_dir/vmlinux" echo "warning: missing readable $remote_vmlinux on target" >&2 - echo "module builds can still work, but VS Code KGDB source debugging needs it" >&2 + echo "module builds can still work, but VS Code source debugging needs it" >&2 echo "run scripts/01-provision-target.sh $target --debug-symbols to install it" >&2 fi diff --git a/scripts/03-build-module.sh b/scripts/03-build-module.sh index 2a0fb81..bbe410e 100755 --- a/scripts/03-build-module.sh +++ b/scripts/03-build-module.sh @@ -1,25 +1,19 @@ #!/usr/bin/env bash set -euo pipefail -usage() { - echo "usage: $0 " >&2 - exit 2 -} - target="${1:-}" -case "$target" in - desktop|server) ;; - *) usage ;; -esac - repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 " + cache_dir="$repo_root/.kernel-cache/$target" kernel_file="$cache_dir/kernel.release" kdir="$cache_dir/build" if [[ ! -f "$kernel_file" || ! -d "$kdir" ]]; then echo "missing kernel cache for $target" >&2 - echo "run scripts/02-setup-wsl-build.sh $target first" >&2 + echo "run scripts/02-setup-host-build.sh $target first" >&2 exit 1 fi diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index ced8c6d..4e26b35 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -1,46 +1,28 @@ #!/usr/bin/env bash set -euo pipefail -usage() { - echo "usage: $0 " >&2 - exit 2 -} - target="${1:-}" -case "$target" in - desktop|server) ;; - *) usage ;; -esac repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 " env_file="$repo_root/lab.local.env" - -if [[ ! -f "$env_file" ]]; then - echo "missing $env_file; copy lab.example.env to lab.local.env and edit it" >&2 - exit 1 -fi - -set -a +[[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" # shellcheck source=/dev/null source "$env_file" -set +a - -prefix="${target^^}" -cfg() { - local name="${prefix}_$1" - printf '%s' "${!name:-}" -} +target_is_configured "$target" ssh_bin="${SSH_BIN:-ssh}" scp_bin="${SCP_BIN:-scp}" sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(cfg SSH_HOST)" -ssh_port="$(cfg SSH_PORT)" -ssh_user="$(cfg SSH_USER)" -ssh_pass="$(cfg SSH_PASS)" -sudo_pass="$(cfg SUDO_PASS)" -remote_dir="$(cfg REMOTE_DIR)" -kgdb_endpoint="$(cfg KGDB_ENDPOINT)" +ssh_host="$(target_require_cfg "$target" SSH_HOST)" +ssh_port="$(target_cfg "$target" SSH_PORT)" +ssh_user="$(target_require_cfg "$target" SSH_USER)" +ssh_pass="$(target_cfg "$target" SSH_PASS)" +sudo_pass="$(target_cfg "$target" SUDO_PASS)" +remote_dir="$(target_cfg "$target" REMOTE_DIR)" +debug_endpoint="$(target_require_cfg "$target" DEBUG_ENDPOINT)" debug_delay_ms="${DEBUG_LOAD_DELAY_MS:-5000}" ssh_port="${ssh_port:-22}" @@ -48,14 +30,9 @@ remote_dir="${remote_dir:-/tmp/small-ko}" sudo_pass="${sudo_pass:-$ssh_pass}" sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" -[[ -n "$ssh_host" && -n "$ssh_user" && -n "$kgdb_endpoint" ]] || { - echo "missing ${prefix}_SSH_HOST, ${prefix}_SSH_USER, or ${prefix}_KGDB_ENDPOINT" >&2 - exit 1 -} - if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it in WSL: sudo apt-get install -y sshpass" >&2 + echo "install it on the build host: sudo apt-get install -y sshpass" >&2 exit 1 fi @@ -81,15 +58,15 @@ vmlinux="$cache_dir/vmlinux" if [[ ! -f "$kernel_file" ]]; then echo "missing kernel cache for $target" >&2 - echo "run scripts/02-setup-wsl-build.sh $target first" >&2 + echo "run scripts/02-setup-host-build.sh $target first" >&2 exit 1 fi if [[ ! -f "$vmlinux" ]]; then echo "missing $vmlinux" >&2 - echo "module builds may work, but VS Code KGDB debugging needs the matching vmlinux" >&2 + echo "module builds may work, but VS Code source debugging needs the matching vmlinux" >&2 echo "install the target's linux-image-*-dbgsym package, then rerun:" >&2 - echo " scripts/02-setup-wsl-build.sh $target" >&2 + echo " scripts/02-setup-host-build.sh $target" >&2 exit 1 fi @@ -106,13 +83,12 @@ fi gdb_dir="$repo_root/.gdb" mkdir -p "$gdb_dir" -ready_file="$gdb_dir/$target.ready" symbols_file="$gdb_dir/$target-module-symbols.gdb" gdb_file="$gdb_dir/$target-kgdb.gdb" loader_log="$gdb_dir/$target-loader.log" loader_script="$gdb_dir/$target-loader.sh" -rm -f "$ready_file" "$symbols_file" "$loader_log" "$loader_script" +rm -f "$symbols_file" "$loader_log" "$loader_script" remote_module="$remote_dir/hello.ko" remote_sudo="$(remote_sudo_prefix)" @@ -138,15 +114,17 @@ set substitute-path $intermediate_dir $repo_root/module directory $repo_root/module/src directory $repo_root/module/include symbol-file $vmlinux -target remote $kgdb_endpoint +target remote $debug_endpoint source $symbols_file EOF +ln -sfn "$target-kgdb.gdb" "$gdb_dir/current-kgdb.gdb" +ln -sfn "../.kernel-cache/$target/vmlinux" "$gdb_dir/current-vmlinux" + { printf '#!/usr/bin/env bash\n' printf 'set -euo pipefail\n' printf 'target=%q\n' "$target" - printf 'ready_file=%q\n' "$ready_file" printf 'symbols_file=%q\n' "$symbols_file" printf 'artifact=%q\n' "$artifact" printf 'ssh_bin=%q\n' "$ssh_bin" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh new file mode 100644 index 0000000..0de37be --- /dev/null +++ b/scripts/lib/common.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +die() { + echo "$*" >&2 + exit 1 +} + +validate_target() { + local target="$1" + local usage="$2" + + [[ -n "$target" ]] || die "$usage" + [[ "$target" =~ ^[A-Za-z][A-Za-z0-9_-]*$ ]] || + die "invalid target '$target'; use letters, numbers, '_' or '-', starting with a letter" +} + +target_is_configured() { + local target="$1" + local known + + declare -p TARGETS >/dev/null 2>&1 || + die "missing TARGETS array in lab.local.env" + + for known in "${TARGETS[@]}"; do + [[ "$known" == "$target" ]] && return 0 + done + + die "unknown target '$target'; add it to TARGETS in lab.local.env" +} + +target_cfg() { + local target="$1" + local key="$2" + local map_name="TARGET_${key}" + + declare -p "$map_name" >/dev/null 2>&1 || return 0 + local -n map="$map_name" + printf '%s' "${map[$target]:-}" +} + +target_require_cfg() { + local target="$1" + local key="$2" + local value + + value="$(target_cfg "$target" "$key")" + [[ -n "$value" ]] || die "missing TARGET_${key}[$target] in lab.local.env" + printf '%s' "$value" +} + +require_debian_host() { + command -v apt-get >/dev/null 2>&1 || + die "missing apt-get; the development host must be Debian-based" +} From 1eeb3e47ccee2bd06248931f7abc370dc25643a5 Mon Sep 17 00:00:00 2001 From: dor Date: Thu, 21 May 2026 00:20:24 +0300 Subject: [PATCH 02/13] remove unused properties.json configurations --- .vscode/c_cpp_properties.json | 84 ----------------------------------- 1 file changed, 84 deletions(-) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 1f4042d..2712355 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -42,90 +42,6 @@ "${workspaceFolder}/.kernel-cache/current/build/include/linux/kconfig.h", "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler_types.h" ] - }, - { - "name": "Linux kernel module - desktop", - "compilerPath": "/usr/bin/gcc-13", - "intelliSenseMode": "linux-gcc-x64", - "cStandard": "gnu11", - "defines": [ - "__KERNEL__", - "MODULE", - "CC_USING_FENTRY", - "KBUILD_MODNAME=\"hello\"", - "KBUILD_BASENAME=\"hello\"", - "__KBUILD_MODNAME=kmod_hello" - ], - "includePath": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", - "${workspaceFolder}/.kernel-cache/desktop/build/include", - "${workspaceFolder}/.kernel-cache/desktop/build/include/uapi", - "${workspaceFolder}/.kernel-cache/desktop/build/include/generated", - "${workspaceFolder}/.kernel-cache/desktop/build/include/generated/uapi", - "${workspaceFolder}/.kernel-cache/desktop/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include/uapi", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include/generated", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include/generated/uapi" - ], - "browse": { - "path": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", - "${workspaceFolder}/.kernel-cache/desktop/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/desktop/build/include", - "${workspaceFolder}/.kernel-cache/desktop/build/arch/x86/include" - ], - "limitSymbolsToIncludedHeaders": true - }, - "forcedInclude": [ - "${workspaceFolder}/.kernel-cache/desktop/build/include/linux/compiler-version.h", - "${workspaceFolder}/.kernel-cache/desktop/build/include/linux/kconfig.h", - "${workspaceFolder}/.kernel-cache/desktop/build/include/linux/compiler_types.h" - ] - }, - { - "name": "Linux kernel module - server", - "compilerPath": "/usr/bin/gcc-13", - "intelliSenseMode": "linux-gcc-x64", - "cStandard": "gnu11", - "defines": [ - "__KERNEL__", - "MODULE", - "CC_USING_FENTRY", - "KBUILD_MODNAME=\"hello\"", - "KBUILD_BASENAME=\"hello\"", - "__KBUILD_MODNAME=kmod_hello" - ], - "includePath": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", - "${workspaceFolder}/.kernel-cache/server/build/include", - "${workspaceFolder}/.kernel-cache/server/build/include/uapi", - "${workspaceFolder}/.kernel-cache/server/build/include/generated", - "${workspaceFolder}/.kernel-cache/server/build/include/generated/uapi", - "${workspaceFolder}/.kernel-cache/server/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include/uapi", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include/generated", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include/generated/uapi" - ], - "browse": { - "path": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", - "${workspaceFolder}/.kernel-cache/server/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/server/build/include", - "${workspaceFolder}/.kernel-cache/server/build/arch/x86/include" - ], - "limitSymbolsToIncludedHeaders": true - }, - "forcedInclude": [ - "${workspaceFolder}/.kernel-cache/server/build/include/linux/compiler-version.h", - "${workspaceFolder}/.kernel-cache/server/build/include/linux/kconfig.h", - "${workspaceFolder}/.kernel-cache/server/build/include/linux/compiler_types.h" - ] } ] } From a176e4cb37a2c4d46f42419b4f5103281a9652ec Mon Sep 17 00:00:00 2001 From: dor Date: Thu, 21 May 2026 22:13:11 +0300 Subject: [PATCH 03/13] changed gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9fd6cc1..2f85823 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ modules.order .vscode/ipch/ .codex - +kernel.gdb +CLAUDE.md From fcdf5bcefa6c22150273f31748ae6382c4b9bf8f Mon Sep 17 00:00:00 2001 From: dor Date: Fri, 22 May 2026 19:51:58 +0300 Subject: [PATCH 04/13] kgdb debugging - good gdb tui speed, super slow vscode source debug --- .gitignore | 1 + .vscode/launch.json | 16 ++++++++++ .vscode/tasks.json | 13 ++++++++ Makefile | 2 +- module/Kbuild | 8 ++++- module/src/hello.c | 2 +- scripts/01-provision-target.sh | 11 +++++-- scripts/04-deploy-debug-vscode.sh | 52 +++++++++++++++++++++++++++++++ 8 files changed, 100 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 2f85823..962f2a6 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ modules.order .codex kernel.gdb CLAUDE.md +.claude/ diff --git a/.vscode/launch.json b/.vscode/launch.json index cc0391d..e87c0bf 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -3,6 +3,22 @@ "configurations": [ { "name": "Kernel: Debug", + "type": "gdb", + "request": "attach", + "executable": "${workspaceFolder}/.gdb/current-vmlinux", + "target": "172.27.240.1:5520", + "remote": true, + "cwd": "${workspaceFolder}", + "preLaunchTask": "Kernel: Deploy Debug", + "valuesFormatting": "parseText", + "autorun": [ + "source ${workspaceFolder}/.gdb/current-kgdb-attached.gdb", + "hbreak chuck_device_register", + "continue" + ] + }, + { + "name": "Kernel: Debug (cppdbg, slow on thread enum)", "type": "cppdbg", "request": "launch", "program": "${workspaceFolder}/.gdb/current-vmlinux", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 2c4d58e..9f58223 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -40,6 +40,19 @@ "type": "shell", "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget}", "problemMatcher": [] + }, + { + "label": "Kernel: GDB", + "type": "shell", + "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget} && gdb -tui ${workspaceFolder}/.gdb/current-vmlinux -x ${workspaceFolder}/.gdb/current-kgdb.gdb", + "presentation": { + "echo": false, + "reveal": "always", + "focus": true, + "panel": "dedicated", + "clear": true + }, + "problemMatcher": [] } ] } diff --git a/Makefile b/Makefile index ac62736..207d072 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ MODULE_DIR ?= $(CURDIR)/module all: modules modules: prepare-build - $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" modules + $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" MODULE_REAL_DIR="$(MODULE_DIR)" modules mkdir -p "$(ARTIFACT_DIR)" mv "$(INTERMEDIATE_DIR)/$(MODULE_NAME).ko" "$(ARTIFACT_DIR)/$(MODULE_NAME).ko" @echo "built $(ARTIFACT_DIR)/$(MODULE_NAME).ko" diff --git a/module/Kbuild b/module/Kbuild index a49ea0a..7ba375e 100644 --- a/module/Kbuild +++ b/module/Kbuild @@ -1,6 +1,12 @@ MODULE_NAME ?= hello +# Real on-disk module source directory. The parent Makefile passes this so the +# compiler can rewrite paths in DWARF debug info (and __FILE__) from the staged +# intermediate dir back to module/. Without that rewrite, IDEs that set +# breakpoints by absolute path (VS Code, CLion, ...) can't match the symtab. +MODULE_REAL_DIR ?= $(src) + obj-m += $(MODULE_NAME).o $(MODULE_NAME)-y := src/hello.o src/chuck_device.o src/chuck_message.o -ccflags-y := -I$(src)/include -g -DDEBUG +ccflags-y := -I$(src)/include -g -DDEBUG -ffile-prefix-map=$(src)=$(MODULE_REAL_DIR) diff --git a/module/src/hello.c b/module/src/hello.c index 91942fb..af12ec7 100644 --- a/module/src/hello.c +++ b/module/src/hello.c @@ -9,7 +9,7 @@ static unsigned int debug_delay_ms = 5000; module_param(debug_delay_ms, uint, 0644); MODULE_PARM_DESC(debug_delay_ms, - "Milliseconds to wait before kgdb_breakpoint so symbols can load"); + "Milliseconds to sleep at the start of init so the host can attach GDB and set breakpoints"); static int hello_init(void) { diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index 75f6bd2..d0d2493 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -57,7 +57,7 @@ fi echo "provisioning $target at $ssh_user@$ssh_host:$ssh_port" SSHPASS="$ssh_pass" "${ssh_cmd[@]}" -t "$ssh_user@$ssh_host" \ - "TARGET_OS='$target_os' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$sudo_pass_b64' bash -s" <<'REMOTE' + "TARGET_OS='$target_os' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$sudo_pass_b64' DEBUG_MAXCPUS='${DEBUG_MAXCPUS:-}' bash -s" <<'REMOTE' set -euo pipefail sudo_run() { @@ -111,7 +111,14 @@ fi grub_file=/etc/default/grub current="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" -for arg in "kgdboc=${KGDB_TTY},${KGDB_BAUD}" "nokaslr" "sysrq_always_enabled=1"; do +# Strip any prior values for the keys we manage so reruns with changed values +# replace rather than append. +current="$(printf '%s' "$current" | sed -E 's/(^| )(kgdboc=|maxcpus=)[^ ]*//g; s/ */ /g; s/^ +//; s/ +$//')" +args=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "nokaslr" "sysrq_always_enabled=1") +if [[ -n "${DEBUG_MAXCPUS:-}" ]]; then + args+=("maxcpus=${DEBUG_MAXCPUS}") +fi +for arg in "${args[@]}"; do case " $current " in *" $arg "*) ;; *) current="${current:+$current }$arg" ;; esac done diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index 4e26b35..c2f5632 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -93,6 +93,34 @@ rm -f "$symbols_file" "$loader_log" "$loader_script" remote_module="$remote_dir/hello.ko" remote_sudo="$(remote_sudo_prefix)" +# Kill any local gdb processes that still hold a TCP connection to the bridge. +# bridge-kgdb.ps1 serves one client at a time; leftover gdb processes from +# previous sessions (zombie bare-gdb invocations, half-torn-down VS Code +# launches, etc.) sit in the bridge's accept queue and block fresh attaches. +debug_host="${debug_endpoint%:*}" +debug_port="${debug_endpoint##*:}" +if command -v ss >/dev/null 2>&1; then + stale_pid_list=$({ ss -ntp 2>/dev/null \ + | grep -F "$debug_host:$debug_port" \ + | grep -oE 'pid=[0-9]+' \ + | cut -d= -f2 \ + | sort -u; } || true) + stale_gdb_pids="" + for pid in $stale_pid_list; do + if [[ "$(ps -p "$pid" -o comm= 2>/dev/null || true)" == "gdb" ]]; then + stale_gdb_pids="$stale_gdb_pids $pid" + fi + done + if [[ -n "$stale_gdb_pids" ]]; then + echo "killing leftover gdb clients on $debug_endpoint:" + # shellcheck disable=SC2086 + ps -p $stale_gdb_pids -o pid,cmd 2>/dev/null | tail -n +2 | sed 's/^/ /' || true + # shellcheck disable=SC2086 + kill -9 $stale_gdb_pids 2>/dev/null || true + sleep 0.3 + fi +fi + echo "uploading $artifact to $ssh_target:$remote_module" SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "mkdir -p '$remote_dir'" SSHPASS="$ssh_pass" "${scp_cmd[@]}" "$artifact" "$ssh_target:$remote_module" @@ -118,7 +146,16 @@ target remote $debug_endpoint source $symbols_file EOF +# Trimmed-down variant for IDEs that handle the gdb-side attach themselves +# (Native Debug, cppdbg with miDebuggerServerAddress). The IDE has already +# called `target remote` and loaded the executable, so commands that touch +# global gdb state (mi-async, remote timeouts, architecture override) error +# with "Cannot change this setting while the inferior is running" — drop them. +gdb_attached_file="$gdb_dir/$target-kgdb-attached.gdb" +grep -vE '^(target remote |set mi-async |set target-async |set tcp connect-timeout |set remotetimeout |set architecture |symbol-file )' "$gdb_file" > "$gdb_attached_file" + ln -sfn "$target-kgdb.gdb" "$gdb_dir/current-kgdb.gdb" +ln -sfn "$target-kgdb-attached.gdb" "$gdb_dir/current-kgdb-attached.gdb" ln -sfn "../.kernel-cache/$target/vmlinux" "$gdb_dir/current-vmlinux" { @@ -179,6 +216,21 @@ while ((SECONDS < deadline)); do echo "echo loaded hello.ko module symbols for $target\\n" } > "$symbols_file" echo "wrote module symbols to $symbols_file" + # Block until kgdb_breakpoint() actually halts the kernel. Returning + # earlier would race the VS Code F5 flow: cppdbg launches gdb + # immediately, gdb sends qSupported into the still-running kernel's + # console buffer where kgdb never sees it, and the launch hangs + # forever. We detect halt by SSH going unreachable. + echo "waiting for kgdb halt..." + halt_deadline=$((SECONDS + 60)) + while ((SECONDS < halt_deadline)); do + if ! SSHPASS="$ssh_pass" timeout 2 "${ssh_cmd[@]}" ":" >/dev/null 2>&1; then + echo "kgdb halted; ready for gdb to attach" + exit 0 + fi + sleep 0.3 + done + echo "warning: kgdb did not halt within 60s; gdb may hang on target remote" >&2 exit 0 fi fi From d38394a3ec2c97a49f32c72b0e6f6cdd9939133e Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 00:43:19 +0300 Subject: [PATCH 05/13] support both kdbg and qemu stub --- .gitignore | 1 + .vscode/launch.json | 6 +- .vscode/tasks.json | 16 ++- README.md | 178 ++++++++++++++++-------------- lab.example.env | 27 +++-- module/src/hello.c | 10 +- scripts/01-provision-target.sh | 30 +++-- scripts/04-deploy-debug-vscode.sh | 52 ++++----- 8 files changed, 178 insertions(+), 142 deletions(-) diff --git a/.gitignore b/.gitignore index 962f2a6..354cc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ modules.order .codex kernel.gdb CLAUDE.md +session.md .claude/ diff --git a/.vscode/launch.json b/.vscode/launch.json index e87c0bf..a5a72e5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,13 +6,13 @@ "type": "gdb", "request": "attach", "executable": "${workspaceFolder}/.gdb/current-vmlinux", - "target": "172.27.240.1:5520", + "target": "127.0.0.1:1234", "remote": true, "cwd": "${workspaceFolder}", "preLaunchTask": "Kernel: Deploy Debug", "valuesFormatting": "parseText", "autorun": [ - "source ${workspaceFolder}/.gdb/current-kgdb-attached.gdb", + "source ${workspaceFolder}/.gdb/current-debug-attached.gdb", "hbreak chuck_device_register", "continue" ] @@ -30,7 +30,7 @@ "setupCommands": [ { "description": "Attach GDB to the selected target", - "text": "source ${workspaceFolder}/.gdb/current-kgdb.gdb", + "text": "source ${workspaceFolder}/.gdb/current-debug.gdb", "ignoreFailures": false } ], diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 9f58223..f29b691 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -10,13 +10,23 @@ "server" ], "default": "server" + }, + { + "id": "debugMethod", + "type": "pickString", + "description": "Debug method", + "options": [ + "kgdb", + "qemu" + ], + "default": "qemu" } ], "tasks": [ { "label": "Kernel: Provision Target", "type": "shell", - "command": "./scripts/01-provision-target.sh ${input:kernelTarget}", + "command": "./scripts/01-provision-target.sh ${input:kernelTarget} ${input:debugMethod}", "problemMatcher": [] }, { @@ -38,13 +48,13 @@ { "label": "Kernel: Deploy Debug", "type": "shell", - "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget}", + "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget} ${input:debugMethod}", "problemMatcher": [] }, { "label": "Kernel: GDB", "type": "shell", - "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget} && gdb -tui ${workspaceFolder}/.gdb/current-vmlinux -x ${workspaceFolder}/.gdb/current-kgdb.gdb", + "command": "./scripts/04-deploy-debug-vscode.sh ${input:kernelTarget} ${input:debugMethod} && gdb -tui ${workspaceFolder}/.gdb/current-vmlinux -x ${workspaceFolder}/.gdb/current-debug.gdb", "presentation": { "echo": false, "reveal": "always", diff --git a/README.md b/README.md index ee1de01..8a35870 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,26 @@ The lab uses two kinds of machines: to install headers, find kernel build trees, and configure debugging for that distro. -For debugging, GDB on the development host connects to an IP address and port. -That endpoint can be opened however you like. This repo documents two provider -options: VMware's built-in debug stub and a VMware serial port bridged to TCP. +## Debug Methods + +This repo supports two debug methods. Provisioning (script 01) and the deploy +step (script 04) both take a `` argument so you can switch between +them per run. + +- **`kgdb`** — in-kernel KGDB talking over the guest's serial port. The + provisioning step adds `kgdboc=ttyS*,115200` and `sysrq_always_enabled=1` to + the guest's kernel command line. To halt the running kernel into GDB, run + `echo g | sudo tee /proc/sysrq-trigger` on the target (or Ctrl+C inside GDB + if your endpoint supports it). Useful when you cannot easily change the + hypervisor's command line, e.g. VMware running as your only option. + +- **`qemu`** — QEMU's built-in gdbstub (`-gdb tcp::PORT` or `-s`). No KGDB in + the guest is required; the hypervisor halts the vCPU directly. VMware's + `debugStub.listen.guest64` is the same shape and slots into this method too. + Connecting GDB to the stub halts the vCPU; set breakpoints, then `continue`. + +The module does **not** call `kgdb_breakpoint()` on insmod. In both methods you +break manually after the deploy step finishes. ## Configure The Lab @@ -56,32 +73,41 @@ declare -A TARGET_SSH_HOST=( Target names may contain letters, numbers, `_`, or `-`. A target named `ubuntu-server` is configured with map keys like `[ubuntu-server]=...`. -The debug endpoint is the host and port that GDB should connect to from the -development host: +Endpoints are split per debug method. Fill the one(s) you intend to use: ```bash -declare -A TARGET_DEBUG_ENDPOINT=( - [server]=127.0.0.1:8864 +declare -A TARGET_DEBUG_ENDPOINT_KGDB=( + [server]=127.0.0.1:5520 +) + +declare -A TARGET_DEBUG_ENDPOINT_QEMU=( + [server]=127.0.0.1:1234 ) ``` +Either map may be left empty per target. Script 04 errors clearly if the +endpoint for the method you picked is missing. + For now, target provisioning and header sync implement `TARGET_OS[...]=ubuntu`. Adding another target distro should happen by adding a new target OS backend to the scripts instead of special-casing a profile name. ## Prepare A Target -Provisioning is target-side setup. It currently supports Ubuntu targets. +Provisioning is target-side setup. It currently supports Ubuntu targets and +takes the debug method as a second argument. ```bash -./scripts/01-provision-target.sh server +./scripts/01-provision-target.sh server kgdb # KGDB path +./scripts/01-provision-target.sh server qemu # QEMU stub path ``` -This installs the running target kernel's headers and `rsync`. It also prepares -the target for the serial KGDB path by adding boot arguments such as: +Both modes install the running target kernel's headers and `rsync`, and add +`nokaslr` to the kernel command line so vmlinux symbols line up with running +addresses. `kgdb` mode additionally adds: ```text -kgdboc=ttyS0,115200 nokaslr sysrq_always_enabled=1 +kgdboc=ttyS0,115200 sysrq_always_enabled=1 ``` Reboot the target VM after provisioning. @@ -89,11 +115,11 @@ Reboot the target VM after provisioning. For source debugging with full kernel symbols, provision with debug symbols: ```bash -./scripts/01-provision-target.sh server --debug-symbols +./scripts/01-provision-target.sh server qemu --debug-symbols ``` -Then sync the target kernel headers, build tree, and optional `vmlinux` into the -development host: +Then sync the target kernel headers, build tree, and optional `vmlinux` into +the development host (script 02 is the same regardless of debug method): ```bash ./scripts/02-setup-host-build.sh server @@ -125,21 +151,18 @@ script updates that link to the most recently synced target. ## Debug From VS Code -The VS Code debug configuration uses the same target-picking mechanism as the -build tasks. The checked-in picker includes `desktop` and `server`; add new -profile names to `.vscode/tasks.json` if you add more target profiles. +The VS Code debug tasks share two pickers: `kernelTarget` and `debugMethod`. +Both are listed in `.vscode/tasks.json`; keep them in sync with `TARGETS` in +`lab.local.env`. 1. Select `Kernel: Debug`. 2. Press F5. 3. VS Code runs the `Kernel: Deploy Debug` prelaunch task. -4. The task prompts for `kernelTarget`. -5. `scripts/04-deploy-debug-vscode.sh ` uploads and loads `hello.ko`, - prepares module symbols, and writes the current GDB files under `.gdb/`. -6. VS Code starts GDB and sources `.gdb/current-kgdb.gdb`. - -Source debugging through `scripts/04-deploy-debug-vscode.sh` and the VS Code -launch task is still work in progress. The current goal is to load module -symbols and stop in `module/src/hello.c`. +4. The task prompts for `kernelTarget` and `debugMethod`. +5. `scripts/04-deploy-debug-vscode.sh ` uploads and loads + `hello.ko`, prepares module symbols, and writes the current GDB files under + `.gdb/`. +6. VS Code starts GDB and sources `.gdb/current-debug.gdb`. The generated GDB script sets the kernel architecture to `i386:x86-64`, loads the target `vmlinux`, adds module symbols from `/sys/module/hello/sections/*`, @@ -153,11 +176,27 @@ GDB only needs an endpoint in this form: : ``` -Put that value in `TARGET_DEBUG_ENDPOINT` in `lab.local.env`. The endpoint can -come from VMware's debug stub, the serial bridge below, or any other bridge that -speaks GDB remote protocol. +Put that value in `TARGET_DEBUG_ENDPOINT_KGDB` or `TARGET_DEBUG_ENDPOINT_QEMU` +in `lab.local.env`, depending on the method you want to use. -### Option A: VMware Debug Stub (Recommended) +### QEMU gdbstub + +Launch QEMU with the built-in stub exposed on a TCP port, e.g.: + +```text +qemu-system-x86_64 ... -gdb tcp::1234 +``` + +(Use `-S` if you want the vCPU paused at boot. Without `-S`, the guest runs +until GDB connects and halts it.) + +```bash +declare -A TARGET_DEBUG_ENDPOINT_QEMU=( + [server]=127.0.0.1:1234 +) +``` + +### VMware Debug Stub Power off the VM, open the target VM's `.vmx` file, and add: @@ -168,60 +207,38 @@ debugStub.listen.guest64.remote = "TRUE" debugStub.hideBreakpoints = "FALSE" ``` -Start the VM after saving the `.vmx` file. Then set the matching endpoint: +Start the VM after saving the `.vmx` file. The VMware stub is the same shape +as QEMU's gdbstub — use the `qemu` debug method: ```bash -declare -A TARGET_DEBUG_ENDPOINT=( +declare -A TARGET_DEBUG_ENDPOINT_QEMU=( [server]=127.0.0.1:8864 ) ``` -If the development host cannot reach the VMware host's loopback address, use an -address for the VMware host that is reachable from the development host. If both -VMs may run at the same time, give each VM a different -`debugStub.port.guest64` value. - -This option does not need a VMware serial port or the PowerShell bridge. - -### Option B: VMware Serial Port To TCP +### KGDB Over Serial → TCP Use this path if you want Linux KGDB over a virtual serial port: ```text -VS Code -> GDB -> TCP port -> named pipe -> VMware serial port -> Ubuntu KGDB -``` - -In VMware Workstation, power off the VM and add a serial port: - -1. Select `Use named pipe`. -2. Set a unique pipe name. -3. Select `This end is the server`. -4. Select `The other end is an application`. -5. Enable `Connect at power on`. - -The repo defaults are: - -```text -desktop VM: \\.\pipe\kgdb-desktop -server VM: \\.\pipe\kgdb-server +VS Code -> GDB -> TCP port -> serial bridge -> guest /dev/ttyS0 -> Ubuntu KGDB ``` -Start the bridge from Windows PowerShell on the VMware host: +How you build the bridge depends on the hypervisor: -```powershell -powershell -ExecutionPolicy Bypass -File .\host\bridge-kgdb.ps1 -Target server -``` - -The default server bridge is: - -```text -\\.\pipe\kgdb-server <-> 127.0.0.1:5520 -``` +- **QEMU:** `-serial tcp:127.0.0.1:5520,server,nowait` +- **VMware Workstation:** named pipe + `host/bridge-kgdb.ps1`. Configure a + serial port with `Use named pipe`, `This end is the server`, `The other end + is an application`, `Connect at power on`. The repo defaults to + `\\.\pipe\kgdb-` and bridges it to `127.0.0.1:5520` via the + PowerShell script. -Set the matching endpoint: +Provision the target with the `kgdb` method and set the matching endpoint: ```bash -declare -A TARGET_DEBUG_ENDPOINT=( +./scripts/01-provision-target.sh server kgdb + +declare -A TARGET_DEBUG_ENDPOINT_KGDB=( [server]=127.0.0.1:5520 ) @@ -241,18 +258,12 @@ The TTY and baud rate must match the boot arguments added by provisioning. From the development host, check the endpoint before launching GDB: ```bash -nc -vz 127.0.0.1 8864 -``` - -For the serial bridge, you can also check from Windows PowerShell: - -```powershell -Test-NetConnection 127.0.0.1 -Port 5520 +nc -vz 127.0.0.1 1234 ``` -If the VMware host can connect but the development host cannot, use an address -for the VMware host that the development host can reach instead of `127.0.0.1`. -On WSL, the Windows host address is often listed as the resolver: +If the hypervisor host can connect but the development host cannot, use an +address for the hypervisor host that the development host can reach instead of +`127.0.0.1`. On WSL, the Windows host address is often listed as the resolver: ```bash grep nameserver /etc/resolv.conf @@ -300,18 +311,19 @@ Leave `TARGET_SUDO_PASS[target]` empty when the sudo password is the same as the SSH password. If F5 fails with `target remote ... Connection timed out`, the endpoint in -`.gdb/current-kgdb.gdb` is not reachable from the development host. Check -`TARGET_DEBUG_ENDPOINT[target]` and verify that your endpoint provider is +`.gdb/current-debug.gdb` is not reachable from the development host. Check +`TARGET_DEBUG_ENDPOINT_KGDB[target]` or `TARGET_DEBUG_ENDPOINT_QEMU[target]` +(depending on the method you picked) and verify that your endpoint provider is listening. If GDB connects but the module does not load, inspect: ```bash -cat .gdb/server-loader.log +cat .gdb/server-qemu-loader.log ``` -Use the target-specific log name for the selected target, for example -`.gdb/desktop-loader.log`. +Use the target- and method-specific log name for your run, for example +`.gdb/desktop-kgdb-loader.log`. If breakpoints bind to files under `build/intermediate/...`, regenerate the GDB files by pressing F5 again. The generated script should map those staged paths diff --git a/lab.example.env b/lab.example.env index a571c10..32c129f 100644 --- a/lab.example.env +++ b/lab.example.env @@ -52,15 +52,27 @@ declare -A TARGET_REMOTE_DIR=( [server]=/tmp/small-ko ) -# GDB connects to these endpoints from the development host. The endpoint can -# come from VMware's debug stub, a serial-port bridge, or any other TCP bridge -# that speaks GDB remote protocol. -declare -A TARGET_DEBUG_ENDPOINT=( +# Debug endpoints — one per debug method. Scripts 01 and 04 take a +# argument and pick the matching endpoint from these maps. +# Either map may be left empty per target if you do not use that method. + +# KGDB-over-serial: a TCP bridge to the target's serial port (kgdboc=ttyS*). +# Typically opened by VMware's named-pipe-to-TCP bridge, socat, or QEMU's +# `-serial tcp:HOST:PORT,server,nowait`. +declare -A TARGET_DEBUG_ENDPOINT_KGDB=( [desktop]=127.0.0.1:5510 [server]=127.0.0.1:5520 ) -# Serial KGDB settings used only by the serial-port debug path. +# QEMU's built-in gdbstub: typically opened by QEMU's `-gdb tcp::PORT` (or +# `-s`). VMware's `debugStub.listen.guest64` is the same shape and goes here +# too. Does not require kgdb in the guest. +declare -A TARGET_DEBUG_ENDPOINT_QEMU=( + [desktop]=127.0.0.1:1234 + [server]=127.0.0.1:1234 +) + +# Serial KGDB settings — only consulted when script 01 is run with `kgdb`. declare -A TARGET_KGDB_TTY=( [desktop]=ttyS0 [server]=ttyS0 @@ -71,6 +83,7 @@ declare -A TARGET_KGDB_BAUD=( [server]=115200 ) -# The module waits briefly before kgdb_breakpoint so the deploy script can read -# /sys/module/hello/sections/* and prepare module symbols for VS Code. +# Module sleeps this long at the start of init so the host can read +# /sys/module/hello/sections/* and so users have a window to break in GDB +# (Ctrl+C against the qemu stub, sysrq+g for kgdb) before init runs. DEBUG_LOAD_DELAY_MS=5000 diff --git a/module/src/hello.c b/module/src/hello.c index af12ec7..406b46d 100644 --- a/module/src/hello.c +++ b/module/src/hello.c @@ -2,8 +2,6 @@ #include "chuck_message.h" #include -#include -#include #include static unsigned int debug_delay_ms = 5000; @@ -18,12 +16,6 @@ static int hello_init(void) if (debug_delay_ms) msleep(debug_delay_ms); -#if IS_ENABLED(CONFIG_KGDB) - kgdb_breakpoint(); -#else - pr_warn("hello: CONFIG_KGDB is disabled; continuing without breakpoint\n"); -#endif - ret = chuck_device_register(); if (ret) { pr_err("hello: failed to register /dev/%s: %d\n", @@ -47,4 +39,4 @@ module_exit(hello_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("small-ko lab"); -MODULE_DESCRIPTION("Multi-file char device module for KGDB source debugging"); +MODULE_DESCRIPTION("Multi-file char device module for kernel source debugging"); diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index d0d2493..6935e8b 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -2,18 +2,20 @@ set -euo pipefail usage() { - echo "usage: $0 [--debug-symbols]" >&2 + echo "usage: $0 [--debug-symbols]" >&2 exit 2 } target="${1:-}" -symbols="${2:-}" +debug_method="${2:-}" +symbols="${3:-}" +case "$debug_method" in kgdb|qemu) ;; *) usage ;; esac case "$symbols" in ""|--debug-symbols) ;; *) usage ;; esac repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" -validate_target "$target" "usage: $0 [--debug-symbols]" +validate_target "$target" "usage: $0 [--debug-symbols]" env_file="$repo_root/lab.local.env" [[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" # shellcheck source=/dev/null @@ -54,10 +56,10 @@ if [[ -n "$ssh_pass" ]]; then ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") fi -echo "provisioning $target at $ssh_user@$ssh_host:$ssh_port" +echo "provisioning $target at $ssh_user@$ssh_host:$ssh_port (debug method: $debug_method)" SSHPASS="$ssh_pass" "${ssh_cmd[@]}" -t "$ssh_user@$ssh_host" \ - "TARGET_OS='$target_os' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$sudo_pass_b64' DEBUG_MAXCPUS='${DEBUG_MAXCPUS:-}' bash -s" <<'REMOTE' + "TARGET_OS='$target_os' DEBUG_METHOD='$debug_method' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$sudo_pass_b64' DEBUG_MAXCPUS='${DEBUG_MAXCPUS:-}' bash -s" <<'REMOTE' set -euo pipefail sudo_run() { @@ -112,9 +114,21 @@ fi grub_file=/etc/default/grub current="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" # Strip any prior values for the keys we manage so reruns with changed values -# replace rather than append. -current="$(printf '%s' "$current" | sed -E 's/(^| )(kgdboc=|maxcpus=)[^ ]*//g; s/ */ /g; s/^ +//; s/ +$//')" -args=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "nokaslr" "sysrq_always_enabled=1") +# replace rather than append. kgdboc and sysrq_always_enabled are also stripped +# so switching from kgdb to qemu mode removes them. +current="$(printf '%s' "$current" | sed -E 's/(^| )(kgdboc=|maxcpus=|sysrq_always_enabled=)[^ ]*//g; s/ */ /g; s/^ +//; s/ +$//')" + +# nokaslr is required in both modes so vmlinux symbols line up with running addresses. +args=("nokaslr") + +# kgdb mode also needs an in-kernel debugger channel (kgdboc) and sysrq enabled +# so users can trigger a halt with "echo g > /proc/sysrq-trigger" — there is no +# longer a kgdb_breakpoint() in the module to halt on insmod. qemu mode skips +# both: the hypervisor stub halts the vCPU directly. +if [[ "$DEBUG_METHOD" == "kgdb" ]]; then + args+=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "sysrq_always_enabled=1") +fi + if [[ -n "${DEBUG_MAXCPUS:-}" ]]; then args+=("maxcpus=${DEBUG_MAXCPUS}") fi diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index c2f5632..55e0442 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -1,12 +1,19 @@ #!/usr/bin/env bash set -euo pipefail +usage() { + echo "usage: $0 " >&2 + exit 2 +} + target="${1:-}" +debug_method="${2:-}" +case "$debug_method" in kgdb|qemu) ;; *) usage ;; esac repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" -validate_target "$target" "usage: $0 " +validate_target "$target" "usage: $0 " env_file="$repo_root/lab.local.env" [[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" # shellcheck source=/dev/null @@ -22,7 +29,8 @@ ssh_user="$(target_require_cfg "$target" SSH_USER)" ssh_pass="$(target_cfg "$target" SSH_PASS)" sudo_pass="$(target_cfg "$target" SUDO_PASS)" remote_dir="$(target_cfg "$target" REMOTE_DIR)" -debug_endpoint="$(target_require_cfg "$target" DEBUG_ENDPOINT)" +endpoint_key="DEBUG_ENDPOINT_${debug_method^^}" +debug_endpoint="$(target_require_cfg "$target" "$endpoint_key")" debug_delay_ms="${DEBUG_LOAD_DELAY_MS:-5000}" ssh_port="${ssh_port:-22}" @@ -83,20 +91,21 @@ fi gdb_dir="$repo_root/.gdb" mkdir -p "$gdb_dir" -symbols_file="$gdb_dir/$target-module-symbols.gdb" -gdb_file="$gdb_dir/$target-kgdb.gdb" -loader_log="$gdb_dir/$target-loader.log" -loader_script="$gdb_dir/$target-loader.sh" +symbols_file="$gdb_dir/$target-$debug_method-module-symbols.gdb" +gdb_file="$gdb_dir/$target-$debug_method.gdb" +loader_log="$gdb_dir/$target-$debug_method-loader.log" +loader_script="$gdb_dir/$target-$debug_method-loader.sh" rm -f "$symbols_file" "$loader_log" "$loader_script" remote_module="$remote_dir/hello.ko" remote_sudo="$(remote_sudo_prefix)" -# Kill any local gdb processes that still hold a TCP connection to the bridge. -# bridge-kgdb.ps1 serves one client at a time; leftover gdb processes from -# previous sessions (zombie bare-gdb invocations, half-torn-down VS Code -# launches, etc.) sit in the bridge's accept queue and block fresh attaches. +# Kill any local gdb processes that still hold a TCP connection to the debug +# endpoint. Both KGDB's serial bridge and QEMU's gdbstub serve one client at a +# time; leftover gdb processes from previous sessions (zombie bare-gdb +# invocations, half-torn-down VS Code launches, etc.) sit in the endpoint's +# accept queue and block fresh attaches. debug_host="${debug_endpoint%:*}" debug_port="${debug_endpoint##*:}" if command -v ss >/dev/null 2>&1; then @@ -151,11 +160,11 @@ EOF # called `target remote` and loaded the executable, so commands that touch # global gdb state (mi-async, remote timeouts, architecture override) error # with "Cannot change this setting while the inferior is running" — drop them. -gdb_attached_file="$gdb_dir/$target-kgdb-attached.gdb" +gdb_attached_file="$gdb_dir/$target-$debug_method-attached.gdb" grep -vE '^(target remote |set mi-async |set target-async |set tcp connect-timeout |set remotetimeout |set architecture |symbol-file )' "$gdb_file" > "$gdb_attached_file" -ln -sfn "$target-kgdb.gdb" "$gdb_dir/current-kgdb.gdb" -ln -sfn "$target-kgdb-attached.gdb" "$gdb_dir/current-kgdb-attached.gdb" +ln -sfn "$target-$debug_method.gdb" "$gdb_dir/current-debug.gdb" +ln -sfn "$target-$debug_method-attached.gdb" "$gdb_dir/current-debug-attached.gdb" ln -sfn "../.kernel-cache/$target/vmlinux" "$gdb_dir/current-vmlinux" { @@ -216,28 +225,13 @@ while ((SECONDS < deadline)); do echo "echo loaded hello.ko module symbols for $target\\n" } > "$symbols_file" echo "wrote module symbols to $symbols_file" - # Block until kgdb_breakpoint() actually halts the kernel. Returning - # earlier would race the VS Code F5 flow: cppdbg launches gdb - # immediately, gdb sends qSupported into the still-running kernel's - # console buffer where kgdb never sees it, and the launch hangs - # forever. We detect halt by SSH going unreachable. - echo "waiting for kgdb halt..." - halt_deadline=$((SECONDS + 60)) - while ((SECONDS < halt_deadline)); do - if ! SSHPASS="$ssh_pass" timeout 2 "${ssh_cmd[@]}" ":" >/dev/null 2>&1; then - echo "kgdb halted; ready for gdb to attach" - exit 0 - fi - sleep 0.3 - done - echo "warning: kgdb did not halt within 60s; gdb may hang on target remote" >&2 exit 0 fi fi sleep 0.2 done -echo "failed to discover /sys/module/hello/sections/.text before breakpoint" +echo "failed to discover /sys/module/hello/sections/.text within 30s" exit 1 LOADER } > "$loader_script" From a9ed59e44eaed0c6e94b7e6537543e04f31e6699 Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 00:57:38 +0300 Subject: [PATCH 06/13] changed launch tasks order --- .vscode/launch.json | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index a5a72e5..3094b4c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,24 +1,8 @@ { "version": "0.2.0", "configurations": [ - { - "name": "Kernel: Debug", - "type": "gdb", - "request": "attach", - "executable": "${workspaceFolder}/.gdb/current-vmlinux", - "target": "127.0.0.1:1234", - "remote": true, - "cwd": "${workspaceFolder}", - "preLaunchTask": "Kernel: Deploy Debug", - "valuesFormatting": "parseText", - "autorun": [ - "source ${workspaceFolder}/.gdb/current-debug-attached.gdb", - "hbreak chuck_device_register", - "continue" - ] - }, - { - "name": "Kernel: Debug (cppdbg, slow on thread enum)", + { + "name": "Kernel: Cppdbg Debug", "type": "cppdbg", "request": "launch", "program": "${workspaceFolder}/.gdb/current-vmlinux", @@ -36,6 +20,20 @@ ], "launchCompleteCommand": "exec-continue", "externalConsole": false + }, + { + "name": "Kernel: Native Debug", + "type": "gdb", + "request": "attach", + "executable": "${workspaceFolder}/.gdb/current-vmlinux", + "target": "127.0.0.1:1234", + "remote": true, + "cwd": "${workspaceFolder}", + "preLaunchTask": "Kernel: Deploy Debug", + "valuesFormatting": "parseText", + "autorun": [ + "source ${workspaceFolder}/.gdb/current-debug-attached.gdb", + ] } ] } From ff5e2bd87cd94825e6428c6acb4c5c1efe4c90de Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 11:33:14 +0300 Subject: [PATCH 07/13] make repo independent of chuck norise module --- .gitignore | 1 + .vscode/c_cpp_properties.json | 14 +- .vscode/launch.json | 14 +- .vscode/settings.json | 13 +- .vscode/tasks.json | 8 +- Makefile | 82 ++++-- README.md | 260 +++++++++++------- compile_commands.json | 27 -- examples/chuck_norise/Makefile | 33 +++ examples/chuck_norise/README.md | 52 ++++ .../chuck_norise}/include/chuck_device.h | 0 .../chuck_norise}/include/chuck_message.h | 0 .../chuck_norise}/src/chuck_device.c | 0 .../chuck_norise}/src/chuck_message.c | 0 {module => examples/chuck_norise}/src/hello.c | 0 host/bridge-kgdb.ps1 | 45 ++- host/restore-snapshot.ps1 | 64 ++--- lab.example.env | 24 +- module/Kbuild | 12 - module/README.md | 52 ++++ scripts/03-build-module.sh | 20 +- scripts/04-deploy-debug-vscode.sh | 49 ++-- 22 files changed, 489 insertions(+), 281 deletions(-) delete mode 100644 compile_commands.json create mode 100644 examples/chuck_norise/Makefile create mode 100644 examples/chuck_norise/README.md rename {module => examples/chuck_norise}/include/chuck_device.h (100%) rename {module => examples/chuck_norise}/include/chuck_message.h (100%) rename {module => examples/chuck_norise}/src/chuck_device.c (100%) rename {module => examples/chuck_norise}/src/chuck_message.c (100%) rename {module => examples/chuck_norise}/src/hello.c (100%) delete mode 100644 module/Kbuild create mode 100644 module/README.md diff --git a/.gitignore b/.gitignore index 354cc3f..b7f1afc 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ build/ Module.symvers modules.order +compile_commands.json .vscode/ipch/ .codex kernel.gdb diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 2712355..9065b7f 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -3,20 +3,19 @@ "configurations": [ { "name": "Linux kernel module - current", - "compilerPath": "/usr/bin/gcc-13", + "compilerPath": "/usr/bin/gcc", "intelliSenseMode": "linux-gcc-x64", "cStandard": "gnu11", "defines": [ "__KERNEL__", "MODULE", "CC_USING_FENTRY", - "KBUILD_MODNAME=\"hello\"", - "KBUILD_BASENAME=\"hello\"", - "__KBUILD_MODNAME=kmod_hello" + "KBUILD_MODNAME=\"kmod\"", + "KBUILD_BASENAME=\"kmod\"", + "__KBUILD_MODNAME=kmod_kmod" ], "includePath": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", + "${workspaceFolder}/module/**", "${workspaceFolder}/.kernel-cache/current/build/include", "${workspaceFolder}/.kernel-cache/current/build/include/uapi", "${workspaceFolder}/.kernel-cache/current/build/include/generated", @@ -29,8 +28,7 @@ ], "browse": { "path": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", + "${workspaceFolder}/module", "${workspaceFolder}/.kernel-cache/current/build/ubuntu/include", "${workspaceFolder}/.kernel-cache/current/build/include", "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include" diff --git a/.vscode/launch.json b/.vscode/launch.json index 3094b4c..abfbc34 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,7 +1,15 @@ { "version": "0.2.0", + "inputs": [ + { + "id": "debugEndpoint", + "type": "promptString", + "description": "GDB remote endpoint (host:port). Used only by the Native Debug launch; cppdbg reads it from the generated .gdb script.", + "default": "127.0.0.1:1234" + } + ], "configurations": [ - { + { "name": "Kernel: Cppdbg Debug", "type": "cppdbg", "request": "launch", @@ -26,13 +34,13 @@ "type": "gdb", "request": "attach", "executable": "${workspaceFolder}/.gdb/current-vmlinux", - "target": "127.0.0.1:1234", + "target": "${input:debugEndpoint}", "remote": true, "cwd": "${workspaceFolder}", "preLaunchTask": "Kernel: Deploy Debug", "valuesFormatting": "parseText", "autorun": [ - "source ${workspaceFolder}/.gdb/current-debug-attached.gdb", + "source ${workspaceFolder}/.gdb/current-debug-attached.gdb" ] } ] diff --git a/.vscode/settings.json b/.vscode/settings.json index 713c7a9..1d56719 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,21 +1,20 @@ { "C_Cpp.default.configurationProvider": "", "C_Cpp.default.configurationName": "Linux kernel module - current", - "C_Cpp.default.compileCommands": "${workspaceFolder}/compile_commands.json", - "C_Cpp.default.compilerPath": "/usr/bin/gcc-13", + "C_Cpp.default.compileCommands": "", + "C_Cpp.default.compilerPath": "/usr/bin/gcc", "C_Cpp.default.cStandard": "gnu11", "C_Cpp.default.intelliSenseMode": "linux-gcc-x64", "C_Cpp.default.defines": [ "__KERNEL__", "MODULE", "CC_USING_FENTRY", - "KBUILD_MODNAME=\"hello\"", - "KBUILD_BASENAME=\"hello\"", - "__KBUILD_MODNAME=kmod_hello" + "KBUILD_MODNAME=\"kmod\"", + "KBUILD_BASENAME=\"kmod\"", + "__KBUILD_MODNAME=kmod_kmod" ], "C_Cpp.default.includePath": [ - "${workspaceFolder}/module/include", - "${workspaceFolder}/module/src", + "${workspaceFolder}/module/**", "${workspaceFolder}/.kernel-cache/current/build/include", "${workspaceFolder}/.kernel-cache/current/build/include/uapi", "${workspaceFolder}/.kernel-cache/current/build/include/generated", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index f29b691..b9190e9 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -3,12 +3,8 @@ "inputs": [ { "id": "kernelTarget", - "type": "pickString", - "description": "Target profile", - "options": [ - "desktop", - "server" - ], + "type": "promptString", + "description": "Target profile (one of the names in TARGETS in lab.local.env)", "default": "server" }, { diff --git a/Makefile b/Makefile index 207d072..5893e0f 100644 --- a/Makefile +++ b/Makefile @@ -1,41 +1,73 @@ -MODULE_NAME := hello -HOST_KERNEL := $(shell uname -r) -CURRENT_CACHE := $(CURDIR)/.kernel-cache/current +# Top-level wrapper that drives the user's kernel module under module/. +# +# The user's module is built out-of-tree against a target-specific kernel +# header tree cached under .kernel-cache//. Build artifacts go under +# build/intermediate/// (Kbuild's M=) and the final .ko is +# moved to build/artifacts///. +# +# Staging exists for two reasons: +# 1. Per-target out-of-tree artifacts. Building the same module/ tree against +# multiple target kernels needs separate intermediate dirs. +# 2. DWARF path rewriting. -ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR) +# rewrites paths in DWARF debug info (and __FILE__) from the staged +# intermediate dir back to module/, so IDEs that bind breakpoints by +# absolute path (VS Code, CLion) match the symtab. +# +# The user's module/Makefile sees only a normal Kbuild invocation rooted at +# the staging directory. KCFLAGS is the kernel build's official escape hatch +# for adding C flags from the outside, so the user's Makefile stays generic. + +HOST_KERNEL := $(shell uname -r) +CURRENT_CACHE := $(CURDIR)/.kernel-cache/current CURRENT_TARGET := $(shell if [ -e "$(CURRENT_CACHE)" ]; then basename "$$(readlink -f "$(CURRENT_CACHE)")"; else printf local; fi) CURRENT_KERNEL := $(shell if [ -f "$(CURRENT_CACHE)/kernel.release" ]; then cat "$(CURRENT_CACHE)/kernel.release"; else printf '$(HOST_KERNEL)'; fi) -KDIR ?= $(if $(wildcard $(CURRENT_CACHE)/build),$(CURRENT_CACHE)/build,/lib/modules/$(HOST_KERNEL)/build) -BUILD_ID ?= $(CURRENT_TARGET)/$(CURRENT_KERNEL) -BUILD_ROOT ?= $(CURDIR)/build +KDIR ?= $(if $(wildcard $(CURRENT_CACHE)/build),$(CURRENT_CACHE)/build,/lib/modules/$(HOST_KERNEL)/build) +BUILD_ID ?= $(CURRENT_TARGET)/$(CURRENT_KERNEL) +BUILD_ROOT ?= $(CURDIR)/build INTERMEDIATE_DIR ?= $(BUILD_ROOT)/intermediate/$(BUILD_ID) -ARTIFACT_DIR ?= $(BUILD_ROOT)/artifacts/$(BUILD_ID) -MODULE_DIR ?= $(CURDIR)/module +ARTIFACT_DIR ?= $(BUILD_ROOT)/artifacts/$(BUILD_ID) +MODULE_DIR ?= $(CURDIR)/module + +EXTRA_CCFLAGS ?= -g -DDEBUG +KCFLAGS_INJECT := -ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR) $(EXTRA_CCFLAGS) .PHONY: all modules prepare-build clean all: modules modules: prepare-build - $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" MODULE_REAL_DIR="$(MODULE_DIR)" modules - mkdir -p "$(ARTIFACT_DIR)" - mv "$(INTERMEDIATE_DIR)/$(MODULE_NAME).ko" "$(ARTIFACT_DIR)/$(MODULE_NAME).ko" - @echo "built $(ARTIFACT_DIR)/$(MODULE_NAME).ko" + $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" KCFLAGS="$(KCFLAGS_INJECT)" modules + @mkdir -p "$(ARTIFACT_DIR)" + @set -e; \ + kos=$$(find "$(INTERMEDIATE_DIR)" -maxdepth 2 -name '*.ko' -type f); \ + if [ -z "$$kos" ]; then \ + echo "no .ko produced under $(INTERMEDIATE_DIR)" >&2; exit 1; \ + fi; \ + for ko in $$kos; do \ + name=$$(basename "$$ko"); \ + mv "$$ko" "$(ARTIFACT_DIR)/$$name"; \ + echo "built $(ARTIFACT_DIR)/$$name"; \ + done +# Mirror module/ into INTERMEDIATE_DIR as a tree of absolute symlinks so +# Kbuild's M= sees the user's sources in a writeable scratch dir without +# polluting module/. `cp -as` is GNU coreutils; it recursively creates dirs +# and symlinks each file. Re-running is safe because we wipe the symlink +# scaffolding first. prepare-build: - mkdir -p "$(INTERMEDIATE_DIR)" - if [ -L "$(INTERMEDIATE_DIR)/src" ]; then rm -f "$(INTERMEDIATE_DIR)/src"; fi - mkdir -p "$(INTERMEDIATE_DIR)/src" - ln -sfn "$(MODULE_DIR)/src/hello.c" "$(INTERMEDIATE_DIR)/src/hello.c" - ln -sfn "$(MODULE_DIR)/src/chuck_device.c" "$(INTERMEDIATE_DIR)/src/chuck_device.c" - ln -sfn "$(MODULE_DIR)/src/chuck_message.c" "$(INTERMEDIATE_DIR)/src/chuck_message.c" - if [ -L "$(INTERMEDIATE_DIR)/include" ]; then rm -f "$(INTERMEDIATE_DIR)/include"; fi - mkdir -p "$(INTERMEDIATE_DIR)/include" - ln -sfn "$(MODULE_DIR)/include/chuck_device.h" "$(INTERMEDIATE_DIR)/include/chuck_device.h" - ln -sfn "$(MODULE_DIR)/include/chuck_message.h" "$(INTERMEDIATE_DIR)/include/chuck_message.h" - ln -sfn "$(MODULE_DIR)/Kbuild" "$(INTERMEDIATE_DIR)/Kbuild" + @if [ ! -e "$(MODULE_DIR)/Makefile" ] && [ ! -e "$(MODULE_DIR)/Kbuild" ]; then \ + echo "no Makefile or Kbuild under $(MODULE_DIR)" >&2; \ + echo "see $(MODULE_DIR)/README.md for the expected layout" >&2; \ + exit 1; \ + fi + @mkdir -p "$(INTERMEDIATE_DIR)" + @find "$(INTERMEDIATE_DIR)" -depth -type l -delete + @find "$(INTERMEDIATE_DIR)" -depth -type d -empty -not -path "$(INTERMEDIATE_DIR)" -delete + @cp -as "$(MODULE_DIR)/." "$(INTERMEDIATE_DIR)/" clean: - if [ -d "$(INTERMEDIATE_DIR)" ] && [ -e "$(INTERMEDIATE_DIR)/Kbuild" ]; then \ - $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" clean; \ + @if [ -d "$(INTERMEDIATE_DIR)" ] && { [ -e "$(INTERMEDIATE_DIR)/Makefile" ] || [ -e "$(INTERMEDIATE_DIR)/Kbuild" ]; }; then \ + $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" clean || true; \ fi rm -rf "$(INTERMEDIATE_DIR)" "$(ARTIFACT_DIR)" diff --git a/README.md b/README.md index 8a35870..adbdd6f 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,27 @@ -# Kernel Module Development Lab - -This repo is a development setup for building and debugging Linux kernel -modules against target VMs. - -The sample module builds as `hello.ko` and exposes `/dev/chuck_norise`. Reads -repeat the exact string `chuck norise!`, preserving the file offset for each -open file descriptor. +# Linux Kernel Module Debug Lab — Template + +A reusable development and source-debugging setup for out-of-tree Linux kernel +modules. The repo is generic: drop your module project under `module/`, point +the lab at a target VM, and source-debug from VS Code or `gdb -tui`. + +A worked example lives under `examples/chuck_norise/` — a tiny multi-file +char device. Copy it into `module/` to run the full flow end-to-end without +writing any module code first. + +## What This Template Gives You + +- A per-target, out-of-tree build that stages your module/ tree into + `build/intermediate///` and writes the final `.ko` under + `build/artifacts///`. +- DWARF path rewriting (`-ffile-prefix-map`) injected via `KCFLAGS`, so + breakpoints set in your IDE bind to the real files under `module/` and + not to the staged copies. +- Provisioning, header sync, build, deploy, load, and symbol discovery + driven from four numbered scripts plus the VS Code task/launch wiring + that calls them. +- Support for two debug methods against the same target — in-kernel KGDB + over serial (`kgdb`) and a hypervisor GDB stub (`qemu`, also covers + VMware's `debugStub.listen.guest64`). ## Machine Model @@ -17,33 +33,74 @@ The lab uses two kinds of machines: `scp`, `rsync`, `make`, and standard GNU userland tools. 2. A target VM. - The target is where the module is loaded and tested. Communication with the - target is over SSH. For now, the supported target profiles are Ubuntu 24.04 - Desktop (`desktop`) and Ubuntu 24.04 Server (`server`). Other Linux target - distros can be added later by teaching the provisioning and sync scripts how - to install headers, find kernel build trees, and configure debugging for that - distro. + The target is where the module is loaded and tested. Communication with + the target is over SSH. For now, the supported target OS is Ubuntu — add + a target OS backend in the scripts to support another distro. -## Debug Methods +## The module/ Contract -This repo supports two debug methods. Provisioning (script 01) and the deploy -step (script 04) both take a `` argument so you can switch between -them per run. +`module/` is the single slot for your module project. The lab is otherwise +generic — it does not know your module's name, source layout, or behavior. -- **`kgdb`** — in-kernel KGDB talking over the guest's serial port. The - provisioning step adds `kgdboc=ttyS*,115200` and `sysrq_always_enabled=1` to - the guest's kernel command line. To halt the running kernel into GDB, run - `echo g | sudo tee /proc/sysrq-trigger` on the target (or Ctrl+C inside GDB - if your endpoint supports it). Useful when you cannot easily change the - hypervisor's command line, e.g. VMware running as your only option. +The build pipeline expects: + +- `module/Makefile` (or `module/Kbuild`) following standard out-of-tree + Kbuild conventions: -- **`qemu`** — QEMU's built-in gdbstub (`-gdb tcp::PORT` or `-s`). No KGDB in - the guest is required; the hypervisor halts the vCPU directly. VMware's - `debugStub.listen.guest64` is the same shape and slots into this method too. - Connecting GDB to the stub halts the vCPU; set breakpoints, then `continue`. + ```makefile + obj-m += my_module.o + my_module-y := src/main.o src/util.o + ccflags-y := -I$(src)/include -g -DDEBUG + ``` -The module does **not** call `kgdb_breakpoint()` on insmod. In both methods you -break manually after the deploy step finishes. + See `examples/chuck_norise/Makefile` for the canonical shape, including + the optional `ifndef KERNELRELEASE` wrapper that also lets you run plain + `make` directly in `module/`. + +- Exactly one `obj-m` entry per build. The artifact name becomes + `.ko`. The lab discovers the module name from the produced `.ko`, + so you do not declare it anywhere else. + +- Optional: a `debug_delay_ms` module parameter that sleeps in `module_init` + before doing anything observable. The lab passes the value from + `DEBUG_LOAD_DELAY_MS` (default 5000 ms) to `insmod`. Modules without the + parameter are loaded plainly. + +You decide everything else: source layout, header layout, license, exported +symbols. + +## Try The Example Module + +```bash +cp -r examples/chuck_norise/. module/ +``` + +Then run the lab flow described below. + +## Debug Methods + +This repo supports two debug methods. Provisioning (script 01) and the +deploy step (script 04) both take a `` argument so you can switch +between them per run. + +- **`kgdb`** — in-kernel KGDB talking over the guest's serial port. The + provisioning step adds `kgdboc=ttyS*,115200` and `sysrq_always_enabled=1` + to the guest's kernel command line. To halt the running kernel into GDB, + run `echo g | sudo tee /proc/sysrq-trigger` on the target. Useful when + you cannot change the hypervisor's command line, e.g. VMware running as + your only option. + +- **`qemu`** — QEMU's built-in gdbstub (`-gdb tcp::PORT` or `-s`). No KGDB + in the guest is required; the hypervisor halts the vCPU directly. + VMware's `debugStub.listen.guest64` is the same shape and slots into this + method too. Connecting GDB to the stub halts the vCPU; set breakpoints, + then `continue`. + +Neither method assumes the module calls `kgdb_breakpoint()` on insmod. In +both methods you break manually after the deploy step finishes. The lab's +convention is for the module to sleep `debug_delay_ms` (default 5000 ms) at +the start of init so you have a window to break before init runs and so the +host has time to read `/sys/module//sections/*`. ## Configure The Lab @@ -53,8 +110,8 @@ Copy the example environment file and edit it for your machines: cp lab.example.env lab.local.env ``` -Targets are data, not variable prefixes. Add profile names to `TARGETS`, then -fill the `TARGET_*` maps with entries keyed by that profile name: +Targets are data, not variable prefixes. Add profile names to `TARGETS`, +then fill the `TARGET_*` maps with entries keyed by that profile name: ```bash TARGETS=(desktop server) @@ -88,9 +145,9 @@ declare -A TARGET_DEBUG_ENDPOINT_QEMU=( Either map may be left empty per target. Script 04 errors clearly if the endpoint for the method you picked is missing. -For now, target provisioning and header sync implement `TARGET_OS[...]=ubuntu`. -Adding another target distro should happen by adding a new target OS backend to -the scripts instead of special-casing a profile name. +The VS Code task pickers prompt for target name as free text, so adding a +target is just an edit to `lab.local.env` — no other edits required for +VS Code to pick it up. ## Prepare A Target @@ -127,8 +184,8 @@ the development host (script 02 is the same regardless of debug method): ## Build -In VS Code, press `Ctrl+Shift+B`. The default `Kernel: Build` task prompts for -`kernelTarget` and runs: +In VS Code, press `Ctrl+Shift+B`. The default `Kernel: Build` task prompts +for `kernelTarget` and runs: ```bash ./scripts/03-build-module.sh @@ -137,7 +194,7 @@ In VS Code, press `Ctrl+Shift+B`. The default `Kernel: Build` task prompts for The module is written to: ```text -build/artifacts///hello.ko +build/artifacts///.ko ``` Build intermediates are kept under: @@ -146,27 +203,28 @@ Build intermediates are kept under: build/intermediate/// ``` -Running plain `make` uses `.kernel-cache/current` when it exists. The setup -script updates that link to the most recently synced target. +Running plain `make` at the repo root uses `.kernel-cache/current` when it +exists, otherwise the host kernel. The setup script updates that link to +the most recently synced target. ## Debug From VS Code -The VS Code debug tasks share two pickers: `kernelTarget` and `debugMethod`. -Both are listed in `.vscode/tasks.json`; keep them in sync with `TARGETS` in -`lab.local.env`. +The VS Code debug tasks share two inputs: `kernelTarget` (free-text prompt) +and `debugMethod` (kgdb|qemu). -1. Select `Kernel: Debug`. +1. Select `Kernel: Cppdbg Debug` (or `Kernel: Native Debug`). 2. Press F5. 3. VS Code runs the `Kernel: Deploy Debug` prelaunch task. 4. The task prompts for `kernelTarget` and `debugMethod`. 5. `scripts/04-deploy-debug-vscode.sh ` uploads and loads - `hello.ko`, prepares module symbols, and writes the current GDB files under - `.gdb/`. + your module, prepares module symbols, and writes the current GDB files + under `.gdb/`. 6. VS Code starts GDB and sources `.gdb/current-debug.gdb`. -The generated GDB script sets the kernel architecture to `i386:x86-64`, loads -the target `vmlinux`, adds module symbols from `/sys/module/hello/sections/*`, -and maps staged Kbuild paths back to the real files under `module/`. +The generated GDB script sets the kernel architecture to `i386:x86-64`, +loads the target `vmlinux`, adds module symbols from +`/sys/module//sections/*`, and maps staged Kbuild paths back to the +real files under `module/`. ## Debug Endpoint Options @@ -228,10 +286,13 @@ How you build the bridge depends on the hypervisor: - **QEMU:** `-serial tcp:127.0.0.1:5520,server,nowait` - **VMware Workstation:** named pipe + `host/bridge-kgdb.ps1`. Configure a - serial port with `Use named pipe`, `This end is the server`, `The other end - is an application`, `Connect at power on`. The repo defaults to - `\\.\pipe\kgdb-` and bridges it to `127.0.0.1:5520` via the - PowerShell script. + serial port with `Use named pipe`, `This end is the server`, `The other + end is an application`, `Connect at power on`. Then run the bridge + script on the Windows host: + + ```powershell + .\host\bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520 + ``` Provision the target with the `kgdb` method and set the matching endpoint: @@ -262,71 +323,43 @@ nc -vz 127.0.0.1 1234 ``` If the hypervisor host can connect but the development host cannot, use an -address for the hypervisor host that the development host can reach instead of -`127.0.0.1`. On WSL, the Windows host address is often listed as the resolver: +address for the hypervisor host that the development host can reach instead +of `127.0.0.1`. On WSL, the Windows host address is often listed as the +resolver: ```bash grep nameserver /etc/resolv.conf ``` -After changing `lab.local.env`, run F5 again so script 04 regenerates the GDB -files under `.gdb/`. - -## Test The Module - -After loading the module, run this on the target VM: - -```bash -head -c 10 /dev/chuck_norise -``` - -Expected output: - -```text -chuck nori -``` - -Use one open file descriptor to see offset-preserving reads: - -```bash -exec 9/dev/null -dd bs=1 count=5 <&9 2>/dev/null -dd bs=1 count=7 <&9 2>/dev/null -exec 9<&- -``` - -Expected output chunks are `chu`, `ck no`, and `rise!ch`. +After changing `lab.local.env`, run F5 again so script 04 regenerates the +GDB files under `.gdb/`. ## Troubleshooting -If a script says `sshpass` is missing and you use SSH passwords, install it on -the development host: +If a script says `sshpass` is missing and you use SSH passwords, install it +on the development host: ```bash sudo apt-get install -y sshpass ``` -Leave `TARGET_SUDO_PASS[target]` empty when the sudo password is the same as -the SSH password. +Leave `TARGET_SUDO_PASS[target]` empty when the sudo password is the same +as the SSH password. If F5 fails with `target remote ... Connection timed out`, the endpoint in `.gdb/current-debug.gdb` is not reachable from the development host. Check `TARGET_DEBUG_ENDPOINT_KGDB[target]` or `TARGET_DEBUG_ENDPOINT_QEMU[target]` -(depending on the method you picked) and verify that your endpoint provider is -listening. +(depending on the method you picked) and verify that your endpoint provider +is listening. If GDB connects but the module does not load, inspect: ```bash -cat .gdb/server-qemu-loader.log +cat .gdb/--loader.log ``` -Use the target- and method-specific log name for your run, for example -`.gdb/desktop-kgdb-loader.log`. - -If breakpoints bind to files under `build/intermediate/...`, regenerate the GDB -files by pressing F5 again. The generated script should map those staged paths +If breakpoints bind to files under `build/intermediate/...`, regenerate the +GDB files by pressing F5 again. The generated script maps those staged paths back to `module/` with `set substitute-path`. ## VS Code Include Errors @@ -335,19 +368,36 @@ The C/C++ extension reads `.vscode/c_cpp_properties.json`. It expects target kernel headers under `.kernel-cache//build`, which are created by: ```bash -./scripts/02-setup-host-build.sh desktop -./scripts/02-setup-host-build.sh server +./scripts/02-setup-host-build.sh ``` -Before that sync runs, VS Code can show include squiggles for kernel headers -such as `linux/module.h` or `linux/fs.h`. If the squiggles remain after syncing -headers, run `C/C++: Reset IntelliSense Database` from the command palette and -select the matching configuration. +Before that sync runs, VS Code can show include squiggles for kernel +headers such as `linux/module.h` or `linux/fs.h`. If the squiggles remain +after syncing headers, run `C/C++: Reset IntelliSense Database` from the +command palette. + +`KBUILD_MODNAME` is set to a placeholder (`"kmod"`) in the IntelliSense +defines. The actual value at compile time is your real module name; the +placeholder is only there so the C/C++ extension can resolve macros that +reference it. ## Repository Layout -The module source is under `module/`: C files in `module/src/`, headers in -`module/include/`, and the Kbuild fragment in `module/Kbuild`. +```text +module/ # your module project lives here +examples/chuck_norise/ # worked example you can copy into module/ +scripts/01-provision-target.sh # target-side: headers, grub args +scripts/02-setup-host-build.sh # host-side: sync headers + vmlinux +scripts/03-build-module.sh # host-side: build via Kbuild +scripts/04-deploy-debug-vscode.sh # host-side: upload, insmod, gen .gdb +scripts/lib/common.sh # shared bash helpers +host/bridge-kgdb.ps1 # Windows-side KGDB serial-to-TCP bridge +host/restore-snapshot.ps1 # Windows-side VMware snapshot helper +Makefile # generic out-of-tree wrapper +lab.example.env # copy to lab.local.env and edit +.vscode/ # tasks, launch, IntelliSense +``` -The top-level scripts and Makefile are development tooling. The optional -PowerShell scripts under `host/` are helpers for VMware Workstation on Windows. +The top-level scripts and `Makefile` are the lab's tooling — generic across +modules. The optional PowerShell scripts under `host/` are helpers for +VMware Workstation on Windows. diff --git a/compile_commands.json b/compile_commands.json deleted file mode 100644 index 04ec83f..0000000 --- a/compile_commands.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"hello\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/src/hello.c", - "file": "/home/dor/small-ko/module/src/hello.c" - }, - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"chuck_device\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/src/chuck_device.c", - "file": "/home/dor/small-ko/module/src/chuck_device.c" - }, - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"chuck_message\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/src/chuck_message.c", - "file": "/home/dor/small-ko/module/src/chuck_message.c" - }, - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -x c-header -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"chuck_device\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/include/chuck_device.h", - "file": "/home/dor/small-ko/module/include/chuck_device.h" - }, - { - "directory": "/home/dor/small-ko", - "command": "/usr/bin/gcc-13 -fsyntax-only -x c-header -nostdinc -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated -I/home/dor/small-ko/.kernel-cache/current/build/include -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/arch/x86/include/generated/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/uapi -I/home/dor/small-ko/.kernel-cache/current/build/include/generated/uapi -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler-version.h -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/kconfig.h -I/home/dor/small-ko/.kernel-cache/current/build/ubuntu/include -include /home/dor/small-ko/.kernel-cache/current/build/include/linux/compiler_types.h -I/home/dor/small-ko/module/include -D__KERNEL__ -DMODULE -DCC_USING_FENTRY -DKBUILD_BASENAME=\\\"chuck_message\\\" -DKBUILD_MODNAME=\\\"hello\\\" -D__KBUILD_MODNAME=kmod_hello -std=gnu11 /home/dor/small-ko/module/include/chuck_message.h", - "file": "/home/dor/small-ko/module/include/chuck_message.h" - } -] diff --git a/examples/chuck_norise/Makefile b/examples/chuck_norise/Makefile new file mode 100644 index 0000000..5aa9583 --- /dev/null +++ b/examples/chuck_norise/Makefile @@ -0,0 +1,33 @@ +# Standard out-of-tree Linux kernel module Makefile. +# +# This file is read in two different modes: +# +# 1. As a Kbuild fragment, when the kernel build system pulls it in via +# `make -C $KDIR M=$THIS_DIR modules`. In that mode KERNELRELEASE is set, +# so the Kbuild assignments below run and the wrapper recipe is skipped. +# +# 2. As a standalone Makefile, when a developer runs `make` in this directory +# directly. In that mode KERNELRELEASE is unset, so the wrapper recipe runs +# and re-invokes mode 1 against the host kernel's build tree. +# +# The lab's outer build (scripts/03-build-module.sh) always drives mode 1 +# against a target-specific kernel cache, but mode 2 is useful for ad-hoc +# builds against the host kernel. + +obj-m += chuck_norise.o +chuck_norise-y := src/hello.o src/chuck_device.o src/chuck_message.o + +ccflags-y := -I$(src)/include -g -DDEBUG + +ifndef KERNELRELEASE +KDIR ?= /lib/modules/$(shell uname -r)/build +PWD := $(CURDIR) + +.PHONY: default clean + +default: + $(MAKE) -C $(KDIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(KDIR) M=$(PWD) clean +endif diff --git a/examples/chuck_norise/README.md b/examples/chuck_norise/README.md new file mode 100644 index 0000000..0576a73 --- /dev/null +++ b/examples/chuck_norise/README.md @@ -0,0 +1,52 @@ +# chuck_norise — example module + +A small multi-file char device module used as the worked example for this lab. +Builds as `chuck_norise.ko` and exposes `/dev/chuck_norise`. Reads return the +exact string `chuck norise!` repeated indefinitely, preserving the file offset +for each open file descriptor. + +It is intentionally tiny so it stays useful as a template walk-through: an init +that sleeps long enough to give you time to attach GDB, a cdev/class lifecycle +split into its own file, and a separately-testable "produce bytes into a +userspace buffer" routine. + +## Use it as a template + +The repo's build pipeline operates on whatever lives under `module/` at the +repo root. To try this example end-to-end, copy it into place: + +```bash +cp -r examples/chuck_norise/. module/ +``` + +Then run the normal lab flow from the repo root: `scripts/01-...` through +`scripts/04-...`, or press F5 in VS Code. After the module loads on the +target VM: + +```bash +head -c 10 /dev/chuck_norise # prints "chuck nori" +``` + +To see offset-preserving reads from a single open fd: + +```bash +exec 9/dev/null # chu +dd bs=1 count=5 <&9 2>/dev/null # ck no +dd bs=1 count=7 <&9 2>/dev/null # rise!ch +exec 9<&- +``` + +## What this example demonstrates + +- A standard out-of-tree Linux kernel module project layout (`src/`, `include/`, + `Makefile`). +- A `Makefile` that doubles as a Kbuild fragment and as a standalone wrapper — + the conventional shape for kernel modules. The lab's outer build invokes it + the Kbuild way; running `make` in this directory directly invokes the wrapper. +- A module init that sleeps for `debug_delay_ms` (default 5000 ms, exposed as + a module parameter) before doing anything observable. The delay gives the + host time to attach GDB and set breakpoints before init runs. +- A `LINUX_VERSION_CODE` shim for the 6.4 `class_create()` signature change, + as a real-world example of one of the things out-of-tree modules have to + carry. diff --git a/module/include/chuck_device.h b/examples/chuck_norise/include/chuck_device.h similarity index 100% rename from module/include/chuck_device.h rename to examples/chuck_norise/include/chuck_device.h diff --git a/module/include/chuck_message.h b/examples/chuck_norise/include/chuck_message.h similarity index 100% rename from module/include/chuck_message.h rename to examples/chuck_norise/include/chuck_message.h diff --git a/module/src/chuck_device.c b/examples/chuck_norise/src/chuck_device.c similarity index 100% rename from module/src/chuck_device.c rename to examples/chuck_norise/src/chuck_device.c diff --git a/module/src/chuck_message.c b/examples/chuck_norise/src/chuck_message.c similarity index 100% rename from module/src/chuck_message.c rename to examples/chuck_norise/src/chuck_message.c diff --git a/module/src/hello.c b/examples/chuck_norise/src/hello.c similarity index 100% rename from module/src/hello.c rename to examples/chuck_norise/src/hello.c diff --git a/host/bridge-kgdb.ps1 b/host/bridge-kgdb.ps1 index fb63d39..0b45997 100644 --- a/host/bridge-kgdb.ps1 +++ b/host/bridge-kgdb.ps1 @@ -1,11 +1,27 @@ +# Generic VMware-Workstation-on-Windows bridge from a guest serial port (exposed +# as a Windows named pipe) to a TCP listener that GDB connects to. +# +# This is the Windows-host side of the `kgdb` debug method when the hypervisor +# is VMware Workstation. Configure a serial port in the VM as: +# Use named pipe = \\.\pipe\ +# This end is the server +# The other end is an application +# Connect at power on +# +# Then run this script on the Windows host whenever you want to debug: +# .\bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520 +# +# The lab does not bundle target-specific defaults here on purpose; pipe names +# and ports are user choices that should live in your own wrapper, in your VM +# settings, or in your shell history. Match TARGET_DEBUG_ENDPOINT_KGDB[] +# in lab.local.env to the -ListenAddress:-Port you pass here. + param( [Parameter(Mandatory = $true)] - [ValidateSet("desktop", "server")] - [string]$Target, - [string]$PipeName, - [int]$Port = 0, + [Parameter(Mandatory = $true)] + [int]$Port, [string]$ListenAddress = "127.0.0.1", @@ -14,25 +30,6 @@ param( $ErrorActionPreference = "Stop" -$Defaults = @{ - desktop = @{ - PipeName = "kgdb-desktop" - Port = 5510 - } - server = @{ - PipeName = "kgdb-server" - Port = 5520 - } -} - -if (-not $PipeName) { - $PipeName = $Defaults[$Target].PipeName -} - -if ($Port -eq 0) { - $Port = $Defaults[$Target].Port -} - if ($PipeName -match '^\\\\[^\\]+\\pipe\\(.+)$') { $PipeName = $Matches[1] } @@ -49,7 +46,7 @@ function Close-Quietly($Resource) { } } -Write-Host "Starting KGDB bridge for $Target" +Write-Host "Starting KGDB bridge" Write-Host " TCP: ${ListenAddress}:${Port}" Write-Host " Pipe: \\.\pipe\$PipeName" Write-Host "Press Ctrl+C to stop." diff --git a/host/restore-snapshot.ps1 b/host/restore-snapshot.ps1 index b614803..a4b4605 100644 --- a/host/restore-snapshot.ps1 +++ b/host/restore-snapshot.ps1 @@ -1,62 +1,54 @@ +# Generic helper for reverting a VMware Workstation VM to a named snapshot +# before each debug session, so each run starts from a known-clean state. +# +# This is optional and intended as a starting point. Wire it into your own +# pre-debug routine if you find it useful. +# +# Example: +# .\restore-snapshot.ps1 -VmxPath 'C:\VMs\target\target.vmx' ` +# -Snapshot clean-debug ` +# -StartAfterRestore + param( [Parameter(Mandatory = $true)] - [ValidateSet("desktop", "server")] - [string]$Target, + [string]$VmxPath, + + [Parameter(Mandatory = $true)] + [string]$Snapshot, + + [string]$VmrunPath = "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe", [switch]$StartAfterRestore ) $ErrorActionPreference = "Stop" -$Vmrun = "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe" - -$Targets = @{ - desktop = @{ - VmxPath = "C:\Users\dor\Documents\Virtual Machines\Ubuntu-Main\Ubuntu-Main.vmx" - Snapshot = "clean-debug" - } - server = @{ - VmxPath = "C:\Users\dor\Documents\Virtual Machines\ubuntu-server\ubuntu-server.vmx" - Snapshot = "clean-debug" - } -} - -if (-not (Test-Path -LiteralPath $Vmrun)) { - throw "vmrun.exe was not found at '$Vmrun'" -} - -$Config = $Targets[$Target] -if (-not $Config) { - throw "Unknown target '$Target'" +if (-not (Test-Path -LiteralPath $VmrunPath)) { + throw "vmrun.exe was not found at '$VmrunPath'" } -if ($Config.VmxPath -like "C:\Path\To\*") { - throw "Edit host/restore-snapshot.ps1 and set the real VMX path for '$Target'" +if (-not (Test-Path -LiteralPath $VmxPath)) { + throw "VMX path does not exist: '$VmxPath'" } -if (-not (Test-Path -LiteralPath $Config.VmxPath)) { - throw "VMX path does not exist: '$($Config.VmxPath)'" -} - -Write-Host "Stopping $Target if it is running..." -& $Vmrun -T ws stop $Config.VmxPath soft +Write-Host "Stopping VM if it is running..." +& $VmrunPath -T ws stop $VmxPath soft if ($LASTEXITCODE -ne 0) { Write-Host "Soft stop failed or VM was not running; continuing to snapshot revert." } -Write-Host "Reverting $Target to snapshot '$($Config.Snapshot)'..." -& $Vmrun -T ws revertToSnapshot $Config.VmxPath $Config.Snapshot +Write-Host "Reverting VM to snapshot '$Snapshot'..." +& $VmrunPath -T ws revertToSnapshot $VmxPath $Snapshot if ($LASTEXITCODE -ne 0) { throw "Snapshot revert failed with exit code $LASTEXITCODE" } if ($StartAfterRestore) { - Write-Host "Starting $Target..." - & $Vmrun -T ws start $Config.VmxPath + Write-Host "Starting VM..." + & $VmrunPath -T ws start $VmxPath if ($LASTEXITCODE -ne 0) { throw "VM start failed with exit code $LASTEXITCODE" } } -Write-Host "Snapshot restore complete for $Target." - +Write-Host "Snapshot restore complete." diff --git a/lab.example.env b/lab.example.env index 32c129f..aaa62da 100644 --- a/lab.example.env +++ b/lab.example.env @@ -7,8 +7,13 @@ SCP_BIN=scp RSYNC_BIN=rsync SSHPASS_BIN=sshpass -# Target profiles. The VS Code picker in .vscode/tasks.json should list the -# same names when you add or remove profiles. +# Target profiles. These names are data, not variable prefixes — every +# TARGET_* map below is keyed by these names. Add a target by adding entries +# to TARGETS and to every map you care about for that target. +# +# Pick whatever names you like ([letters][letters digits _ -]*). The VS Code +# task pickers accept any name you type; there is no per-target editing +# required there. TARGETS=(desktop server) # Only ubuntu targets are implemented for now. Add a new target OS backend in @@ -46,10 +51,11 @@ declare -A TARGET_SUDO_PASS=( [server]= ) -# Remote staging directory used for uploading hello.ko. +# Remote staging directory used for uploading the built module. Any writable +# path is fine; /tmp/* gets cleaned on reboot which is usually what you want. declare -A TARGET_REMOTE_DIR=( - [desktop]=/tmp/small-ko - [server]=/tmp/small-ko + [desktop]=/tmp/kmod-debug-lab + [server]=/tmp/kmod-debug-lab ) # Debug endpoints — one per debug method. Scripts 01 and 04 take a @@ -83,7 +89,9 @@ declare -A TARGET_KGDB_BAUD=( [server]=115200 ) -# Module sleeps this long at the start of init so the host can read -# /sys/module/hello/sections/* and so users have a window to break in GDB -# (Ctrl+C against the qemu stub, sysrq+g for kgdb) before init runs. +# Module sleeps this long at the start of init, IF it exposes the +# `debug_delay_ms` module parameter. The lab uses this convention so the host +# can read /sys/module//sections/* and so users have a window to break +# in GDB (Ctrl+C against the qemu stub, sysrq+g for kgdb) before init runs. +# Modules that do not declare the parameter get loaded without it; no harm. DEBUG_LOAD_DELAY_MS=5000 diff --git a/module/Kbuild b/module/Kbuild deleted file mode 100644 index 7ba375e..0000000 --- a/module/Kbuild +++ /dev/null @@ -1,12 +0,0 @@ -MODULE_NAME ?= hello - -# Real on-disk module source directory. The parent Makefile passes this so the -# compiler can rewrite paths in DWARF debug info (and __FILE__) from the staged -# intermediate dir back to module/. Without that rewrite, IDEs that set -# breakpoints by absolute path (VS Code, CLion, ...) can't match the symtab. -MODULE_REAL_DIR ?= $(src) - -obj-m += $(MODULE_NAME).o -$(MODULE_NAME)-y := src/hello.o src/chuck_device.o src/chuck_message.o - -ccflags-y := -I$(src)/include -g -DDEBUG -ffile-prefix-map=$(src)=$(MODULE_REAL_DIR) diff --git a/module/README.md b/module/README.md new file mode 100644 index 0000000..0576a73 --- /dev/null +++ b/module/README.md @@ -0,0 +1,52 @@ +# chuck_norise — example module + +A small multi-file char device module used as the worked example for this lab. +Builds as `chuck_norise.ko` and exposes `/dev/chuck_norise`. Reads return the +exact string `chuck norise!` repeated indefinitely, preserving the file offset +for each open file descriptor. + +It is intentionally tiny so it stays useful as a template walk-through: an init +that sleeps long enough to give you time to attach GDB, a cdev/class lifecycle +split into its own file, and a separately-testable "produce bytes into a +userspace buffer" routine. + +## Use it as a template + +The repo's build pipeline operates on whatever lives under `module/` at the +repo root. To try this example end-to-end, copy it into place: + +```bash +cp -r examples/chuck_norise/. module/ +``` + +Then run the normal lab flow from the repo root: `scripts/01-...` through +`scripts/04-...`, or press F5 in VS Code. After the module loads on the +target VM: + +```bash +head -c 10 /dev/chuck_norise # prints "chuck nori" +``` + +To see offset-preserving reads from a single open fd: + +```bash +exec 9/dev/null # chu +dd bs=1 count=5 <&9 2>/dev/null # ck no +dd bs=1 count=7 <&9 2>/dev/null # rise!ch +exec 9<&- +``` + +## What this example demonstrates + +- A standard out-of-tree Linux kernel module project layout (`src/`, `include/`, + `Makefile`). +- A `Makefile` that doubles as a Kbuild fragment and as a standalone wrapper — + the conventional shape for kernel modules. The lab's outer build invokes it + the Kbuild way; running `make` in this directory directly invokes the wrapper. +- A module init that sleeps for `debug_delay_ms` (default 5000 ms, exposed as + a module parameter) before doing anything observable. The delay gives the + host time to attach GDB and set breakpoints before init runs. +- A `LINUX_VERSION_CODE` shim for the 6.4 `class_create()` signature change, + as a real-world example of one of the things out-of-tree modules have to + carry. diff --git a/scripts/03-build-module.sh b/scripts/03-build-module.sh index bbe410e..9d01b3b 100755 --- a/scripts/03-build-module.sh +++ b/scripts/03-build-module.sh @@ -3,7 +3,6 @@ set -euo pipefail target="${1:-}" repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" validate_target "$target" "usage: $0 " @@ -17,16 +16,31 @@ if [[ ! -f "$kernel_file" || ! -d "$kdir" ]]; then exit 1 fi +module_dir="$repo_root/module" +if [[ ! -e "$module_dir/Makefile" && ! -e "$module_dir/Kbuild" ]]; then + echo "no Makefile or Kbuild under $module_dir" >&2 + echo "drop your module project under module/ (see module/README.md)" >&2 + exit 1 +fi + kernel="$(<"$kernel_file")" intermediate_dir="$repo_root/build/intermediate/$target/$kernel" artifact_dir="$repo_root/build/artifacts/$target/$kernel" -echo "building hello.ko for $target kernel $kernel" +echo "building module under $module_dir for $target kernel $kernel" make -C "$repo_root" \ KDIR="$kdir" \ BUILD_ID="$target/$kernel" \ INTERMEDIATE_DIR="$intermediate_dir" \ ARTIFACT_DIR="$artifact_dir" \ + MODULE_DIR="$module_dir" \ clean modules -echo "built $artifact_dir/hello.ko" +mapfile -t kos < <(find "$artifact_dir" -maxdepth 1 -name '*.ko' -type f | sort) +if [[ ${#kos[@]} -eq 0 ]]; then + echo "build produced no .ko under $artifact_dir" >&2 + exit 1 +fi +for ko in "${kos[@]}"; do + echo "built $ko" +done diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index 55e0442..99ec5e2 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -34,7 +34,7 @@ debug_endpoint="$(target_require_cfg "$target" "$endpoint_key")" debug_delay_ms="${DEBUG_LOAD_DELAY_MS:-5000}" ssh_port="${ssh_port:-22}" -remote_dir="${remote_dir:-/tmp/small-ko}" +remote_dir="${remote_dir:-/tmp/kmod-debug-lab}" sudo_pass="${sudo_pass:-$ssh_pass}" sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" @@ -79,14 +79,24 @@ if [[ ! -f "$vmlinux" ]]; then fi kernel="$(<"$kernel_file")" -artifact="$repo_root/build/artifacts/$target/$kernel/hello.ko" +artifact_dir="$repo_root/build/artifacts/$target/$kernel" intermediate_dir="$repo_root/build/intermediate/$target/$kernel" +module_dir="$repo_root/module" -if [[ ! -f "$artifact" ]]; then - echo "missing $artifact" >&2 +mapfile -t artifacts < <(find "$artifact_dir" -maxdepth 1 -name '*.ko' -type f | sort) +if [[ ${#artifacts[@]} -eq 0 ]]; then + echo "no .ko under $artifact_dir" >&2 echo "run scripts/03-build-module.sh $target first" >&2 exit 1 fi +if [[ ${#artifacts[@]} -gt 1 ]]; then + echo "expected exactly one .ko under $artifact_dir; found:" >&2 + printf ' %s\n' "${artifacts[@]}" >&2 + echo "the lab assumes a single obj-m per build" >&2 + exit 1 +fi +artifact="${artifacts[0]}" +module_name="$(basename "$artifact" .ko)" gdb_dir="$repo_root/.gdb" mkdir -p "$gdb_dir" @@ -98,7 +108,7 @@ loader_script="$gdb_dir/$target-$debug_method-loader.sh" rm -f "$symbols_file" "$loader_log" "$loader_script" -remote_module="$remote_dir/hello.ko" +remote_module="$remote_dir/$module_name.ko" remote_sudo="$(remote_sudo_prefix)" # Kill any local gdb processes that still hold a TCP connection to the debug @@ -145,11 +155,8 @@ set disassemble-next-line on set tcp connect-timeout 60 set remotetimeout 60 set architecture i386:x86-64 -set substitute-path $intermediate_dir/src $repo_root/module/src -set substitute-path $intermediate_dir/include $repo_root/module/include -set substitute-path $intermediate_dir $repo_root/module -directory $repo_root/module/src -directory $repo_root/module/include +set substitute-path $intermediate_dir $module_dir +directory $module_dir symbol-file $vmlinux target remote $debug_endpoint source $symbols_file @@ -171,6 +178,7 @@ ln -sfn "../.kernel-cache/$target/vmlinux" "$gdb_dir/current-vmlinux" printf '#!/usr/bin/env bash\n' printf 'set -euo pipefail\n' printf 'target=%q\n' "$target" + printf 'module_name=%q\n' "$module_name" printf 'symbols_file=%q\n' "$symbols_file" printf 'artifact=%q\n' "$artifact" printf 'ssh_bin=%q\n' "$ssh_bin" @@ -198,10 +206,17 @@ if [[ -n "$ssh_pass" ]]; then fi remote_sudo="$(remote_sudo_prefix)" -echo "loading module on $target and discovering symbols" -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}rmmod hello >/dev/null 2>&1 || true" +echo "loading $module_name on $target and discovering symbols" +SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}rmmod '$module_name' >/dev/null 2>&1 || true" + +# debug_delay_ms is the lab's convention for "sleep this long in module_init so +# the host has time to attach GDB and discover sections". Modules that declare +# it as a module_param accept the kv argument; modules that don't will reject +# insmod with -EINVAL on the unknown parameter. We try with the parameter and +# fall back to a plain insmod, inside a single sh -c so the || chain runs +# inside one backgrounded process. SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "nohup ${remote_sudo}insmod '$remote_module' debug_delay_ms='$debug_delay_ms' > '$remote_dir/insmod.log' 2>&1 &" + "nohup ${remote_sudo}sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$remote_dir/insmod.log' 2>&1 &" tmp_sections="$(mktemp)" trap 'rm -f "$tmp_sections"' EXIT @@ -210,7 +225,7 @@ section_list=".text .data .bss .rodata .init.text .exit.text .text.unlikely .rod deadline=$((SECONDS + 30)) while ((SECONDS < deadline)); do if SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "${remote_sudo}sh -c 'for sec in $section_list; do path=/sys/module/hello/sections/\$sec; if [ -r \"\$path\" ]; then printf \"%s %s\n\" \"\$sec\" \"\$(cat \"\$path\")\"; fi; done'" \ + "${remote_sudo}sh -c 'for sec in $section_list; do path=/sys/module/$module_name/sections/\$sec; if [ -r \"\$path\" ]; then printf \"%s %s\n\" \"\$sec\" \"\$(cat \"\$path\")\"; fi; done'" \ > "$tmp_sections"; then text_addr="$(awk '$1 == ".text" { print $2 }' "$tmp_sections")" if [[ -n "$text_addr" ]]; then @@ -222,7 +237,7 @@ while ((SECONDS < deadline)); do done < "$tmp_sections" { echo "$cmd" - echo "echo loaded hello.ko module symbols for $target\\n" + echo "echo loaded $module_name module symbols for $target\\n" } > "$symbols_file" echo "wrote module symbols to $symbols_file" exit 0 @@ -231,7 +246,7 @@ while ((SECONDS < deadline)); do sleep 0.2 done -echo "failed to discover /sys/module/hello/sections/.text within 30s" +echo "failed to discover /sys/module/$module_name/sections/.text within 30s" exit 1 LOADER } > "$loader_script" @@ -239,4 +254,4 @@ chmod +x "$loader_script" "$loader_script" > "$loader_log" 2>&1 echo "generated $gdb_file" -echo "loaded module and wrote symbols; log: $loader_log" +echo "loaded $module_name and wrote symbols; log: $loader_log" From e63c774a0dfcb2886bd335166ec532d93e8e9a1f Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 12:53:17 +0300 Subject: [PATCH 08/13] download the kernel source and match symbols for debugger to show source --- README.md | 40 ++++++++++- scripts/01-provision-target.sh | 53 +++++++++++--- scripts/02-setup-host-build.sh | 115 +++++++++++++++++++++++++++++- scripts/04-deploy-debug-vscode.sh | 45 +++++++++++- 4 files changed, 237 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index adbdd6f..9301728 100644 --- a/README.md +++ b/README.md @@ -169,19 +169,28 @@ kgdboc=ttyS0,115200 sysrq_always_enabled=1 Reboot the target VM after provisioning. -For source debugging with full kernel symbols, provision with debug symbols: +For source debugging with full kernel symbols *and the ability to step into +kernel code* (not just disassembly), provision with debug symbols: ```bash ./scripts/01-provision-target.sh server qemu --debug-symbols ``` -Then sync the target kernel headers, build tree, and optional `vmlinux` into -the development host (script 02 is the same regardless of debug method): +`--debug-symbols` installs both the matching `vmlinux` debug image and the +`linux-source-` package, then extracts the source tarball in place under +`/usr/src/`. Script 02 then syncs both alongside the headers. + +Sync the target kernel headers, build tree, `vmlinux`, and kernel source +into the development host (script 02 is the same regardless of debug method): ```bash ./scripts/02-setup-host-build.sh server ``` +After sync, `.kernel-cache//source` is a symlink to the synced +kernel source tree (or absent if the source wasn't installed on the target). +Script 04 picks it up automatically. + ## Build In VS Code, press `Ctrl+Shift+B`. The default `Kernel: Build` task prompts @@ -226,6 +235,31 @@ loads the target `vmlinux`, adds module symbols from `/sys/module//sections/*`, and maps staged Kbuild paths back to the real files under `module/`. +## Stepping Into Kernel Code + +With both `vmlinux` debug symbols and the kernel source tree synced (i.e. +script 01 was run with `--debug-symbols` and script 02 has picked up the +extracted source under `/usr/src/linux-source-*`), GDB can show kernel +source on step-into instead of just disassembly. + +Script 04 discovers Ubuntu's build-time source prefix by asking `addr2line` +where `start_kernel` lives in `vmlinux` — the path is always +`/init/main.c`, so the prefix is whatever comes before the +known suffix. It then emits: + +```text +set substitute-path +directory +``` + +into the generated `.gdb` file, where `` is +`.kernel-cache//source`. GDB transparently remaps any DWARF source +references in `vmlinux` to your local copy. + +If the kernel source wasn't installed on the target, script 04 prints a +note and skips the substitute-path — module debugging still works, but +stepping into kernel functions shows disassembly. + ## Debug Endpoint Options GDB only needs an endpoint in this form: diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index 6935e8b..0e7895a 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -94,20 +94,51 @@ apt_retry install -y linux-headers-"$kernel" rsync } vmlinux="/usr/lib/debug/boot/vmlinux-${kernel}" -if [[ "$INSTALL_DEBUG_SYMBOLS" == "--debug-symbols" && ! -r "$vmlinux" ]]; then - apt_retry install -y ubuntu-dbgsym-keyring - cat </dev/null +if [[ "$INSTALL_DEBUG_SYMBOLS" == "--debug-symbols" ]]; then + if [[ ! -r "$vmlinux" ]]; then + apt_retry install -y ubuntu-dbgsym-keyring + cat </dev/null deb http://ddebs.ubuntu.com ${codename} main restricted universe multiverse deb http://ddebs.ubuntu.com ${codename}-updates main restricted universe multiverse EOF - sudo_run apt-get clean - apt_retry update - apt_retry install -y "linux-image-${kernel}-dbgsym" || - apt_retry install -y "linux-image-unsigned-${kernel}-dbgsym" || { - echo "failed to install debug symbols for $kernel" >&2 - echo "retry later if ddebs.ubuntu.com is returning 503" >&2 - exit 1 - } + sudo_run apt-get clean + apt_retry update + apt_retry install -y "linux-image-${kernel}-dbgsym" || + apt_retry install -y "linux-image-unsigned-${kernel}-dbgsym" || { + echo "failed to install debug symbols for $kernel" >&2 + echo "retry later if ddebs.ubuntu.com is returning 503" >&2 + exit 1 + } + fi + + # Kernel source. Ubuntu's linux-source- package drops a + # tarball under /usr/src/. We don't extract on the target — script 02 + # rsyncs the tarball back to the host and extracts there, so the target + # stays unmodified beyond the package install. With the source available + # locally, GDB can step into kernel functions instead of just + # disassembling them. + short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" + src_pkg="" + for candidate in "linux-source-${short_kver}" "linux-source"; do + [[ -z "$candidate" || "$candidate" == "linux-source-" ]] && continue + if apt_retry install -y "$candidate"; then + src_pkg="$candidate" + break + fi + done + if [[ -z "$src_pkg" ]]; then + echo "warning: could not install a linux-source package; step-into-kernel won't have source files" >&2 + else + # Confirm the tarball is somewhere on disk for script 02 to find. + # Ubuntu has shipped both /usr/src/linux-source-X.tar.bz2 and + # /usr/src/linux-source-X/linux-source-X.tar.bz2 layouts over time — + # we accept either. + if ls -1 /usr/src/linux-source-*.tar.* /usr/src/linux-source-*/linux-source-*.tar.* 2>/dev/null | head -1 >/dev/null; then + echo "kernel source tarball is in /usr/src on the target; script 02 will sync and extract on the host" + else + echo "warning: $src_pkg installed but no linux-source tarball found under /usr/src/" >&2 + fi + fi fi [[ -r "$vmlinux" ]] || echo "note: $vmlinux is missing; VS Code source debugging may need it" diff --git a/scripts/02-setup-host-build.sh b/scripts/02-setup-host-build.sh index 1d92dd7..8a7ea71 100755 --- a/scripts/02-setup-host-build.sh +++ b/scripts/02-setup-host-build.sh @@ -117,12 +117,34 @@ fi base="\${kernel%-generic}" base="\${base%-lowlatency}" -for d in "/usr/src/linux-headers-\$base" "/usr/src/linux-source-\$base"; do +for d in "/usr/src/linux-headers-\$base"; do [[ -d "\$d" ]] && printf '%s\n' "\$d" done REMOTE ) +# Kernel source is discovered separately because it can be either a tarball +# (a single file at /usr/src/linux-source-X.tar.bz2) or a directory (possibly +# containing the tarball inside it, as some Ubuntu releases ship it). +# Classify each match as \t in a single round-trip so the sync +# loop below can stay dumb. +mapfile -t remote_source_entries < <(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "bash -s" <<'REMOTE' +set -euo pipefail +shopt -s nullglob +seen=() +for p in /usr/src/linux-source-*.tar.* /usr/src/linux-source-*; do + [[ -e "$p" ]] || continue + dup= + for s in "${seen[@]}"; do [[ "$s" == "$p" ]] && dup=1 && break; done + [[ -n "$dup" ]] && continue + seen+=("$p") + if [[ -d "$p" ]]; then printf '%s\tdir\n' "$p" + elif [[ -f "$p" ]]; then printf '%s\tfile\n' "$p" + fi +done +REMOTE +) + sync_header_dir() { local remote_dir="$1" local name @@ -139,11 +161,41 @@ sync_header_dir() { "$ssh_target:$remote_dir/" "$dest/" } +sync_remote_path() { + # Sync either a remote dir (contents synced into $dest/) or a remote file + # (file synced as $dest). Used for kernel-source paths where the remote + # layout can be either. + local remote_path="$1" + local kind="$2" + local name + local dest + + name="$(basename "$remote_path")" + dest="$usr_src_dir/$name" + + if [[ "$kind" == "dir" ]]; then + echo "syncing $remote_path/ -> $dest/" + SSHPASS="$ssh_pass" "$rsync_bin" -a --delete -e "$rsync_rsh" \ + "$ssh_target:$remote_path/" "$dest/" + else + echo "syncing $remote_path -> $dest" + SSHPASS="$ssh_pass" "$rsync_bin" -a -e "$rsync_rsh" \ + "$ssh_target:$remote_path" "$dest" + fi +} + printf '%s\n' "${remote_header_dirs[@]}" | awk 'NF && !seen[$0]++' | while IFS= read -r remote_dir; do sync_header_dir "$remote_dir" done +for entry in "${remote_source_entries[@]}"; do + [[ -z "$entry" ]] && continue + path="${entry%$'\t'*}" + kind="${entry##*$'\t'}" + sync_remote_path "$path" "$kind" +done + build_name="$(basename "$build_real")" rm -rf "$build_dir" ln -s "usr-src/$build_name" "$build_dir" @@ -153,6 +205,67 @@ if [[ ! -f "$build_dir/Makefile" ]]; then exit 1 fi +# Kernel source: find a usable source root under usr-src/ and point the +# stable `source` symlink at it. Ubuntu has shipped three layouts: +# 1. /usr/src/linux-source-X.tar.bz2 (tarball, no enclosing dir) +# 2. /usr/src/linux-source-X/ (pre-extracted, files at top) +# 3. /usr/src/linux-source-X/linux-source-X/ (pre-extracted, nested one deep) +# Layout 1 also occurs nested as /usr/src/linux-source-X/linux-source-X.tar.bz2. +# +# Strategy: look for a Makefile (the kernel's top-level marker) at depth ≤ 3 +# under usr-src/. If found, that's our source root — symlink directly to it, +# no extraction needed. Otherwise look for a tarball and extract on the host +# into $cache_dir/source-tree/. +# +# Extraction happens on the host, not the target, for two reasons: it avoids +# sudo + tar on the target for what's purely a host-side debug resource, and +# it's self-healing — re-running 02 fixes broken/partial trees without +# needing to touch the target. +source_link="$cache_dir/source" +source_tree="$cache_dir/source-tree" +rm -f "$source_link" + +is_kernel_source_root() { + [[ -f "$1/Makefile" && -d "$1/init" && -f "$1/init/main.c" ]] +} + +found_root="" +while IFS= read -r mf; do + candidate="$(dirname "$mf")" + if is_kernel_source_root "$candidate"; then + found_root="$candidate" + break + fi +done < <(find "$usr_src_dir" -mindepth 1 -maxdepth 3 -name Makefile -path '*linux-source-*' 2>/dev/null | sort) + +if [[ -n "$found_root" ]]; then + rm -rf "$source_tree" + rel="${found_root#$cache_dir/}" + ln -s "$rel" "$source_link" + echo "kernel source: $source_link -> $rel" +else + shopt -s nullglob + tarballs=( "$usr_src_dir"/linux-source-*.tar.* "$usr_src_dir"/linux-source-*/linux-source-*.tar.* ) + shopt -u nullglob + if [[ ${#tarballs[@]} -gt 0 ]]; then + tarball="${tarballs[0]}" + if is_kernel_source_root "$source_tree"; then + echo "kernel source already extracted at $source_tree" + else + echo "extracting $tarball -> $source_tree" + rm -rf "$source_tree" + mkdir -p "$source_tree" + tar -C "$source_tree" --strip-components=1 -xf "$tarball" + fi + ln -s source-tree "$source_link" + echo "kernel source: $source_link -> source-tree (extracted from $(basename "$tarball"))" + else + rm -rf "$source_tree" + echo "no kernel source tarball or extracted tree on target; step-into-kernel will only show disassembly" + echo "run scripts/01-provision-target.sh $target --debug-symbols to install it" + fi +fi + echo "checking vmlinux debug image" remote_sudo="$(remote_sudo_prefix)" if SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}test -r '$remote_vmlinux'"; then diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index 99ec5e2..26f879c 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -144,7 +144,45 @@ echo "uploading $artifact to $ssh_target:$remote_module" SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "mkdir -p '$remote_dir'" SSHPASS="$ssh_pass" "${scp_cmd[@]}" "$artifact" "$ssh_target:$remote_module" -cat > "$gdb_file" </dev/null 2>&1 && command -v addr2line >/dev/null 2>&1; then + # `|| true` because pipefail + SIGPIPE: nm dumps every symbol in + # vmlinux (millions), awk's `exit` after the first match closes the + # pipe, nm gets SIGPIPE on its next write, and the pipeline exits + # 141. We still capture the address awk printed before exiting. + sym_addr="$(nm "$vmlinux" 2>/dev/null | awk 'NF==3 && $3=="start_kernel" {print $1; exit}' || true)" + if [[ -n "$sym_addr" ]]; then + sym_loc="$(addr2line -e "$vmlinux" "$sym_addr" 2>/dev/null | head -1 | cut -d: -f1 || true)" + # Expected shape: /init/main.c + build_prefix="${sym_loc%/init/main.c}" + if [[ -n "$build_prefix" && "$build_prefix" != "$sym_loc" ]]; then + kernel_substitute_line="set substitute-path $build_prefix $kernel_src_root" + echo "kernel source mapping: $build_prefix -> $kernel_src_root" + else + echo "warning: addr2line returned unexpected location for start_kernel: $sym_loc" + fi + else + echo "warning: could not find start_kernel in vmlinux; skipping kernel source remap" + fi + fi +else + echo "note: $kernel_src_link missing; GDB will show kernel disassembly instead of source" + echo " run scripts/01 with --debug-symbols then re-run scripts/02 to enable step-into-kernel" +fi + +{ + cat < "$gdb_file" # Trimmed-down variant for IDEs that handle the gdb-side attach themselves # (Native Debug, cppdbg with miDebuggerServerAddress). The IDE has already From 6c1a7dbc063e4e42edeada4625f91423c281b148 Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 14:49:58 +0300 Subject: [PATCH 09/13] cleaned up scripts, and make ready for prod --- .editorconfig | 28 ++ .github/workflows/lint.yml | 50 +++ .gitignore | 17 +- .vscode/c_cpp_properties.json | 5 + .vscode/extensions.json | 4 +- .vscode/launch.json | 8 +- .vscode/settings.json | 37 +- CONTRIBUTING.md | 83 +++++ LICENSE | 29 ++ Makefile | 47 ++- README.md | 548 +++++++++++++----------------- lab.example.env | 27 +- module/README.md | 89 ++--- scripts/01-provision-target.sh | 135 ++++---- scripts/02-setup-host-build.sh | 229 ++++++------- scripts/03-build-module.sh | 30 +- scripts/04-deploy-debug-vscode.sh | 372 ++++++++++---------- scripts/lib/common.sh | 137 +++++++- 18 files changed, 1045 insertions(+), 830 deletions(-) create mode 100644 .editorconfig create mode 100644 .github/workflows/lint.yml create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b9dbf81 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,28 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true + +[*.{sh,bash}] +indent_style = tab + +[Makefile] +indent_style = tab + +[Kbuild] +indent_style = tab + +[*.{json,md,yml,yaml}] +indent_style = space +indent_size = 2 + +[*.{c,h}] +indent_style = tab +indent_size = 8 + +[*.ps1] +indent_style = space +indent_size = 4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..32bf870 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,50 @@ +name: lint + +on: + push: + branches: [main] + pull_request: + +jobs: + shellcheck: + name: shellcheck + bash -n + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + + - name: bash -n (syntax check) + run: | + for f in scripts/lib/*.sh scripts/0*.sh; do + echo "::group::bash -n $f" + bash -n "$f" + echo "::endgroup::" + done + + - name: shellcheck + run: shellcheck -x scripts/lib/common.sh scripts/0*.sh + + json: + name: validate JSON + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: validate .vscode/*.json + run: | + for f in .vscode/*.json; do + echo "::group::python -m json.tool $f" + python3 -m json.tool "$f" > /dev/null + echo "::endgroup::" + done + + makefile: + name: Makefile syntax + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: make help (dry-parse) + run: make help diff --git a/.gitignore b/.gitignore index b7f1afc..16fdcb6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,16 @@ +# Per-machine config (lab.example.env is the tracked template). lab.local.env +# Per-target synced kernel headers, source, vmlinux. .kernel-cache/ + +# Generated GDB init files (regenerated by scripts/04 on every F5). .gdb/ + +# Per-target build intermediates and final .ko artifacts. build/ +# Kbuild stragglers in case anything runs out of place. *.ko *.mod *.mod.c @@ -12,10 +19,16 @@ build/ Module.symvers modules.order +# IDE / editor noise. compile_commands.json .vscode/ipch/ .codex -kernel.gdb +.claude/ + +# Agent / session local notes (CLAUDE.md is consumed by Claude Code; keep it +# out of the public repo so workflow guidance stays separate from user docs). CLAUDE.md session.md -.claude/ + +# Stray scratch. +kernel.gdb diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 9065b7f..248ab5d 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -41,5 +41,10 @@ "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler_types.h" ] } + ], + "_notes": [ + "KBUILD_MODNAME is a placeholder; the actual value at compile time is your module's real name. IntelliSense only needs the macro to resolve, not match exactly.", + "ubuntu/include is Ubuntu-kernel-specific (where the distro's extra headers live); on non-Ubuntu kernels VS Code silently ignores it.", + "All paths under .kernel-cache/current/ resolve only after scripts/02-setup-host-build.sh has run. Before that, expect include squiggles." ] } diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 9a91c37..97d3205 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,6 +1,6 @@ { "recommendations": [ - "ms-vscode.cpptools" + "ms-vscode.cpptools", + "webfreak.debug" ] } - diff --git a/.vscode/launch.json b/.vscode/launch.json index abfbc34..cb0c619 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,13 +4,13 @@ { "id": "debugEndpoint", "type": "promptString", - "description": "GDB remote endpoint (host:port). Used only by the Native Debug launch; cppdbg reads it from the generated .gdb script.", + "description": "GDB remote endpoint (host:port). Only used by Native Debug — cppdbg reads it from the generated .gdb script.", "default": "127.0.0.1:1234" } ], "configurations": [ { - "name": "Kernel: Cppdbg Debug", + "name": "Kernel: cppdbg (full GDB MI)", "type": "cppdbg", "request": "launch", "program": "${workspaceFolder}/.gdb/current-vmlinux", @@ -21,7 +21,7 @@ "preLaunchTask": "Kernel: Deploy Debug", "setupCommands": [ { - "description": "Attach GDB to the selected target", + "description": "Source the GDB init script generated by scripts/04", "text": "source ${workspaceFolder}/.gdb/current-debug.gdb", "ignoreFailures": false } @@ -30,7 +30,7 @@ "externalConsole": false }, { - "name": "Kernel: Native Debug", + "name": "Kernel: Native Debug (faster, fewer features)", "type": "gdb", "request": "attach", "executable": "${workspaceFolder}/.gdb/current-vmlinux", diff --git a/.vscode/settings.json b/.vscode/settings.json index 1d56719..9503432 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,38 +1,15 @@ { "C_Cpp.default.configurationProvider": "", "C_Cpp.default.configurationName": "Linux kernel module - current", - "C_Cpp.default.compileCommands": "", - "C_Cpp.default.compilerPath": "/usr/bin/gcc", - "C_Cpp.default.cStandard": "gnu11", - "C_Cpp.default.intelliSenseMode": "linux-gcc-x64", - "C_Cpp.default.defines": [ - "__KERNEL__", - "MODULE", - "CC_USING_FENTRY", - "KBUILD_MODNAME=\"kmod\"", - "KBUILD_BASENAME=\"kmod\"", - "__KBUILD_MODNAME=kmod_kmod" - ], - "C_Cpp.default.includePath": [ - "${workspaceFolder}/module/**", - "${workspaceFolder}/.kernel-cache/current/build/include", - "${workspaceFolder}/.kernel-cache/current/build/include/uapi", - "${workspaceFolder}/.kernel-cache/current/build/include/generated", - "${workspaceFolder}/.kernel-cache/current/build/include/generated/uapi", - "${workspaceFolder}/.kernel-cache/current/build/ubuntu/include", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include/uapi", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include/generated", - "${workspaceFolder}/.kernel-cache/current/build/arch/x86/include/generated/uapi" - ], - "C_Cpp.default.forcedInclude": [ - "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler-version.h", - "${workspaceFolder}/.kernel-cache/current/build/include/linux/kconfig.h", - "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler_types.h" - ], "C_Cpp.errorSquiggles": "enabled", "files.associations": { "*.h": "c", - "*.c": "c" + "*.c": "c", + "Kbuild": "makefile" + }, + "files.exclude": { + ".kernel-cache": true, + ".gdb": true, + "build": true } } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e518573 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,83 @@ +# Contributing + +Bug reports, fixes, and improvements are welcome. This file describes what +that looks like for this repo. + +## What this repo is (and isn't) + +This is a **template** for building and source-debugging out-of-tree Linux +kernel modules. The lab tooling lives in `Makefile`, `scripts/`, +`.vscode/`, `host/`, and the docs. The example module under +`examples/chuck_norise/` exists to demonstrate the workflow end-to-end. + +PRs that **belong** here: + +- Bugs in the lab tooling — broken scripts, wrong assumptions about target + state, scripts that fail on re-run, unclear error messages. +- Support for a new target OS family (currently only Ubuntu). See the + `TARGET_OS` checks in scripts 01 and 02 for where to add a backend. +- New debug methods that fit the ` -> GDB` shape (anything + speaking the GDB remote serial protocol). +- Documentation that fixes misleading or missing information. +- Quality-of-life improvements: better error messages, more idempotent + re-runs, faster sync, smaller artifact dirs. + +PRs that **do not** belong here: + +- Changes to `module/` (that's the user's slot; we keep it as a placeholder + with just a README). +- New example modules under `examples/`. We keep one example focused and + small so it stays useful as a walkthrough. +- Project-management features (work logs, task tracking, etc.). The lab + stays focused on build + debug. + +## Reporting bugs + +Open an issue with: + +- Your **dev host** (distro, version, kernel) and **target VM** (distro, + version, kernel, hypervisor). +- The debug method you tried (`kgdb` or `qemu`) and the endpoint you + pointed it at. +- The command you ran and the full output. For deploy failures, attach + `.gdb/--loader.log`. +- What you expected, what happened instead. + +## Submitting changes + +1. Fork and branch from `main`. +2. Make focused commits — one logical change per commit. The git history + should read top-to-bottom as a small set of intentional steps. +3. Keep scripts shellcheck-clean (`shellcheck -x scripts/lib/common.sh + scripts/0*.sh`) and bash-syntax-clean (`bash -n`). +4. Update docs in the same PR. If you touched a script, check `README.md` + and `module/README.md` for anything the change makes inaccurate. +5. Open a PR with a description that explains *why* the change is needed, + not just what it does. The diff already says what. + +## Style + +- **Shell:** tabs for indentation. `set -euo pipefail` at the top of every + script. Errors via the `die` helper. Use `lab_*` helpers from + `scripts/lib/common.sh` for SSH/SCP/rsync — do not call those binaries + directly from numbered scripts. +- **Makefile:** tabs for recipes. Explicit `.PHONY` declarations. +- **Markdown:** wrap prose at ~80 columns. Use fenced code blocks with + language tags. +- **JSON (VS Code config):** 2-space indent, trailing newline. + +The repo includes an `.editorconfig` that captures these — most editors +will apply it automatically. + +## Testing changes locally + +There is no automated test suite for the build/debug flow (it requires real +hardware-or-VM targets). Before sending a PR: + +- Run `bash -n scripts/0*.sh scripts/lib/*.sh` to catch syntax errors. +- If you have `shellcheck`, run it too: `shellcheck -x scripts/lib/common.sh + scripts/0*.sh`. +- Smoke-test the full chain against at least one target VM you have access + to: `01 --debug-symbols` → reboot → `02` → `03` → `04` → F5 in VS Code + → set breakpoints in both your module and a kernel function and confirm + both bind. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4db60c3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +MIT License + +Copyright (c) 2026 Dor Grosglik and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +--- + +This MIT license covers the lab tooling in this repository (Makefile, +scripts/, .vscode/, host/, docs). The example module under +examples/chuck_norise/ is also MIT-licensed for ease of reuse. Any kernel +module you place under module/ is yours — this lab's license does not impose +terms on your module's source. diff --git a/Makefile b/Makefile index 5893e0f..3077963 100644 --- a/Makefile +++ b/Makefile @@ -29,20 +29,41 @@ INTERMEDIATE_DIR ?= $(BUILD_ROOT)/intermediate/$(BUILD_ID) ARTIFACT_DIR ?= $(BUILD_ROOT)/artifacts/$(BUILD_ID) MODULE_DIR ?= $(CURDIR)/module -EXTRA_CCFLAGS ?= -g -DDEBUG +EXTRA_CCFLAGS ?= -g -DDEBUG KCFLAGS_INJECT := -ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR) $(EXTRA_CCFLAGS) -.PHONY: all modules prepare-build clean +.PHONY: all modules prepare-build clean help all: modules +help: + @echo "Top-level lab Makefile." + @echo "" + @echo "Normally invoked by scripts/03-build-module.sh , which sets" + @echo "KDIR, BUILD_ID, INTERMEDIATE_DIR, and ARTIFACT_DIR to per-target paths." + @echo "" + @echo "Targets:" + @echo " modules build module/ against KDIR (default)" + @echo " clean remove intermediate and artifact dirs for BUILD_ID" + @echo " prepare-build stage module/ into INTERMEDIATE_DIR (internal)" + @echo "" + @echo "Vars (auto-resolved when .kernel-cache/current points at a target):" + @echo " MODULE_DIR $(MODULE_DIR)" + @echo " KDIR $(KDIR)" + @echo " BUILD_ID $(BUILD_ID)" + @echo " INTERMEDIATE_DIR $(INTERMEDIATE_DIR)" + @echo " ARTIFACT_DIR $(ARTIFACT_DIR)" + @echo " EXTRA_CCFLAGS $(EXTRA_CCFLAGS) (appended to KCFLAGS)" + modules: prepare-build $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" KCFLAGS="$(KCFLAGS_INJECT)" modules @mkdir -p "$(ARTIFACT_DIR)" @set -e; \ kos=$$(find "$(INTERMEDIATE_DIR)" -maxdepth 2 -name '*.ko' -type f); \ if [ -z "$$kos" ]; then \ - echo "no .ko produced under $(INTERMEDIATE_DIR)" >&2; exit 1; \ + echo "Makefile: no .ko produced under $(INTERMEDIATE_DIR);" >&2; \ + echo " check the Kbuild output above for compile errors." >&2; \ + exit 1; \ fi; \ for ko in $$kos; do \ name=$$(basename "$$ko"); \ @@ -50,17 +71,23 @@ modules: prepare-build echo "built $(ARTIFACT_DIR)/$$name"; \ done -# Mirror module/ into INTERMEDIATE_DIR as a tree of absolute symlinks so -# Kbuild's M= sees the user's sources in a writeable scratch dir without -# polluting module/. `cp -as` is GNU coreutils; it recursively creates dirs -# and symlinks each file. Re-running is safe because we wipe the symlink -# scaffolding first. +# Mirror $(MODULE_DIR) into $(INTERMEDIATE_DIR) as a tree of absolute +# symlinks so Kbuild's `M=` sees the user's sources in a writeable scratch +# dir without polluting module/. `cp -as` is GNU coreutils; it recursively +# creates dirs and symlinks each file. Re-running is idempotent because we +# wipe the symlink scaffolding first. prepare-build: @if [ ! -e "$(MODULE_DIR)/Makefile" ] && [ ! -e "$(MODULE_DIR)/Kbuild" ]; then \ - echo "no Makefile or Kbuild under $(MODULE_DIR)" >&2; \ - echo "see $(MODULE_DIR)/README.md for the expected layout" >&2; \ + echo "Makefile: no Makefile or Kbuild under $(MODULE_DIR);" >&2; \ + echo " drop your kernel module project there (see $(MODULE_DIR)/README.md)" >&2; \ + echo " or try the example: cp -r examples/chuck_norise/. module/" >&2; \ exit 1; \ fi + @cp --help 2>&1 | grep -q -- '--symbolic-link' || { \ + echo "Makefile: GNU 'cp' (with --symbolic-link, --archive) is required;" >&2; \ + echo " the lab targets Debian-based hosts. On macOS: brew install coreutils, then re-run with CP=gcp." >&2; \ + exit 1; \ + } @mkdir -p "$(INTERMEDIATE_DIR)" @find "$(INTERMEDIATE_DIR)" -depth -type l -delete @find "$(INTERMEDIATE_DIR)" -depth -type d -empty -not -path "$(INTERMEDIATE_DIR)" -delete diff --git a/README.md b/README.md index 9301728..1665d5e 100644 --- a/README.md +++ b/README.md @@ -1,296 +1,201 @@ -# Linux Kernel Module Debug Lab — Template +# Linux Kernel Module Debug Lab -A reusable development and source-debugging setup for out-of-tree Linux kernel -modules. The repo is generic: drop your module project under `module/`, point -the lab at a target VM, and source-debug from VS Code or `gdb -tui`. +A template for building and **source-debugging** out-of-tree Linux kernel +modules against one or more target VMs. Drop your module under `module/`, +point the lab at a target, press F5 — step through your code (and into +kernel code) in VS Code or `gdb -tui`. -A worked example lives under `examples/chuck_norise/` — a tiny multi-file -char device. Copy it into `module/` to run the full flow end-to-end without -writing any module code first. +What you get out of the box: -## What This Template Gives You +- **One slot, any module.** `module/` is your project; the lab is generic. +- **Per-target builds.** Cross-build the same source against multiple + target kernels without polluting your tree. +- **Breakpoints just work.** DWARF `-ffile-prefix-map` is injected for + you so IDE breakpoints by absolute path bind to your real source files. +- **Step into the kernel.** With one flag (`--debug-symbols`), the lab + pulls down `vmlinux` + kernel source and wires GDB `substitute-path` + so you can step from your module's read handler into `vfs_read`. +- **Two debug methods.** In-kernel KGDB over serial (`kgdb`) and any + hypervisor GDB stub (`qemu`, also VMware's `debugStub`). +- **One-command flow.** Four numbered scripts + a VS Code F5. -- A per-target, out-of-tree build that stages your module/ tree into - `build/intermediate///` and writes the final `.ko` under - `build/artifacts///`. -- DWARF path rewriting (`-ffile-prefix-map`) injected via `KCFLAGS`, so - breakpoints set in your IDE bind to the real files under `module/` and - not to the staged copies. -- Provisioning, header sync, build, deploy, load, and symbol discovery - driven from four numbered scripts plus the VS Code task/launch wiring - that calls them. -- Support for two debug methods against the same target — in-kernel KGDB - over serial (`kgdb`) and a hypervisor GDB stub (`qemu`, also covers - VMware's `debugStub.listen.guest64`). +The repo ships with a small worked example (`examples/chuck_norise/`) so +you can run the whole flow end-to-end before writing a line of module +code. -## Machine Model +--- -The lab uses two kinds of machines: +## Quickstart -1. A Debian-based development host. - This can be Debian, Ubuntu Desktop, Ubuntu under WSL, or another - Debian-based distro. The scripts assume `apt`/`apt-get`, `bash`, `ssh`, - `scp`, `rsync`, `make`, and standard GNU userland tools. - -2. A target VM. - The target is where the module is loaded and tested. Communication with - the target is over SSH. For now, the supported target OS is Ubuntu — add - a target OS backend in the scripts to support another distro. - -## The module/ Contract - -`module/` is the single slot for your module project. The lab is otherwise -generic — it does not know your module's name, source layout, or behavior. - -The build pipeline expects: - -- `module/Makefile` (or `module/Kbuild`) following standard out-of-tree - Kbuild conventions: - - ```makefile - obj-m += my_module.o - my_module-y := src/main.o src/util.o - ccflags-y := -I$(src)/include -g -DDEBUG - ``` - - See `examples/chuck_norise/Makefile` for the canonical shape, including - the optional `ifndef KERNELRELEASE` wrapper that also lets you run plain - `make` directly in `module/`. - -- Exactly one `obj-m` entry per build. The artifact name becomes - `.ko`. The lab discovers the module name from the produced `.ko`, - so you do not declare it anywhere else. - -- Optional: a `debug_delay_ms` module parameter that sleeps in `module_init` - before doing anything observable. The lab passes the value from - `DEBUG_LOAD_DELAY_MS` (default 5000 ms) to `insmod`. Modules without the - parameter are loaded plainly. - -You decide everything else: source layout, header layout, license, exported -symbols. - -## Try The Example Module +You need a **Debian-based dev host** (Debian, Ubuntu, WSL Ubuntu, …) and +at least one **Ubuntu target VM** reachable over SSH. Five steps from a +fresh clone to a stopped breakpoint: ```bash -cp -r examples/chuck_norise/. module/ -``` - -Then run the lab flow described below. - -## Debug Methods - -This repo supports two debug methods. Provisioning (script 01) and the -deploy step (script 04) both take a `` argument so you can switch -between them per run. - -- **`kgdb`** — in-kernel KGDB talking over the guest's serial port. The - provisioning step adds `kgdboc=ttyS*,115200` and `sysrq_always_enabled=1` - to the guest's kernel command line. To halt the running kernel into GDB, - run `echo g | sudo tee /proc/sysrq-trigger` on the target. Useful when - you cannot change the hypervisor's command line, e.g. VMware running as - your only option. +# 0. clone + configure +git clone kmod-debug-lab && cd kmod-debug-lab +cp lab.example.env lab.local.env +$EDITOR lab.local.env # set TARGET_SSH_*, endpoints, etc. -- **`qemu`** — QEMU's built-in gdbstub (`-gdb tcp::PORT` or `-s`). No KGDB - in the guest is required; the hypervisor halts the vCPU directly. - VMware's `debugStub.listen.guest64` is the same shape and slots into this - method too. Connecting GDB to the stub halts the vCPU; set breakpoints, - then `continue`. +# 1. one-time target setup (installs headers, vmlinux dbg, kernel source) +./scripts/01-provision-target.sh server qemu --debug-symbols +# ...reboot the target VM so new GRUB args take effect... -Neither method assumes the module calls `kgdb_breakpoint()` on insmod. In -both methods you break manually after the deploy step finishes. The lab's -convention is for the module to sleep `debug_delay_ms` (default 5000 ms) at -the start of init so you have a window to break before init runs and so the -host has time to read `/sys/module//sections/*`. +# 2. one-time host setup (syncs everything to .kernel-cache/server/) +./scripts/02-setup-host-build.sh server -## Configure The Lab +# 3. try the example module (replace with your own when ready) +cp -r examples/chuck_norise/. module/ +./scripts/03-build-module.sh server -Copy the example environment file and edit it for your machines: +# 4. deploy + load on the target +./scripts/04-deploy-debug-vscode.sh server qemu -```bash -cp lab.example.env lab.local.env +# 5. open the repo in VS Code and press F5 ("Kernel: cppdbg") +code . ``` -Targets are data, not variable prefixes. Add profile names to `TARGETS`, -then fill the `TARGET_*` maps with entries keyed by that profile name: +Set a breakpoint in `module/src/hello.c` and one in `vfs_read` — both +bind. You're stepping through kernel code from your module. -```bash -TARGETS=(desktop server) +--- -declare -A TARGET_OS=( - [desktop]=ubuntu - [server]=ubuntu -) +## How the pieces fit -declare -A TARGET_SSH_HOST=( - [desktop]=ubuntu-desktop.local - [server]=ubuntu-server.local -) ``` +module/ your module project (Makefile + sources) +examples/chuck_norise/ worked example you can copy into module/ -Target names may contain letters, numbers, `_`, or `-`. A target named -`ubuntu-server` is configured with map keys like `[ubuntu-server]=...`. - -Endpoints are split per debug method. Fill the one(s) you intend to use: +scripts/01-provision-target.sh target-side: install headers, vmlinux dbg, source, GRUB args +scripts/02-setup-host-build.sh dev-host: sync headers + vmlinux + kernel source into .kernel-cache/ +scripts/03-build-module.sh dev-host: build module/ via Kbuild against the synced headers +scripts/04-deploy-debug-vscode.sh dev-host: upload, insmod, write .gdb/-.gdb -```bash -declare -A TARGET_DEBUG_ENDPOINT_KGDB=( - [server]=127.0.0.1:5520 -) +.kernel-cache// per-target build/source/vmlinux cache (gitignored) +build/ per-target intermediate + final .ko (gitignored) +.gdb/ generated GDB init files (gitignored, regenerated on F5) -declare -A TARGET_DEBUG_ENDPOINT_QEMU=( - [server]=127.0.0.1:1234 -) +host/ optional helpers for VMware-on-Windows users +lab.example.env -> lab.local.env per-machine config (gitignored copy) ``` -Either map may be left empty per target. Script 04 errors clearly if the -endpoint for the method you picked is missing. - -The VS Code task pickers prompt for target name as free text, so adding a -target is just an edit to `lab.local.env` — no other edits required for -VS Code to pick it up. +The numbered scripts are designed to be run in order. After the one-time +01 + 02 against a target, the inner loop is **03 → 04 → F5** (or just F5, +which runs 04 as a prelaunch task and rebuilds via 03 if you ran it +manually first). -## Prepare A Target +--- -Provisioning is target-side setup. It currently supports Ubuntu targets and -takes the debug method as a second argument. +## The `module/` contract -```bash -./scripts/01-provision-target.sh server kgdb # KGDB path -./scripts/01-provision-target.sh server qemu # QEMU stub path -``` +`module/` is the lab's only slot for your kernel module project. The lab +does not know your module's name, source layout, or behavior; it expects +exactly: -Both modes install the running target kernel's headers and `rsync`, and add -`nokaslr` to the kernel command line so vmlinux symbols line up with running -addresses. `kgdb` mode additionally adds: +- **A standard Kbuild file** at `module/Makefile` (or `module/Kbuild`): -```text -kgdboc=ttyS0,115200 sysrq_always_enabled=1 -``` + ```makefile + obj-m += my_module.o + my_module-y := src/main.o src/util.o + ccflags-y := -I$(src)/include -g -DDEBUG + ``` -Reboot the target VM after provisioning. + See `examples/chuck_norise/Makefile` for the canonical shape, including + the optional `ifndef KERNELRELEASE` wrapper that lets you also run plain + `make` directly in `module/`. -For source debugging with full kernel symbols *and the ability to step into -kernel code* (not just disassembly), provision with debug symbols: +- **Exactly one `obj-m` entry per build.** The lab discovers the module + name from the produced `.ko` — you don't declare it anywhere + else. -```bash -./scripts/01-provision-target.sh server qemu --debug-symbols -``` +- **Optional:** a `debug_delay_ms` module parameter: -`--debug-symbols` installs both the matching `vmlinux` debug image and the -`linux-source-` package, then extracts the source tarball in place under -`/usr/src/`. Script 02 then syncs both alongside the headers. + ```c + static unsigned int debug_delay_ms = 5000; + module_param(debug_delay_ms, uint, 0644); + ``` -Sync the target kernel headers, build tree, `vmlinux`, and kernel source -into the development host (script 02 is the same regardless of debug method): + When present, scripts/04 passes the value from `DEBUG_LOAD_DELAY_MS` + (in `lab.local.env`, default 5000 ms) so your `module_init` sleeps + long enough for the host to read `/sys/module//sections/*` and + for you to break in GDB before init does anything. Modules without + the parameter would reject `insmod foo.ko debug_delay_ms=...` with + `-EINVAL`; the loader catches that and retries with a plain `insmod`, + so undeclared modules still load. -```bash -./scripts/02-setup-host-build.sh server -``` +Source layout, headers, license — those are yours. The lab passes +through whatever your Kbuild file declares. -After sync, `.kernel-cache//source` is a symlink to the synced -kernel source tree (or absent if the source wasn't installed on the target). -Script 04 picks it up automatically. +--- -## Build +## Configuring the lab -In VS Code, press `Ctrl+Shift+B`. The default `Kernel: Build` task prompts -for `kernelTarget` and runs: +Copy the example env and edit it for your machines: ```bash -./scripts/03-build-module.sh +cp lab.example.env lab.local.env ``` -The module is written to: +Targets are **data, not variable prefixes**. Add a profile name to +`TARGETS`, then add entries to the `TARGET_*` maps under that name: -```text -build/artifacts///.ko -``` +```bash +TARGETS=(desktop server) -Build intermediates are kept under: +declare -A TARGET_SSH_HOST=( + [desktop]=ubuntu-desktop.local + [server]=ubuntu-server.local +) +declare -A TARGET_SSH_USER=([desktop]=user [server]=user) +declare -A TARGET_SSH_PASS=([desktop]= [server]=) # empty when using SSH keys -```text -build/intermediate/// +declare -A TARGET_DEBUG_ENDPOINT_QEMU=( + [desktop]=127.0.0.1:1234 + [server]=127.0.0.1:1234 +) ``` -Running plain `make` at the repo root uses `.kernel-cache/current` when it -exists, otherwise the host kernel. The setup script updates that link to -the most recently synced target. - -## Debug From VS Code - -The VS Code debug tasks share two inputs: `kernelTarget` (free-text prompt) -and `debugMethod` (kgdb|qemu). - -1. Select `Kernel: Cppdbg Debug` (or `Kernel: Native Debug`). -2. Press F5. -3. VS Code runs the `Kernel: Deploy Debug` prelaunch task. -4. The task prompts for `kernelTarget` and `debugMethod`. -5. `scripts/04-deploy-debug-vscode.sh ` uploads and loads - your module, prepares module symbols, and writes the current GDB files - under `.gdb/`. -6. VS Code starts GDB and sources `.gdb/current-debug.gdb`. - -The generated GDB script sets the kernel architecture to `i386:x86-64`, -loads the target `vmlinux`, adds module symbols from -`/sys/module//sections/*`, and maps staged Kbuild paths back to the -real files under `module/`. - -## Stepping Into Kernel Code +Endpoints are split per debug method +(`TARGET_DEBUG_ENDPOINT_KGDB` and `TARGET_DEBUG_ENDPOINT_QEMU`). Either +map may be left empty per target if you don't use that method. Script 04 +errors clearly if you pick a method without an endpoint. -With both `vmlinux` debug symbols and the kernel source tree synced (i.e. -script 01 was run with `--debug-symbols` and script 02 has picked up the -extracted source under `/usr/src/linux-source-*`), GDB can show kernel -source on step-into instead of just disassembly. +Target names accept `[A-Za-z][A-Za-z0-9_-]*`. The VS Code task picker +accepts any name you type — no editing of `.vscode/` needed when adding +a target. -Script 04 discovers Ubuntu's build-time source prefix by asking `addr2line` -where `start_kernel` lives in `vmlinux` — the path is always -`/init/main.c`, so the prefix is whatever comes before the -known suffix. It then emits: +--- -```text -set substitute-path -directory -``` +## Debug methods -into the generated `.gdb` file, where `` is -`.kernel-cache//source`. GDB transparently remaps any DWARF source -references in `vmlinux` to your local copy. +Provisioning (script 01) and deploy (script 04) both take a +`` argument: -If the kernel source wasn't installed on the target, script 04 prints a -note and skips the substitute-path — module debugging still works, but -stepping into kernel functions shows disassembly. +- **`kgdb`** — in-kernel KGDB over the guest's serial port. Script 01 + adds `kgdboc=ttyS0,115200` and `sysrq_always_enabled=1` to the guest's + GRUB command line. Break with `echo g | sudo tee /proc/sysrq-trigger` + on the target. Useful when you cannot change the hypervisor's command + line (VMware Workstation is the common case). -## Debug Endpoint Options - -GDB only needs an endpoint in this form: - -```text -: -``` +- **`qemu`** — the hypervisor's built-in GDB stub + (QEMU's `-gdb tcp::PORT` / `-s`, VMware's `debugStub.listen.guest64`). + No KGDB in the guest; the hypervisor halts the vCPU directly. -Put that value in `TARGET_DEBUG_ENDPOINT_KGDB` or `TARGET_DEBUG_ENDPOINT_QEMU` -in `lab.local.env`, depending on the method you want to use. +Neither method assumes any in-module `kgdb_breakpoint()` call. In both, +you break manually after script 04 finishes. ### QEMU gdbstub -Launch QEMU with the built-in stub exposed on a TCP port, e.g.: - ```text qemu-system-x86_64 ... -gdb tcp::1234 ``` -(Use `-S` if you want the vCPU paused at boot. Without `-S`, the guest runs -until GDB connects and halts it.) - -```bash -declare -A TARGET_DEBUG_ENDPOINT_QEMU=( - [server]=127.0.0.1:1234 -) -``` +Use `-S` to pause the vCPU at boot. Without `-S`, the guest runs until +GDB connects. -### VMware Debug Stub +### VMware debug stub -Power off the VM, open the target VM's `.vmx` file, and add: +Power off the VM, edit the VM's `.vmx`: ```text debugStub.listen.guest64 = "TRUE" @@ -299,139 +204,142 @@ debugStub.listen.guest64.remote = "TRUE" debugStub.hideBreakpoints = "FALSE" ``` -Start the VM after saving the `.vmx` file. The VMware stub is the same shape -as QEMU's gdbstub — use the `qemu` debug method: - -```bash -declare -A TARGET_DEBUG_ENDPOINT_QEMU=( - [server]=127.0.0.1:8864 -) -``` - -### KGDB Over Serial → TCP - -Use this path if you want Linux KGDB over a virtual serial port: +Same shape as the QEMU stub — set +`TARGET_DEBUG_ENDPOINT_QEMU[server]=127.0.0.1:8864`. -```text -VS Code -> GDB -> TCP port -> serial bridge -> guest /dev/ttyS0 -> Ubuntu KGDB -``` +### KGDB over serial → TCP -How you build the bridge depends on the hypervisor: +Set up a serial bridge from the guest's `/dev/ttyS0` to a TCP listener +GDB can connect to. How depends on the hypervisor: - **QEMU:** `-serial tcp:127.0.0.1:5520,server,nowait` -- **VMware Workstation:** named pipe + `host/bridge-kgdb.ps1`. Configure a - serial port with `Use named pipe`, `This end is the server`, `The other - end is an application`, `Connect at power on`. Then run the bridge - script on the Windows host: +- **VMware Workstation on Windows:** named pipe in the VM's serial port + + `host/bridge-kgdb.ps1`: ```powershell .\host\bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520 ``` -Provision the target with the `kgdb` method and set the matching endpoint: +Provision the target with `kgdb`: ```bash ./scripts/01-provision-target.sh server kgdb +``` -declare -A TARGET_DEBUG_ENDPOINT_KGDB=( - [server]=127.0.0.1:5520 -) +`TARGET_KGDB_TTY` and `TARGET_KGDB_BAUD` in `lab.local.env` must match +the GRUB args added by provisioning (defaults `ttyS0` / `115200`). -declare -A TARGET_KGDB_TTY=( - [server]=ttyS0 -) +Useful sanity check before F5: -declare -A TARGET_KGDB_BAUD=( - [server]=115200 -) +```bash +nc -vz 127.0.0.1 5520 ``` -The TTY and baud rate must match the boot arguments added by provisioning. +--- -## Verify The Debug Endpoint +## Stepping into kernel code -From the development host, check the endpoint before launching GDB: +Pass `--debug-symbols` to script 01 to install both the matching +`vmlinux` debug image and the `linux-source-X` package on the target. +Script 02 syncs both, extracts the source on the host, and creates a +stable `.kernel-cache//source` symlink. -```bash -nc -vz 127.0.0.1 1234 +Script 04 then asks `addr2line` where `start_kernel` lives in `vmlinux` +(always `/init/main.c`) and emits the corresponding GDB +remap into the generated `.gdb` file: + +```text +set substitute-path +directory ``` -If the hypervisor host can connect but the development host cannot, use an -address for the hypervisor host that the development host can reach instead -of `127.0.0.1`. On WSL, the Windows host address is often listed as the -resolver: +That's it — set a breakpoint in `vfs_read` and step through it. -```bash -grep nameserver /etc/resolv.conf +This step gracefully degrades. If `vmlinux` is missing, script 04 errors +clearly. If the kernel source wasn't synced, script 04 prints a one-line +note and continues — module debugging still works; you just see +disassembly when stepping into kernel functions. Check +`.gdb/--loader.log` for details when something is off. + +--- + +## Build outputs + +``` +build/intermediate/// staged Kbuild tree (recursive symlinks back to module/) +build/artifacts///.ko final module +.kernel-cache//build -> synced linux-headers +.kernel-cache//source -> synced linux-source (when --debug-symbols) +.kernel-cache//vmlinux debug-symbol vmlinux from the target +.kernel-cache/current -> drives bare `make` and IntelliSense +.gdb/-.gdb full GDB init: arch, vmlinux, target remote, sourced symbols +.gdb/--attached.gdb trimmed variant for IDEs that already attach themselves +.gdb/--loader.log output of remote insmod + section discovery +.gdb/current-debug{.gdb,-attached.gdb} symlinks the VS Code launch configs read (re-pointed each F5) ``` -After changing `lab.local.env`, run F5 again so script 04 regenerates the -GDB files under `.gdb/`. +`build/`, `.kernel-cache/`, `.gdb/`, `lab.local.env`, and +`compile_commands.json` are gitignored. + +--- ## Troubleshooting -If a script says `sshpass` is missing and you use SSH passwords, install it -on the development host: +**`sshpass: command not found`.** You configured `TARGET_SSH_PASS`. On +the dev host: `sudo apt-get install -y sshpass`. Leave +`TARGET_SUDO_PASS` empty when the sudo password matches the SSH +password. + +**F5 fails with `target remote ... Connection timed out`.** The endpoint +in `.gdb/current-debug.gdb` isn't reachable from the dev host. Verify +with `nc -vz `. On WSL, the Windows host address is often +in `/etc/resolv.conf`: ```bash -sudo apt-get install -y sshpass +grep nameserver /etc/resolv.conf ``` -Leave `TARGET_SUDO_PASS[target]` empty when the sudo password is the same -as the SSH password. - -If F5 fails with `target remote ... Connection timed out`, the endpoint in -`.gdb/current-debug.gdb` is not reachable from the development host. Check -`TARGET_DEBUG_ENDPOINT_KGDB[target]` or `TARGET_DEBUG_ENDPOINT_QEMU[target]` -(depending on the method you picked) and verify that your endpoint provider -is listening. +After editing `lab.local.env`, re-run F5 so script 04 regenerates the +`.gdb/` files. -If GDB connects but the module does not load, inspect: +**GDB connects but the module never loads.** Check +`.gdb/--loader.log` for the insmod error. You can also +SSH in and try the insmod by hand: ```bash -cat .gdb/--loader.log +ssh sudo insmod /tmp/kmod-debug-lab/.ko +ssh sudo dmesg | tail -40 ``` -If breakpoints bind to files under `build/intermediate/...`, regenerate the -GDB files by pressing F5 again. The generated script maps those staged paths -back to `module/` with `set substitute-path`. +**Breakpoints bind to files under `build/intermediate/...`.** Re-run F5 +so `.gdb/current-debug.gdb` is regenerated with the correct +`substitute-path`. Don't edit `.gdb/` by hand — it's regenerated every +F5. -## VS Code Include Errors +**VS Code shows include squiggles for `linux/module.h`.** Script 02 +hasn't synced the headers yet, or you switched targets and IntelliSense +is caching old paths. Re-run `scripts/02-setup-host-build.sh ` +and `C/C++: Reset IntelliSense Database` from the command palette. -The C/C++ extension reads `.vscode/c_cpp_properties.json`. It expects target -kernel headers under `.kernel-cache//build`, which are created by: +**`/sys/module//sections/.text` didn't appear within 30s.** Your +module either failed to load, or it's slow to init on the target. The +loader log will show `insmod` output. Bump the timeout with +`INSMOD_WAIT_SECS=60 ./scripts/04-... ...` if needed. -```bash -./scripts/02-setup-host-build.sh -``` +--- -Before that sync runs, VS Code can show include squiggles for kernel -headers such as `linux/module.h` or `linux/fs.h`. If the squiggles remain -after syncing headers, run `C/C++: Reset IntelliSense Database` from the -command palette. +## Adding a new target OS -`KBUILD_MODNAME` is set to a placeholder (`"kmod"`) in the IntelliSense -defines. The actual value at compile time is your real module name; the -placeholder is only there so the C/C++ extension can resolve macros that -reference it. +Currently only `TARGET_OS[*]=ubuntu` is implemented in scripts 01 and 02. +Adding (say) Fedora means teaching script 01 how to install headers / +dbgsym / source via `dnf`, and teaching script 02 how to discover the +header tree paths under `/usr/src/`. Both scripts check `target_os` near +the top and fail fast on unknown values — that's where to add a backend. +PRs welcome. -## Repository Layout +--- -```text -module/ # your module project lives here -examples/chuck_norise/ # worked example you can copy into module/ -scripts/01-provision-target.sh # target-side: headers, grub args -scripts/02-setup-host-build.sh # host-side: sync headers + vmlinux -scripts/03-build-module.sh # host-side: build via Kbuild -scripts/04-deploy-debug-vscode.sh # host-side: upload, insmod, gen .gdb -scripts/lib/common.sh # shared bash helpers -host/bridge-kgdb.ps1 # Windows-side KGDB serial-to-TCP bridge -host/restore-snapshot.ps1 # Windows-side VMware snapshot helper -Makefile # generic out-of-tree wrapper -lab.example.env # copy to lab.local.env and edit -.vscode/ # tasks, launch, IntelliSense -``` +## License -The top-level scripts and `Makefile` are the lab's tooling — generic across -modules. The optional PowerShell scripts under `host/` are helpers for -VMware Workstation on Windows. +MIT — see [LICENSE](LICENSE). Your module under `module/` is yours; the +lab's license does not impose terms on what you build. diff --git a/lab.example.env b/lab.example.env index aaa62da..23c7be6 100644 --- a/lab.example.env +++ b/lab.example.env @@ -1,7 +1,9 @@ # Copy this file to lab.local.env and edit it for your machines. +# lab.local.env is gitignored; lab.example.env is the only tracked copy. -# Common local tools on the Debian-based development host. -GDB_BIN=gdb +# Local tools on the Debian-based development host. The defaults below +# expect each to be on $PATH; override here only if you need an absolute +# path or an alternate binary. SSH_BIN=ssh SCP_BIN=scp RSYNC_BIN=rsync @@ -89,9 +91,20 @@ declare -A TARGET_KGDB_BAUD=( [server]=115200 ) -# Module sleeps this long at the start of init, IF it exposes the -# `debug_delay_ms` module parameter. The lab uses this convention so the host -# can read /sys/module//sections/* and so users have a window to break -# in GDB (Ctrl+C against the qemu stub, sysrq+g for kgdb) before init runs. -# Modules that do not declare the parameter get loaded without it; no harm. +# When your module declares `module_param(debug_delay_ms, uint, 0644);`, +# scripts/04 passes this value (in milliseconds) on `insmod`. Your +# `module_init` should sleep that long so the host can read +# /sys/module//sections/* and you have a window to break in GDB +# (Ctrl+C against the qemu stub, sysrq+g for kgdb) before init runs. +# Modules that do not declare the parameter get loaded plainly; no harm. DEBUG_LOAD_DELAY_MS=5000 + +# Optional. Sets `maxcpus=` on the target's GRUB command line — useful +# when you want to pin debugging to a known CPU count (e.g. 1 to avoid +# CPU migration confusing breakpoints). Unset to leave it off. +#DEBUG_MAXCPUS=1 + +# Optional. Override how long scripts/04 polls for the loaded module's +# section addresses to appear under /sys/module//sections/. +# Increase on slow target VMs. Default 30. +#INSMOD_WAIT_SECS=30 diff --git a/module/README.md b/module/README.md index 0576a73..fcca4d9 100644 --- a/module/README.md +++ b/module/README.md @@ -1,52 +1,61 @@ -# chuck_norise — example module +# module/ — your kernel module goes here -A small multi-file char device module used as the worked example for this lab. -Builds as `chuck_norise.ko` and exposes `/dev/chuck_norise`. Reads return the -exact string `chuck norise!` repeated indefinitely, preserving the file offset -for each open file descriptor. +This directory is the lab's only slot for the kernel module being built and +debugged. The repo is otherwise generic — it does not know your module's +name, source layout, or what it does. -It is intentionally tiny so it stays useful as a template walk-through: an init -that sleeps long enough to give you time to attach GDB, a cdev/class lifecycle -split into its own file, and a separately-testable "produce bytes into a -userspace buffer" routine. +## Contract -## Use it as a template +Drop a standard out-of-tree Linux kernel module project here. The lab +expects: -The repo's build pipeline operates on whatever lives under `module/` at the -repo root. To try this example end-to-end, copy it into place: +- **`module/Makefile`** (or `module/Kbuild`) following standard Kbuild + conventions for out-of-tree modules: -```bash -cp -r examples/chuck_norise/. module/ -``` + ```makefile + obj-m += my_module.o + my_module-y := src/main.o src/util.o + ccflags-y := -I$(src)/include -g -DDEBUG + ``` -Then run the normal lab flow from the repo root: `scripts/01-...` through -`scripts/04-...`, or press F5 in VS Code. After the module loads on the -target VM: + See `examples/chuck_norise/Makefile` for the canonical shape, including + the optional `ifndef KERNELRELEASE` wrapper that lets the same file be + driven by `make` directly. -```bash -head -c 10 /dev/chuck_norise # prints "chuck nori" -``` +- **Sources anywhere you like.** The lab does not impose a `src/` or + `include/` layout — your Makefile's `*-y` line picks the source files + and `ccflags-y` picks the include paths. + +- **Exactly one `obj-m` entry per build.** The lab discovers the module + name from the produced `.ko`, so you don't declare it anywhere + else. + +- **Optionally a `debug_delay_ms` module parameter** that sleeps in + `module_init` so the host has time to attach GDB before init runs. + Declared as: -To see offset-preserving reads from a single open fd: + ```c + static unsigned int debug_delay_ms = 5000; + module_param(debug_delay_ms, uint, 0644); + ``` + + When declared, scripts/04 passes `DEBUG_LOAD_DELAY_MS` from + `lab.local.env` to `insmod`. When not declared, the module loads + without it. + +That's the entire contract. The lab handles per-target staging, DWARF +path rewriting, uploading, loading, and symbol discovery on the target. + +## Try the example ```bash -exec 9/dev/null # chu -dd bs=1 count=5 <&9 2>/dev/null # ck no -dd bs=1 count=7 <&9 2>/dev/null # rise!ch -exec 9<&- +cp -r examples/chuck_norise/. module/ ``` -## What this example demonstrates - -- A standard out-of-tree Linux kernel module project layout (`src/`, `include/`, - `Makefile`). -- A `Makefile` that doubles as a Kbuild fragment and as a standalone wrapper — - the conventional shape for kernel modules. The lab's outer build invokes it - the Kbuild way; running `make` in this directory directly invokes the wrapper. -- A module init that sleeps for `debug_delay_ms` (default 5000 ms, exposed as - a module parameter) before doing anything observable. The delay gives the - host time to attach GDB and set breakpoints before init runs. -- A `LINUX_VERSION_CODE` shim for the 6.4 `class_create()` signature change, - as a real-world example of one of the things out-of-tree modules have to - carry. +Then build and debug as in the top-level `README.md`. + +## Replacing the example with your own module + +Delete everything under `module/` except this README (or replace this +README too — the lab does not depend on it). Drop your project in. Edit +your `Makefile` to set `obj-m`, `-y`, and `ccflags-y`. Rebuild. diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index 0e7895a..87217c5 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -1,4 +1,14 @@ #!/usr/bin/env bash +# Provision a target VM for kernel module debugging. +# +# Target-side setup: install kernel headers (so the host build can sync them), +# optionally install matching vmlinux debug image + kernel source for +# step-into-kernel, and update the guest's GRUB command line with the boot +# args this lab needs (nokaslr always; kgdboc + sysrq for the kgdb method). +# +# Reboot the target after this script completes so the new boot args take +# effect. The script itself does not reboot — staying out of the user's way +# matters for VMs that are reverted from snapshots and shouldn't be touched. set -euo pipefail usage() { @@ -16,50 +26,29 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" validate_target "$target" "usage: $0 [--debug-symbols]" -env_file="$repo_root/lab.local.env" -[[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" + +env_file="$(lab_env_file "$repo_root")" # shellcheck source=/dev/null source "$env_file" -target_is_configured "$target" - -ssh_bin="${SSH_BIN:-ssh}" -sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(target_require_cfg "$target" SSH_HOST)" -ssh_port="$(target_cfg "$target" SSH_PORT)" -ssh_user="$(target_require_cfg "$target" SSH_USER)" -ssh_pass="$(target_cfg "$target" SSH_PASS)" -sudo_pass="$(target_cfg "$target" SUDO_PASS)" +lab_load_target "$target" + target_os="$(target_cfg "$target" OS)" kgdb_tty="$(target_cfg "$target" KGDB_TTY)" kgdb_baud="$(target_cfg "$target" KGDB_BAUD)" - -ssh_port="${ssh_port:-22}" target_os="${target_os:-ubuntu}" kgdb_tty="${kgdb_tty:-ttyS0}" kgdb_baud="${kgdb_baud:-115200}" -sudo_pass="${sudo_pass:-$ssh_pass}" -sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" case "$target_os" in ubuntu) ;; *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; esac -if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it on the build host: sudo apt-get install -y sshpass" >&2 - exit 1 -fi - -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port") -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") -fi - -echo "provisioning $target at $ssh_user@$ssh_host:$ssh_port (debug method: $debug_method)" +symbols_blurb="${symbols:+, with debug symbols}" +echo "provisioning $target at $LAB_SSH_TARGET:$LAB_SSH_PORT (debug method: $debug_method$symbols_blurb)" -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" -t "$ssh_user@$ssh_host" \ - "TARGET_OS='$target_os' DEBUG_METHOD='$debug_method' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$sudo_pass_b64' DEBUG_MAXCPUS='${DEBUG_MAXCPUS:-}' bash -s" <<'REMOTE' +SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" -t "$LAB_SSH_TARGET" \ + "TARGET_OS='$target_os' DEBUG_METHOD='$debug_method' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$LAB_SUDO_PASS_B64' DEBUG_MAXCPUS='${DEBUG_MAXCPUS:-}' bash -s" <<'REMOTE' set -euo pipefail sudo_run() { @@ -75,51 +64,51 @@ sudo_auth() { } apt_retry() { - sudo_run apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=20 "$@" + sudo_run apt-get \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=20 \ + -o Acquire::https::Timeout=20 \ + "$@" } . /etc/os-release kernel="$(uname -r)" codename="${VERSION_CODENAME:-noble}" [[ "${TARGET_OS:-ubuntu}" == "ubuntu" ]] || { echo "unsupported target OS: ${TARGET_OS:-}" >&2; exit 1; } -[[ "${ID:-}" == "ubuntu" ]] || { echo "target is not Ubuntu" >&2; exit 1; } +[[ "${ID:-}" == "ubuntu" ]] || { echo "target is not Ubuntu (os-release ID=$ID); only ubuntu targets are implemented" >&2; exit 1; } sudo_auth -echo "installing packages for running kernel $kernel" +echo "[1/3] installing kernel headers for running kernel $kernel" apt_retry update -apt_retry install -y linux-headers-"$kernel" rsync -[[ -d "/lib/modules/$kernel/build" ]] || { - echo "missing /lib/modules/$kernel/build after installing headers" >&2 - exit 1 -} +apt_retry install -y "linux-headers-$kernel" rsync +[[ -d "/lib/modules/$kernel/build" ]] || + { echo "missing /lib/modules/$kernel/build after installing headers; check apt output above" >&2; exit 1; } -vmlinux="/usr/lib/debug/boot/vmlinux-${kernel}" if [[ "$INSTALL_DEBUG_SYMBOLS" == "--debug-symbols" ]]; then + echo "[2/3] installing vmlinux debug image + kernel source for step-into-kernel" + vmlinux="/usr/lib/debug/boot/vmlinux-$kernel" if [[ ! -r "$vmlinux" ]]; then apt_retry install -y ubuntu-dbgsym-keyring cat </dev/null -deb http://ddebs.ubuntu.com ${codename} main restricted universe multiverse +deb http://ddebs.ubuntu.com $codename main restricted universe multiverse deb http://ddebs.ubuntu.com ${codename}-updates main restricted universe multiverse EOF sudo_run apt-get clean apt_retry update apt_retry install -y "linux-image-${kernel}-dbgsym" || - apt_retry install -y "linux-image-unsigned-${kernel}-dbgsym" || { - echo "failed to install debug symbols for $kernel" >&2 - echo "retry later if ddebs.ubuntu.com is returning 503" >&2 - exit 1 - } + apt_retry install -y "linux-image-unsigned-${kernel}-dbgsym" || + { echo "failed to install debug symbols for $kernel; retry later if ddebs.ubuntu.com is returning 503" >&2; exit 1; } fi + [[ -r "$vmlinux" ]] || echo "warning: $vmlinux still missing after install; kernel source debugging will be unavailable" - # Kernel source. Ubuntu's linux-source- package drops a - # tarball under /usr/src/. We don't extract on the target — script 02 - # rsyncs the tarball back to the host and extracts there, so the target - # stays unmodified beyond the package install. With the source available - # locally, GDB can step into kernel functions instead of just - # disassembling them. + # Kernel source. Try the major.minor.patch-suffixed package first + # (e.g. linux-source-6.8.0); fall back to the unversioned meta-package. + # Extraction is deferred to script 02 — the host has more incentive to + # extract (it's where GDB lives) and doing it there avoids requiring + # sudo + tar on the target. short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" src_pkg="" - for candidate in "linux-source-${short_kver}" "linux-source"; do + for candidate in "linux-source-$short_kver" "linux-source"; do [[ -z "$candidate" || "$candidate" == "linux-source-" ]] && continue if apt_retry install -y "$candidate"; then src_pkg="$candidate" @@ -127,39 +116,33 @@ EOF fi done if [[ -z "$src_pkg" ]]; then - echo "warning: could not install a linux-source package; step-into-kernel won't have source files" >&2 + echo "warning: could not install a linux-source package; step-into-kernel will only show disassembly" >&2 + elif ls -1 /usr/src/linux-source-*.tar.* /usr/src/linux-source-*/linux-source-*.tar.* 2>/dev/null | head -1 >/dev/null; then + echo " kernel source tarball is in /usr/src on the target; script 02 will sync and extract it" else - # Confirm the tarball is somewhere on disk for script 02 to find. - # Ubuntu has shipped both /usr/src/linux-source-X.tar.bz2 and - # /usr/src/linux-source-X/linux-source-X.tar.bz2 layouts over time — - # we accept either. - if ls -1 /usr/src/linux-source-*.tar.* /usr/src/linux-source-*/linux-source-*.tar.* 2>/dev/null | head -1 >/dev/null; then - echo "kernel source tarball is in /usr/src on the target; script 02 will sync and extract on the host" - else - echo "warning: $src_pkg installed but no linux-source tarball found under /usr/src/" >&2 - fi + echo "warning: $src_pkg installed but no linux-source tarball found under /usr/src/" >&2 fi +else + echo "[2/3] skipping debug symbols (pass --debug-symbols to enable step-into-kernel)" fi -[[ -r "$vmlinux" ]] || echo "note: $vmlinux is missing; VS Code source debugging may need it" +echo "[3/3] updating GRUB command line" grub_file=/etc/default/grub current="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" # Strip any prior values for the keys we manage so reruns with changed values -# replace rather than append. kgdboc and sysrq_always_enabled are also stripped -# so switching from kgdb to qemu mode removes them. +# replace rather than append. kgdboc and sysrq_always_enabled are stripped +# even when switching to qemu mode so they go away. current="$(printf '%s' "$current" | sed -E 's/(^| )(kgdboc=|maxcpus=|sysrq_always_enabled=)[^ ]*//g; s/ */ /g; s/^ +//; s/ +$//')" -# nokaslr is required in both modes so vmlinux symbols line up with running addresses. +# nokaslr is required in both modes so vmlinux symbols line up with running +# addresses. kgdb mode also needs an in-kernel debugger channel (kgdboc) and +# sysrq enabled so users can trigger a halt with `echo g > /proc/sysrq-trigger` +# — there is no kgdb_breakpoint() call in any user module to halt on insmod. +# qemu mode skips both: the hypervisor stub halts the vCPU directly. args=("nokaslr") - -# kgdb mode also needs an in-kernel debugger channel (kgdboc) and sysrq enabled -# so users can trigger a halt with "echo g > /proc/sysrq-trigger" — there is no -# longer a kgdb_breakpoint() in the module to halt on insmod. qemu mode skips -# both: the hypervisor stub halts the vCPU directly. if [[ "$DEBUG_METHOD" == "kgdb" ]]; then args+=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "sysrq_always_enabled=1") fi - if [[ -n "${DEBUG_MAXCPUS:-}" ]]; then args+=("maxcpus=${DEBUG_MAXCPUS}") fi @@ -167,7 +150,7 @@ for arg in "${args[@]}"; do case " $current " in *" $arg "*) ;; *) current="${current:+$current }$arg" ;; esac done -sudo_run cp "$grub_file" "$grub_file.small-ko.$(date +%Y%m%d%H%M%S).bak" +sudo_run cp "$grub_file" "$grub_file.kmod-debug-lab.$(date +%Y%m%d%H%M%S).bak" if grep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' "$grub_file"; then sudo_run sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$current\"|" "$grub_file" else @@ -175,5 +158,11 @@ else fi sudo_run update-grub -echo "done. reboot the VM before debugging. boot args: $current" +echo +echo "done. reboot the target VM before debugging." +echo " boot args: $current" REMOTE + +echo +echo "provisioning complete; reboot the $target VM, then run:" +echo " scripts/02-setup-host-build.sh $target" diff --git a/scripts/02-setup-host-build.sh b/scripts/02-setup-host-build.sh index 8a7ea71..3d20fdd 100755 --- a/scripts/02-setup-host-build.sh +++ b/scripts/02-setup-host-build.sh @@ -1,4 +1,17 @@ #!/usr/bin/env bash +# Sync the target's kernel build tree to the host so an out-of-tree module +# can be cross-built against the target's exact kernel without copying the +# whole source tree per build. +# +# Result: .kernel-cache// populated with: +# build/ -> symlink into the synced linux-headers tree +# source/ -> symlink into the synced linux-source tree (if --debug-symbols was used in 01) +# vmlinux -> debug-symbol vmlinux from the target (if available) +# kernel.release -> running kernel release (e.g. 6.8.0-117-generic) +# remote.build.path -> where headers came from on the target +# remote.header.paths -> list of all paths synced from /usr/src/ +# Also updates .kernel-cache/current -> so plain `make` and the +# IntelliSense config track the most recently synced target. set -euo pipefail target="${1:-}" @@ -7,73 +20,33 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" validate_target "$target" "usage: $0 " -env_file="$repo_root/lab.local.env" -[[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" +require_debian_host + +env_file="$(lab_env_file "$repo_root")" # shellcheck source=/dev/null source "$env_file" -require_debian_host -target_is_configured "$target" - -ssh_bin="${SSH_BIN:-ssh}" -rsync_bin="${RSYNC_BIN:-rsync}" -sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(target_require_cfg "$target" SSH_HOST)" -ssh_port="$(target_cfg "$target" SSH_PORT)" -ssh_user="$(target_require_cfg "$target" SSH_USER)" -ssh_pass="$(target_cfg "$target" SSH_PASS)" -sudo_pass="$(target_cfg "$target" SUDO_PASS)" +lab_load_target "$target" + target_os="$(target_cfg "$target" OS)" -ssh_port="${ssh_port:-22}" -sudo_pass="${sudo_pass:-$ssh_pass}" -sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" target_os="${target_os:-ubuntu}" - case "$target_os" in ubuntu) ;; *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; esac -if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it on the build host: sudo apt-get install -y sshpass" >&2 - exit 1 -fi - -ssh_target="$ssh_user@$ssh_host" -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -rsync_rsh="$ssh_bin -o StrictHostKeyChecking=accept-new -p $ssh_port" -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") - rsync_rsh="$sshpass_bin -e $ssh_bin -o StrictHostKeyChecking=accept-new -p $ssh_port" -fi - -remote_sudo_prefix() { - if [[ -n "$sudo_pass_b64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$sudo_pass_b64" - else - printf "sudo " - fi -} - -apt_update() { - sudo apt-get \ - -o Acquire::Retries=3 \ - -o Acquire::http::Timeout=20 \ - -o Acquire::https::Timeout=20 \ - update -} +# --- Install dev-host build prerequisites --------------------------------- -apt_install() { +apt_get() { sudo apt-get \ -o Acquire::Retries=3 \ -o Acquire::http::Timeout=20 \ -o Acquire::https::Timeout=20 \ - install -y "$@" + "$@" } -echo "installing/validating host kernel-module build packages" -apt_update -apt_install \ +echo "[1/5] installing host build prerequisites" +apt_get update +apt_get install -y \ bc \ bison \ build-essential \ @@ -86,25 +59,27 @@ apt_install \ rsync \ sshpass -kernel="$(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "uname -r")" +# --- Discover the target's kernel + header layout ------------------------- + +echo "[2/5] discovering target kernel" +kernel="$(lab_ssh 'uname -r')" +echo " target $target is running $kernel" + cache_dir="$repo_root/.kernel-cache/$target" -build_dir="$cache_dir/build" usr_src_dir="$cache_dir/usr-src" +build_dir="$cache_dir/build" remote_vmlinux="/usr/lib/debug/boot/vmlinux-$kernel" mkdir -p "$cache_dir" "$usr_src_dir" -echo "target $target is currently running kernel $kernel" -echo "discovering target kernel header layout" -build_real="$(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "readlink -f '/lib/modules/$kernel/build'")" -if ! SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "test -d '$build_real'"; then - echo "missing target build directory: $build_real" >&2 - echo "run scripts/01-provision-target.sh $target first" >&2 - exit 1 +build_real="$(lab_ssh "readlink -f '/lib/modules/$kernel/build'")" +if ! lab_ssh "test -d '$build_real'"; then + die "missing target build directory $build_real; +- on the target, check: ls -la /lib/modules/$kernel/build +- if it's missing, re-run: scripts/01-provision-target.sh $target " fi -mapfile -t remote_header_dirs < <(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "bash -s" <\t in a single round-trip so the sync -# loop below can stay dumb. -mapfile -t remote_source_entries < <(SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "bash -s" <<'REMOTE' +# Kernel source is discovered separately: it can be a tarball (a single file +# at /usr/src/linux-source-X.tar.bz2) OR a directory (possibly containing +# the tarball inside it, on some Ubuntu releases). One round-trip classifies +# each path as file|dir so the sync loop stays trivial. +mapfile -t remote_source_entries < <(lab_ssh "bash -s" <<'REMOTE' set -euo pipefail shopt -s nullglob seen=() @@ -145,45 +119,37 @@ done REMOTE ) +# --- Sync ---------------------------------------------------------------- + sync_header_dir() { local remote_dir="$1" - local name local dest - - name="$(basename "$remote_dir")" - dest="$usr_src_dir/$name" - - echo "syncing $remote_dir -> $dest" + dest="$usr_src_dir/$(basename "$remote_dir")" + echo " $remote_dir -> ${dest#"$repo_root"/}" # Preserve symlinks inside Ubuntu's kernel header trees. Some optional - # symlinks, such as rust support links, may be dangling and are harmless for + # symlinks (e.g. rust support) may be dangling and are harmless for # external C module builds. - SSHPASS="$ssh_pass" "$rsync_bin" -a --delete -e "$rsync_rsh" \ - "$ssh_target:$remote_dir/" "$dest/" + lab_rsync_from "$remote_dir/" "$dest/" --delete } -sync_remote_path() { - # Sync either a remote dir (contents synced into $dest/) or a remote file - # (file synced as $dest). Used for kernel-source paths where the remote - # layout can be either. +sync_source_path() { local remote_path="$1" local kind="$2" - local name local dest - - name="$(basename "$remote_path")" - dest="$usr_src_dir/$name" - + dest="$usr_src_dir/$(basename "$remote_path")" + echo " $remote_path -> ${dest#"$repo_root"/}" if [[ "$kind" == "dir" ]]; then - echo "syncing $remote_path/ -> $dest/" - SSHPASS="$ssh_pass" "$rsync_bin" -a --delete -e "$rsync_rsh" \ - "$ssh_target:$remote_path/" "$dest/" + lab_rsync_from "$remote_path/" "$dest/" --delete else - echo "syncing $remote_path -> $dest" - SSHPASS="$ssh_pass" "$rsync_bin" -a -e "$rsync_rsh" \ - "$ssh_target:$remote_path" "$dest" + lab_rsync_from "$remote_path" "$dest" fi } +echo "[3/5] syncing kernel headers and source" +# Dedup the discovered header paths; the discovery on the target can +# legitimately list the same canonical path twice (e.g. when +# /lib/modules/$kernel/source and /usr/src/linux-headers-$base resolve to +# the same dir via symlink chains). printf '%s\n' "${remote_header_dirs[@]}" | awk 'NF && !seen[$0]++' | while IFS= read -r remote_dir; do sync_header_dir "$remote_dir" @@ -191,36 +157,44 @@ done for entry in "${remote_source_entries[@]}"; do [[ -z "$entry" ]] && continue - path="${entry%$'\t'*}" - kind="${entry##*$'\t'}" - sync_remote_path "$path" "$kind" + sync_source_path "${entry%$'\t'*}" "${entry##*$'\t'}" done build_name="$(basename "$build_real")" rm -rf "$build_dir" ln -s "usr-src/$build_name" "$build_dir" +[[ -f "$build_dir/Makefile" ]] || die "synced build tree is missing Makefile: $build_dir" + +# --- vmlinux ------------------------------------------------------------- -if [[ ! -f "$build_dir/Makefile" ]]; then - echo "synced build tree is missing Makefile: $build_dir" >&2 - exit 1 +echo "[4/5] checking vmlinux debug image" +if lab_ssh_sudo "test -r '$remote_vmlinux'"; then + echo " copying $remote_vmlinux" + lab_ssh_sudo "cat '$remote_vmlinux'" > "$cache_dir/vmlinux" +else + rm -f "$cache_dir/vmlinux" + echo " no vmlinux on target ($remote_vmlinux is missing)" + echo " module debugging will work; kernel source debugging will not" + echo " to enable, run: scripts/01-provision-target.sh $target --debug-symbols" fi -# Kernel source: find a usable source root under usr-src/ and point the -# stable `source` symlink at it. Ubuntu has shipped three layouts: +# --- Kernel source resolution -------------------------------------------- +# +# Find a usable kernel-source root under usr-src/. Ubuntu has shipped three +# layouts over time: # 1. /usr/src/linux-source-X.tar.bz2 (tarball, no enclosing dir) # 2. /usr/src/linux-source-X/ (pre-extracted, files at top) # 3. /usr/src/linux-source-X/linux-source-X/ (pre-extracted, nested one deep) # Layout 1 also occurs nested as /usr/src/linux-source-X/linux-source-X.tar.bz2. # -# Strategy: look for a Makefile (the kernel's top-level marker) at depth ≤ 3 -# under usr-src/. If found, that's our source root — symlink directly to it, -# no extraction needed. Otherwise look for a tarball and extract on the host -# into $cache_dir/source-tree/. -# -# Extraction happens on the host, not the target, for two reasons: it avoids -# sudo + tar on the target for what's purely a host-side debug resource, and -# it's self-healing — re-running 02 fixes broken/partial trees without -# needing to touch the target. +# Strategy: look for a Makefile + init/main.c (the kernel's top-level +# markers) at depth ≤ 3 under usr-src/. If found, symlink directly to it; +# otherwise look for a tarball and extract on the host into source-tree/. +# Extraction on the host (not the target) avoids a sudo+tar dependency on +# the target for what's purely a host-side debug resource, and makes the +# step self-healing: re-running 02 fixes partial/broken trees. + +echo "[5/5] resolving kernel source for step-into-kernel" source_link="$cache_dir/source" source_tree="$cache_dir/source-tree" rm -f "$source_link" @@ -240,49 +214,42 @@ done < <(find "$usr_src_dir" -mindepth 1 -maxdepth 3 -name Makefile -path '*linu if [[ -n "$found_root" ]]; then rm -rf "$source_tree" - rel="${found_root#$cache_dir/}" + rel="${found_root#"$cache_dir"/}" ln -s "$rel" "$source_link" - echo "kernel source: $source_link -> $rel" + echo " kernel source: $source_link -> $rel" else shopt -s nullglob - tarballs=( "$usr_src_dir"/linux-source-*.tar.* "$usr_src_dir"/linux-source-*/linux-source-*.tar.* ) + tarballs=("$usr_src_dir"/linux-source-*.tar.* "$usr_src_dir"/linux-source-*/linux-source-*.tar.*) shopt -u nullglob if [[ ${#tarballs[@]} -gt 0 ]]; then tarball="${tarballs[0]}" if is_kernel_source_root "$source_tree"; then - echo "kernel source already extracted at $source_tree" + echo " kernel source already extracted at $source_tree" else - echo "extracting $tarball -> $source_tree" + echo " extracting $(basename "$tarball") -> source-tree/ (one-time, ~30s)" rm -rf "$source_tree" mkdir -p "$source_tree" tar -C "$source_tree" --strip-components=1 -xf "$tarball" fi ln -s source-tree "$source_link" - echo "kernel source: $source_link -> source-tree (extracted from $(basename "$tarball"))" + echo " kernel source: $source_link -> source-tree" else rm -rf "$source_tree" - echo "no kernel source tarball or extracted tree on target; step-into-kernel will only show disassembly" - echo "run scripts/01-provision-target.sh $target --debug-symbols to install it" + echo " no kernel source on target; kernel step-into will show disassembly only" + echo " to enable, run: scripts/01-provision-target.sh $target --debug-symbols" fi fi -echo "checking vmlinux debug image" -remote_sudo="$(remote_sudo_prefix)" -if SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}test -r '$remote_vmlinux'"; then - echo "copying $remote_vmlinux" - SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}cat '$remote_vmlinux'" > "$cache_dir/vmlinux" -else - rm -f "$cache_dir/vmlinux" - echo "warning: missing readable $remote_vmlinux on target" >&2 - echo "module builds can still work, but VS Code source debugging needs it" >&2 - echo "run scripts/01-provision-target.sh $target --debug-symbols to install it" >&2 -fi +# --- Bookkeeping --------------------------------------------------------- printf '%s\n' "$kernel" > "$cache_dir/kernel.release" printf '%s\n' "$build_real" > "$cache_dir/remote.build.path" printf '%s\n' "${remote_header_dirs[@]}" > "$cache_dir/remote.header.paths" ln -sfn "$target" "$repo_root/.kernel-cache/current" -echo "cached $target kernel $kernel in $cache_dir" -echo "local KDIR: $build_dir" -echo "current IntelliSense target: $target" +echo +echo "synced $target kernel $kernel" +echo " build -> $build_dir" +[[ -L "$source_link" ]] && echo " source -> $(readlink -f "$source_link")" +[[ -f "$cache_dir/vmlinux" ]] && echo " vmlinux: $cache_dir/vmlinux" +echo "next: scripts/03-build-module.sh $target" diff --git a/scripts/03-build-module.sh b/scripts/03-build-module.sh index 9d01b3b..3c54c7f 100755 --- a/scripts/03-build-module.sh +++ b/scripts/03-build-module.sh @@ -1,8 +1,16 @@ #!/usr/bin/env bash +# Build module/ against the synced headers for . +# +# Driven by the top-level Makefile, which stages module/ into +# build/intermediate/// and runs Kbuild from there with +# -ffile-prefix-map injected via KCFLAGS. Result is one .ko under +# build/artifacts///. The module name is whatever the user's +# obj-m declares; the lab does not need to know it. set -euo pipefail target="${1:-}" repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" validate_target "$target" "usage: $0 " @@ -10,24 +18,18 @@ cache_dir="$repo_root/.kernel-cache/$target" kernel_file="$cache_dir/kernel.release" kdir="$cache_dir/build" -if [[ ! -f "$kernel_file" || ! -d "$kdir" ]]; then - echo "missing kernel cache for $target" >&2 - echo "run scripts/02-setup-host-build.sh $target first" >&2 - exit 1 -fi +[[ -f "$kernel_file" && -d "$kdir" ]] || + die "missing kernel cache for $target; run: scripts/02-setup-host-build.sh $target" module_dir="$repo_root/module" -if [[ ! -e "$module_dir/Makefile" && ! -e "$module_dir/Kbuild" ]]; then - echo "no Makefile or Kbuild under $module_dir" >&2 - echo "drop your module project under module/ (see module/README.md)" >&2 - exit 1 -fi +[[ -e "$module_dir/Makefile" || -e "$module_dir/Kbuild" ]] || + die "no Makefile or Kbuild under $module_dir; drop your module project there (see $module_dir/README.md), or try the example: cp -r examples/chuck_norise/. module/" kernel="$(<"$kernel_file")" intermediate_dir="$repo_root/build/intermediate/$target/$kernel" artifact_dir="$repo_root/build/artifacts/$target/$kernel" -echo "building module under $module_dir for $target kernel $kernel" +echo "building $module_dir for $target kernel $kernel" make -C "$repo_root" \ KDIR="$kdir" \ BUILD_ID="$target/$kernel" \ @@ -37,10 +39,8 @@ make -C "$repo_root" \ clean modules mapfile -t kos < <(find "$artifact_dir" -maxdepth 1 -name '*.ko' -type f | sort) -if [[ ${#kos[@]} -eq 0 ]]; then - echo "build produced no .ko under $artifact_dir" >&2 - exit 1 -fi +[[ ${#kos[@]} -gt 0 ]] || die "build produced no .ko under $artifact_dir; check the make output above" + for ko in "${kos[@]}"; do echo "built $ko" done diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index 26f879c..1e01e95 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -1,4 +1,26 @@ #!/usr/bin/env bash +# Deploy the built module to a target, load it, and write the GDB init files +# VS Code (or `gdb -tui`) needs to attach and source-debug it. +# +# Steps: +# 1. Free the debug-endpoint TCP port from any stale GDB clients. Both +# KGDB's serial bridge and QEMU's gdbstub serve one client at a time, +# so any leftover gdb owning that socket blocks fresh attaches. This +# script kills any *local* gdb process holding the endpoint — be aware +# if you have unrelated gdb sessions on the same host:port. +# 2. Upload module/.ko to the target. +# 3. Insmod it (with debug_delay_ms if the module declares the param, +# falling back to a plain insmod otherwise). +# 4. Poll /sys/module//sections/ for the module's runtime load +# addresses; convert to an `add-symbol-file ... -s .name addr ...` +# line saved as the per-(target,method) symbols file. +# 5. Emit .gdb/-.gdb and -attached.gdb variants, plus +# current-debug{,attached}.gdb and current-vmlinux symlinks the IDE +# launch configs point at. +# +# Env knobs (default values in parens): +# DEBUG_LOAD_DELAY_MS (5000) passed to the module if it declares debug_delay_ms +# INSMOD_WAIT_SECS (30) how long to wait for /sys/module/.../sections/.text set -euo pipefail usage() { @@ -14,69 +36,31 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/common.sh source "$repo_root/scripts/lib/common.sh" validate_target "$target" "usage: $0 " -env_file="$repo_root/lab.local.env" -[[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it" + +env_file="$(lab_env_file "$repo_root")" # shellcheck source=/dev/null source "$env_file" -target_is_configured "$target" +lab_load_target "$target" -ssh_bin="${SSH_BIN:-ssh}" -scp_bin="${SCP_BIN:-scp}" -sshpass_bin="${SSHPASS_BIN:-sshpass}" -ssh_host="$(target_require_cfg "$target" SSH_HOST)" -ssh_port="$(target_cfg "$target" SSH_PORT)" -ssh_user="$(target_require_cfg "$target" SSH_USER)" -ssh_pass="$(target_cfg "$target" SSH_PASS)" -sudo_pass="$(target_cfg "$target" SUDO_PASS)" -remote_dir="$(target_cfg "$target" REMOTE_DIR)" endpoint_key="DEBUG_ENDPOINT_${debug_method^^}" debug_endpoint="$(target_require_cfg "$target" "$endpoint_key")" debug_delay_ms="${DEBUG_LOAD_DELAY_MS:-5000}" +insmod_wait_secs="${INSMOD_WAIT_SECS:-30}" -ssh_port="${ssh_port:-22}" -remote_dir="${remote_dir:-/tmp/kmod-debug-lab}" -sudo_pass="${sudo_pass:-$ssh_pass}" -sudo_pass_b64="$(printf '%s' "$sudo_pass" | base64 -w0)" - -if [[ -n "$ssh_pass" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - echo "ssh password is configured, but $sshpass_bin is not installed" >&2 - echo "install it on the build host: sudo apt-get install -y sshpass" >&2 - exit 1 -fi - -ssh_target="$ssh_user@$ssh_host" -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -scp_cmd=("$scp_bin" -o StrictHostKeyChecking=accept-new -P "$ssh_port") -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") - scp_cmd=("$sshpass_bin" -e "${scp_cmd[@]}") -fi - -remote_sudo_prefix() { - if [[ -n "$sudo_pass_b64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$sudo_pass_b64" - else - printf "sudo " - fi -} +# --- Locate built artifact + vmlinux -------------------------------------- cache_dir="$repo_root/.kernel-cache/$target" kernel_file="$cache_dir/kernel.release" vmlinux="$cache_dir/vmlinux" -if [[ ! -f "$kernel_file" ]]; then - echo "missing kernel cache for $target" >&2 - echo "run scripts/02-setup-host-build.sh $target first" >&2 - exit 1 -fi +[[ -f "$kernel_file" ]] || + die "missing kernel cache for $target; run: scripts/02-setup-host-build.sh $target" -if [[ ! -f "$vmlinux" ]]; then - echo "missing $vmlinux" >&2 - echo "module builds may work, but VS Code source debugging needs the matching vmlinux" >&2 - echo "install the target's linux-image-*-dbgsym package, then rerun:" >&2 - echo " scripts/02-setup-host-build.sh $target" >&2 - exit 1 -fi +[[ -f "$vmlinux" ]] || + die "missing $vmlinux — source debugging needs the matching vmlinux from the target. +- install + sync it with: + scripts/01-provision-target.sh $target $debug_method --debug-symbols + scripts/02-setup-host-build.sh $target" kernel="$(<"$kernel_file")" artifact_dir="$repo_root/build/artifacts/$target/$kernel" @@ -84,87 +68,163 @@ intermediate_dir="$repo_root/build/intermediate/$target/$kernel" module_dir="$repo_root/module" mapfile -t artifacts < <(find "$artifact_dir" -maxdepth 1 -name '*.ko' -type f | sort) -if [[ ${#artifacts[@]} -eq 0 ]]; then - echo "no .ko under $artifact_dir" >&2 - echo "run scripts/03-build-module.sh $target first" >&2 - exit 1 -fi -if [[ ${#artifacts[@]} -gt 1 ]]; then - echo "expected exactly one .ko under $artifact_dir; found:" >&2 - printf ' %s\n' "${artifacts[@]}" >&2 - echo "the lab assumes a single obj-m per build" >&2 - exit 1 -fi +case ${#artifacts[@]} in + 0) die "no .ko under $artifact_dir; run: scripts/03-build-module.sh $target" ;; + 1) ;; + *) + echo "expected one .ko under $artifact_dir; found:" >&2 + printf ' %s\n' "${artifacts[@]}" >&2 + die "the lab assumes a single obj-m per build. Merge sources into one module (obj-m += foo.o; foo-y := a.o b.o ...) or split into separate module/ trees." + ;; +esac artifact="${artifacts[0]}" module_name="$(basename "$artifact" .ko)" +# --- GDB output paths ----------------------------------------------------- + gdb_dir="$repo_root/.gdb" mkdir -p "$gdb_dir" symbols_file="$gdb_dir/$target-$debug_method-module-symbols.gdb" gdb_file="$gdb_dir/$target-$debug_method.gdb" +gdb_attached_file="$gdb_dir/$target-$debug_method-attached.gdb" loader_log="$gdb_dir/$target-$debug_method-loader.log" -loader_script="$gdb_dir/$target-$debug_method-loader.sh" -rm -f "$symbols_file" "$loader_log" "$loader_script" - -remote_module="$remote_dir/$module_name.ko" -remote_sudo="$(remote_sudo_prefix)" - -# Kill any local gdb processes that still hold a TCP connection to the debug -# endpoint. Both KGDB's serial bridge and QEMU's gdbstub serve one client at a -# time; leftover gdb processes from previous sessions (zombie bare-gdb -# invocations, half-torn-down VS Code launches, etc.) sit in the endpoint's -# accept queue and block fresh attaches. -debug_host="${debug_endpoint%:*}" -debug_port="${debug_endpoint##*:}" -if command -v ss >/dev/null 2>&1; then - stale_pid_list=$({ ss -ntp 2>/dev/null \ - | grep -F "$debug_host:$debug_port" \ +rm -f "$symbols_file" "$loader_log" + +# --- Free the debug endpoint from stale GDB clients ----------------------- +# +# Both KGDB's serial bridge and QEMU's gdbstub serve one client at a time. +# A zombie gdb left over from a previous session sits in the endpoint's +# accept queue and blocks fresh attaches. We best-effort kill any local gdb +# process that holds a connection to the endpoint. Errors here never fail +# the deploy — at worst the user has to `killall gdb` manually. +free_debug_endpoint() { + command -v ss >/dev/null 2>&1 || return 0 + local host="${debug_endpoint%:*}" port="${debug_endpoint##*:}" + local pids + pids="$(ss -ntp 2>/dev/null \ + | grep -F "$host:$port" \ | grep -oE 'pid=[0-9]+' \ | cut -d= -f2 \ - | sort -u; } || true) - stale_gdb_pids="" - for pid in $stale_pid_list; do - if [[ "$(ps -p "$pid" -o comm= 2>/dev/null || true)" == "gdb" ]]; then - stale_gdb_pids="$stale_gdb_pids $pid" - fi + | sort -u || true)" + local gdb_pids=() + local pid + for pid in $pids; do + [[ "$(ps -p "$pid" -o comm= 2>/dev/null || true)" == "gdb" ]] && gdb_pids+=("$pid") done - if [[ -n "$stale_gdb_pids" ]]; then + if [[ ${#gdb_pids[@]} -gt 0 ]]; then echo "killing leftover gdb clients on $debug_endpoint:" - # shellcheck disable=SC2086 - ps -p $stale_gdb_pids -o pid,cmd 2>/dev/null | tail -n +2 | sed 's/^/ /' || true - # shellcheck disable=SC2086 - kill -9 $stale_gdb_pids 2>/dev/null || true + ps -p "${gdb_pids[@]}" -o pid,cmd 2>/dev/null | tail -n +2 | sed 's/^/ /' || true + kill -9 "${gdb_pids[@]}" 2>/dev/null || true sleep 0.3 fi -fi +} +free_debug_endpoint || true + +# --- Upload + load on target ---------------------------------------------- + +remote_module="$LAB_REMOTE_DIR/$module_name.ko" + +echo "uploading $artifact -> $LAB_SSH_TARGET:$remote_module" +lab_ssh "mkdir -p '$LAB_REMOTE_DIR'" +lab_scp_to "$artifact" "$remote_module" + +# Load the module and discover the addresses /sys/module//sections/ +# exposes once it's live. The insmod runs inside a single nohup'd sh -c so +# the debug_delay_ms-then-plain-insmod fallback chains correctly: if we +# instead ran two `nohup ... &` invocations the shell would background them +# independently and the `||` between them would be a no-op (it'd just check +# the success of the backgrounding, not the insmod itself). +# +# Output is captured to $loader_log; we surface either the success tail +# line or the full log on failure. +load_and_discover_symbols() { + echo "loading $module_name on $target (debug_delay_ms=$debug_delay_ms)" + lab_ssh_sudo "rmmod '$module_name' >/dev/null 2>&1 || true" + # nohup must wrap the sudo invocation so the load survives the SSH + # session closing; we keep the sudo prefix inline rather than going + # through lab_ssh_sudo here. + lab_ssh "nohup $(lab_remote_sudo_prefix)sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$LAB_REMOTE_DIR/insmod.log' 2>&1 &" + + echo "polling /sys/module/$module_name/sections/ (timeout ${insmod_wait_secs}s)" + + # Dump every readable file under /sys/module//sections/. Discovery + # is dynamic so modules with non-standard sections (.text.hot, custom + # __ksymtab subsections, ...) get their addresses picked up too. + # + # The sh -c body MUST be single-quoted in the SSH command so the outer + # remote bash doesn't expand $(ls -A) and $f before sh -c sees them. + # With double quotes, the outer bash would evaluate the substitution in + # its own CWD; the for loop would iterate over the wrong filenames and + # we'd never find .text — even though the module had already loaded. + local dump_body + # shellcheck disable=SC2016 # $(...) is intentionally not expanded locally — see comment above. + dump_body='cd "/sys/module/'"$module_name"'/sections" 2>/dev/null && for f in $(ls -A 2>/dev/null); do [ -r "$f" ] && printf "%s %s\n" "$f" "$(cat "$f")"; done' + + local deadline=$((SECONDS + insmod_wait_secs)) + local tmp + tmp="$(mktemp)" + while ((SECONDS < deadline)); do + if lab_ssh_sudo "sh -c '$dump_body'" > "$tmp" 2>/dev/null; then + local text_addr + text_addr="$(awk '$1 == ".text" { print $2 }' "$tmp")" + if [[ -n "$text_addr" ]]; then + local cmd="add-symbol-file $artifact $text_addr" + while read -r sec addr; do + [[ "$sec" != ".text" && -n "$addr" ]] && cmd+=" -s $sec $addr" + done < "$tmp" + { + echo "$cmd" + printf 'echo loaded %s module symbols for %s\\n\n' "$module_name" "$target" + } > "$symbols_file" + rm -f "$tmp" + echo "wrote $symbols_file" + return 0 + fi + fi + sleep 0.2 + done -echo "uploading $artifact to $ssh_target:$remote_module" -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "mkdir -p '$remote_dir'" -SSHPASS="$ssh_pass" "${scp_cmd[@]}" "$artifact" "$ssh_target:$remote_module" + rm -f "$tmp" + echo "timeout: /sys/module/$module_name/sections/.text did not appear within ${insmod_wait_secs}s" + echo "remote insmod.log (empty = insmod produced no output; module may still have failed to load):" + lab_ssh "cat '$LAB_REMOTE_DIR/insmod.log' 2>/dev/null || true" | sed 's/^/ /' + echo "hint: ssh into the target and try the load by hand:" + echo " ssh $LAB_SSH_TARGET sudo insmod $remote_module" + echo " ssh $LAB_SSH_TARGET sudo dmesg | tail -40" + echo " ssh $LAB_SSH_TARGET ls -la /sys/module/$module_name/sections/ 2>/dev/null" + return 1 +} -# If we have a synced kernel source tree (from scripts/02 + provision with -# --debug-symbols), try to map Ubuntu's build-time source prefix to our local -# copy so GDB can show kernel source on step-into. We discover the prefix -# dynamically by asking addr2line for the file/line of `start_kernel`, which -# always lives at init/main.c in any Linux source tree — the prefix is -# whatever comes before that suffix. -kernel_src_link="$cache_dir/source" +if load_and_discover_symbols > "$loader_log" 2>&1; then + tail -1 "$loader_log" +else + echo "module load / symbol discovery failed (full log: $loader_log):" + sed 's/^/ /' "$loader_log" + exit 1 +fi + +# --- Build the GDB init files -------------------------------------------- +# +# If the kernel source has been synced (scripts/01 --debug-symbols + scripts/02), +# discover Ubuntu's build-time source prefix from vmlinux so GDB can remap +# DWARF references to our local copy. We ask addr2line where `start_kernel` +# lives — the path is always /init/main.c, so stripping the +# known suffix yields the prefix. kernel_substitute_line="" kernel_directory_line="" +kernel_src_link="$cache_dir/source" if [[ -L "$kernel_src_link" || -d "$kernel_src_link" ]]; then kernel_src_root="$(readlink -f "$kernel_src_link")" kernel_directory_line="directory $kernel_src_root" if command -v nm >/dev/null 2>&1 && command -v addr2line >/dev/null 2>&1; then - # `|| true` because pipefail + SIGPIPE: nm dumps every symbol in - # vmlinux (millions), awk's `exit` after the first match closes the - # pipe, nm gets SIGPIPE on its next write, and the pipeline exits - # 141. We still capture the address awk printed before exiting. + # `|| true` because pipefail + SIGPIPE: nm dumps every vmlinux + # symbol, awk's `exit` after the first match closes the pipe and nm + # gets SIGPIPE on its next write. We still capture awk's output. sym_addr="$(nm "$vmlinux" 2>/dev/null | awk 'NF==3 && $3=="start_kernel" {print $1; exit}' || true)" if [[ -n "$sym_addr" ]]; then sym_loc="$(addr2line -e "$vmlinux" "$sym_addr" 2>/dev/null | head -1 | cut -d: -f1 || true)" - # Expected shape: /init/main.c build_prefix="${sym_loc%/init/main.c}" if [[ -n "$build_prefix" && "$build_prefix" != "$sym_loc" ]]; then kernel_substitute_line="set substitute-path $build_prefix $kernel_src_root" @@ -173,12 +233,12 @@ if [[ -L "$kernel_src_link" || -d "$kernel_src_link" ]]; then echo "warning: addr2line returned unexpected location for start_kernel: $sym_loc" fi else - echo "warning: could not find start_kernel in vmlinux; skipping kernel source remap" + echo "warning: could not find start_kernel in vmlinux; kernel step-into will show disassembly" fi fi else - echo "note: $kernel_src_link missing; GDB will show kernel disassembly instead of source" - echo " run scripts/01 with --debug-symbols then re-run scripts/02 to enable step-into-kernel" + echo "note: no kernel source synced; kernel step-into will show disassembly" + echo " to enable: scripts/01-... --debug-symbols && scripts/02-... $target" fi { @@ -205,96 +265,18 @@ source $symbols_file EOF } > "$gdb_file" -# Trimmed-down variant for IDEs that handle the gdb-side attach themselves -# (Native Debug, cppdbg with miDebuggerServerAddress). The IDE has already -# called `target remote` and loaded the executable, so commands that touch -# global gdb state (mi-async, remote timeouts, architecture override) error -# with "Cannot change this setting while the inferior is running" — drop them. -gdb_attached_file="$gdb_dir/$target-$debug_method-attached.gdb" -grep -vE '^(target remote |set mi-async |set target-async |set tcp connect-timeout |set remotetimeout |set architecture |symbol-file )' "$gdb_file" > "$gdb_attached_file" +# Native Debug ("type": "gdb") and cppdbg with miDebuggerServerAddress have +# already called `target remote` and loaded the executable by the time they +# source our script, so commands that touch global gdb state error with +# "Cannot change this setting while the inferior is running". Strip them. +grep -vE '^(target remote |set mi-async |set target-async |set tcp connect-timeout |set remotetimeout |set architecture |symbol-file )' \ + "$gdb_file" > "$gdb_attached_file" ln -sfn "$target-$debug_method.gdb" "$gdb_dir/current-debug.gdb" ln -sfn "$target-$debug_method-attached.gdb" "$gdb_dir/current-debug-attached.gdb" ln -sfn "../.kernel-cache/$target/vmlinux" "$gdb_dir/current-vmlinux" -{ - printf '#!/usr/bin/env bash\n' - printf 'set -euo pipefail\n' - printf 'target=%q\n' "$target" - printf 'module_name=%q\n' "$module_name" - printf 'symbols_file=%q\n' "$symbols_file" - printf 'artifact=%q\n' "$artifact" - printf 'ssh_bin=%q\n' "$ssh_bin" - printf 'sshpass_bin=%q\n' "$sshpass_bin" - printf 'ssh_pass=%q\n' "$ssh_pass" - printf 'ssh_port=%q\n' "$ssh_port" - printf 'ssh_target=%q\n' "$ssh_target" - printf 'sudo_pass_b64=%q\n' "$sudo_pass_b64" - printf 'remote_module=%q\n' "$remote_module" - printf 'remote_dir=%q\n' "$remote_dir" - printf 'debug_delay_ms=%q\n' "$debug_delay_ms" - cat <<'LOADER' - -remote_sudo_prefix() { - if [[ -n "$sudo_pass_b64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$sudo_pass_b64" - else - printf "sudo " - fi -} - -ssh_cmd=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$ssh_port" "$ssh_target") -if [[ -n "$ssh_pass" ]]; then - ssh_cmd=("$sshpass_bin" -e "${ssh_cmd[@]}") -fi -remote_sudo="$(remote_sudo_prefix)" - -echo "loading $module_name on $target and discovering symbols" -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" "${remote_sudo}rmmod '$module_name' >/dev/null 2>&1 || true" - -# debug_delay_ms is the lab's convention for "sleep this long in module_init so -# the host has time to attach GDB and discover sections". Modules that declare -# it as a module_param accept the kv argument; modules that don't will reject -# insmod with -EINVAL on the unknown parameter. We try with the parameter and -# fall back to a plain insmod, inside a single sh -c so the || chain runs -# inside one backgrounded process. -SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "nohup ${remote_sudo}sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$remote_dir/insmod.log' 2>&1 &" - -tmp_sections="$(mktemp)" -trap 'rm -f "$tmp_sections"' EXIT - -section_list=".text .data .bss .rodata .init.text .exit.text .text.unlikely .rodata.str1.1 .rodata.str1.8 .init.data .exit.data" -deadline=$((SECONDS + 30)) -while ((SECONDS < deadline)); do - if SSHPASS="$ssh_pass" "${ssh_cmd[@]}" \ - "${remote_sudo}sh -c 'for sec in $section_list; do path=/sys/module/$module_name/sections/\$sec; if [ -r \"\$path\" ]; then printf \"%s %s\n\" \"\$sec\" \"\$(cat \"\$path\")\"; fi; done'" \ - > "$tmp_sections"; then - text_addr="$(awk '$1 == ".text" { print $2 }' "$tmp_sections")" - if [[ -n "$text_addr" ]]; then - cmd="add-symbol-file $artifact $text_addr" - while read -r sec addr; do - if [[ "$sec" != ".text" && -n "$addr" ]]; then - cmd="$cmd -s $sec $addr" - fi - done < "$tmp_sections" - { - echo "$cmd" - echo "echo loaded $module_name module symbols for $target\\n" - } > "$symbols_file" - echo "wrote module symbols to $symbols_file" - exit 0 - fi - fi - sleep 0.2 -done - -echo "failed to discover /sys/module/$module_name/sections/.text within 30s" -exit 1 -LOADER -} > "$loader_script" -chmod +x "$loader_script" -"$loader_script" > "$loader_log" 2>&1 - -echo "generated $gdb_file" -echo "loaded $module_name and wrote symbols; log: $loader_log" +echo +echo "ready to attach GDB at $debug_endpoint" +echo " gdb script: $gdb_file" +echo " loader log: $loader_log" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 0de37be..a16a084 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -1,10 +1,25 @@ #!/usr/bin/env bash +# scripts/lib/common.sh — shared helpers sourced by every numbered script. +# +# Naming: +# die, validate_target, target_* bare functions; no global state required. +# lab_* higher-level helpers; rely on globals +# populated by `lab_load_target`. +# LAB_* globals populated by `lab_load_target`. +# Scripts should call lab_ssh / lab_scp_to +# / lab_rsync_from rather than touching +# the underlying ssh/scp/sshpass binaries. +# +# Sourcing this file is side-effect-free. State is created when a script +# calls `lab_load_target ` after sourcing `lab.local.env`. die() { echo "$*" >&2 exit 1 } +# --- Target validation ---------------------------------------------------- + validate_target() { local target="$1" local usage="$2" @@ -50,5 +65,125 @@ target_require_cfg() { require_debian_host() { command -v apt-get >/dev/null 2>&1 || - die "missing apt-get; the development host must be Debian-based" + die "missing apt-get; the development host must be Debian-based (Debian, Ubuntu, WSL Ubuntu, ...)" +} + +# --- Lab env loading ------------------------------------------------------ + +# Validate that lab.local.env exists under repo_root and return its path on +# stdout. Caller is expected to `source` the returned path directly so the +# `declare -A TARGET_*=...` entries land in the script's global scope (a +# function-internal source would scope them to the function). +lab_env_file() { + local repo_root="$1" + local env_file="$repo_root/lab.local.env" + [[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it for your machines" + printf '%s' "$env_file" +} + +# --- Per-target connection setup ------------------------------------------ + +# Read TARGET_* config for the given target and populate LAB_* globals used +# by every remote helper below. Validates that ssh password tooling is +# available if a password is configured. +# +# After this returns, the following globals are set: +# LAB_TARGET target name +# LAB_SSH_HOST/PORT/USER raw connection details +# LAB_SSH_PASS ssh password ("" when using keys) +# LAB_SUDO_PASS_B64 base64-encoded sudo password (for piping into sudo -S) +# LAB_REMOTE_DIR target staging dir (with default) +# LAB_SSH_TARGET "user@host" form for use in commands +# LAB_SSH_CMD array, ready to invoke (includes sshpass wrapping) +# LAB_SCP_CMD array, ready to invoke (includes sshpass wrapping) +# LAB_RSYNC_RSH rsync -e value (includes sshpass wrapping) +lab_load_target() { + local target="$1" + target_is_configured "$target" + + # shellcheck disable=SC2034 # LAB_TARGET is consumed by sourcing scripts. + LAB_TARGET="$target" + LAB_SSH_HOST="$(target_require_cfg "$target" SSH_HOST)" + LAB_SSH_PORT="$(target_cfg "$target" SSH_PORT)" + LAB_SSH_USER="$(target_require_cfg "$target" SSH_USER)" + LAB_SSH_PASS="$(target_cfg "$target" SSH_PASS)" + LAB_REMOTE_DIR="$(target_cfg "$target" REMOTE_DIR)" + + local sudo_pass + sudo_pass="$(target_cfg "$target" SUDO_PASS)" + + LAB_SSH_PORT="${LAB_SSH_PORT:-22}" + sudo_pass="${sudo_pass:-$LAB_SSH_PASS}" + LAB_SUDO_PASS_B64="$(printf '%s' "$sudo_pass" | base64 -w0)" + LAB_REMOTE_DIR="${LAB_REMOTE_DIR:-/tmp/kmod-debug-lab}" + + local ssh_bin scp_bin sshpass_bin + ssh_bin="${SSH_BIN:-ssh}" + scp_bin="${SCP_BIN:-scp}" + sshpass_bin="${SSHPASS_BIN:-sshpass}" + + if [[ -n "$LAB_SSH_PASS" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then + die "TARGET_SSH_PASS[$target] is set but '$sshpass_bin' is not installed; install it with: sudo apt-get install -y sshpass" + fi + + LAB_SSH_TARGET="$LAB_SSH_USER@$LAB_SSH_HOST" + + # LAB_SSH_CMD and LAB_SCP_CMD are flag-only — they do NOT include the + # target. Helpers (lab_ssh, lab_scp_to) and direct callers append the + # target themselves. This matters because ssh treats anything after the + # host as the remote command, so flags like `-t` must be inserted before + # the target. + LAB_SSH_CMD=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$LAB_SSH_PORT") + LAB_SCP_CMD=("$scp_bin" -o StrictHostKeyChecking=accept-new -P "$LAB_SSH_PORT") + LAB_RSYNC_RSH="$ssh_bin -o StrictHostKeyChecking=accept-new -p $LAB_SSH_PORT" + if [[ -n "$LAB_SSH_PASS" ]]; then + LAB_SSH_CMD=("$sshpass_bin" -e "${LAB_SSH_CMD[@]}") + LAB_SCP_CMD=("$sshpass_bin" -e "${LAB_SCP_CMD[@]}") + LAB_RSYNC_RSH="$sshpass_bin -e $LAB_RSYNC_RSH" + fi +} + +# --- Remote command helpers ----------------------------------------------- +# +# All of these require a prior `lab_load_target` call. + +# Emit a shell prefix that pipes the cached sudo password into `sudo -S`. Used +# inside remote command strings, e.g. +# lab_ssh "$(lab_remote_sudo_prefix)cat /usr/lib/debug/boot/vmlinux-*" +# When no password was configured, the prefix is plain `sudo `. +lab_remote_sudo_prefix() { + if [[ -n "$LAB_SUDO_PASS_B64" ]]; then + printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$LAB_SUDO_PASS_B64" + else + printf "sudo " + fi +} + +# Run a command on the target. Arguments after the function name are passed +# through to ssh as the remote command (typically one quoted shell string). +lab_ssh() { + SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "$@" +} + +# Run a command on the target with the sudo prefix prepended. Arguments are +# concatenated into a single shell string after the prefix. +lab_ssh_sudo() { + lab_ssh "$(lab_remote_sudo_prefix)$*" +} + +# Copy a local file to the target. The remote path is absolute. +lab_scp_to() { + local local_path="$1" + local remote_path="$2" + SSHPASS="$LAB_SSH_PASS" "${LAB_SCP_CMD[@]}" "$local_path" "$LAB_SSH_TARGET:$remote_path" +} + +# Rsync a remote path (file or dir) into a local destination. Extra rsync +# args can be appended. +lab_rsync_from() { + local remote_src="$1" + local local_dst="$2" + shift 2 + SSHPASS="$LAB_SSH_PASS" "${RSYNC_BIN:-rsync}" -a -e "$LAB_RSYNC_RSH" "$@" \ + "$LAB_SSH_TARGET:$remote_src" "$local_dst" } From 71b632055c7a044a9b543466121b067bd6fe1375 Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 15:42:15 +0300 Subject: [PATCH 10/13] fix scripts ssh password leak --- lab.example.env | 16 +++- scripts/00-check-target.sh | 151 ++++++++++++++++++++++++++++++ scripts/01-provision-target.sh | 1 + scripts/02-setup-host-build.sh | 9 +- scripts/04-deploy-debug-vscode.sh | 48 +++++++--- scripts/lib/common.sh | 106 +++++++++++++-------- 6 files changed, 275 insertions(+), 56 deletions(-) create mode 100755 scripts/00-check-target.sh diff --git a/lab.example.env b/lab.example.env index 23c7be6..498bfd6 100644 --- a/lab.example.env +++ b/lab.example.env @@ -41,13 +41,19 @@ declare -A TARGET_SSH_USER=( [server]=user ) -# Leave empty when using SSH keys. +# SSH password. Leave empty when using SSH keys (recommended for anything +# beyond throwaway dev VMs). The lab pipes any configured password into ssh +# via SSHPASS; it is not written to disk and not visible in `ps` on either +# end. declare -A TARGET_SSH_PASS=( [desktop]= [server]= ) -# Leave empty to reuse TARGET_SSH_PASS for sudo. +# Sudo password. Leave empty to reuse TARGET_SSH_PASS. The lab pipes this +# into `sudo -S` over ssh's stdin, so it is not visible in `ps` on the +# target. For long-lived dev targets, configure NOPASSWD for the lab user +# (` ALL=(ALL) NOPASSWD: ALL` in sudoers) and leave this empty. declare -A TARGET_SUDO_PASS=( [desktop]= [server]= @@ -55,9 +61,11 @@ declare -A TARGET_SUDO_PASS=( # Remote staging directory used for uploading the built module. Any writable # path is fine; /tmp/* gets cleaned on reboot which is usually what you want. +# Leave empty to use the per-user default `/tmp/kmod-debug-lab-`, +# which avoids collisions when multiple devs share a target host. declare -A TARGET_REMOTE_DIR=( - [desktop]=/tmp/kmod-debug-lab - [server]=/tmp/kmod-debug-lab + [desktop]= + [server]= ) # Debug endpoints — one per debug method. Scripts 01 and 04 take a diff --git a/scripts/00-check-target.sh b/scripts/00-check-target.sh new file mode 100755 index 0000000..c4dc5e9 --- /dev/null +++ b/scripts/00-check-target.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Non-mutating health check for a target. Run this before scripts/01 to +# catch configuration mistakes before any state on the target changes. +# +# Reports, one line each: +# ssh is the target reachable over ssh? +# sudo does the configured (or fallback) sudo work? +# os distro and codename (must be ubuntu for now) +# kernel running kernel release +# headers /lib/modules/$kernel/build present? +# vmlinux debug-symbol vmlinux available? +# kernel-source linux-source-* package extracted? +# debug endpoint TCP port for the picked debug method reachable from the dev host? +# +# A "debug method" argument is optional; if omitted, both kgdb and qemu +# endpoints are checked when configured. +set -euo pipefail + +usage() { + echo "usage: $0 [kgdb|qemu]" >&2 + exit 2 +} + +target="${1:-}" +debug_method="${2:-}" +case "$debug_method" in ""|kgdb|qemu) ;; *) usage ;; esac + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/lib/common.sh +source "$repo_root/scripts/lib/common.sh" +validate_target "$target" "usage: $0 [kgdb|qemu]" + +env_file="$(lab_env_file "$repo_root")" +# shellcheck source=/dev/null +source "$env_file" +lab_load_target "$target" + +# Colored OK / FAIL / WARN, falling back to plain text when stdout isn't a TTY. +if [[ -t 1 ]]; then + GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[0;33m'; RESET='\033[0m' +else + GREEN=''; RED=''; YELLOW=''; RESET='' +fi +ok() { printf " ${GREEN}OK${RESET} %s\n" "$1"; } +fail() { printf " ${RED}FAIL${RESET} %s\n" "$1"; failures=$((failures+1)); } +warn() { printf " ${YELLOW}WARN${RESET} %s\n" "$1"; } +failures=0 + +echo "checking $LAB_TARGET ($LAB_SSH_TARGET:$LAB_SSH_PORT)" + +# --- ssh --- +if lab_ssh 'true' 2>/dev/null; then + ok "ssh: reachable" +else + fail "ssh: cannot reach $LAB_SSH_TARGET on port $LAB_SSH_PORT" + echo + echo "fix one of:" + echo " - TARGET_SSH_HOST/PORT/USER for '$target' in lab.local.env" + echo " - the target VM is up and openssh-server is running" + echo " - if password auth: TARGET_SSH_PASS[$target] is set" + exit 1 +fi + +# --- sudo --- +if lab_ssh_sudo 'true' 2>/dev/null; then + ok "sudo: works" +else + fail "sudo: configured password rejected or sudo not installed" + echo + echo "fix one of:" + echo " - TARGET_SUDO_PASS[$target] in lab.local.env (defaults to TARGET_SSH_PASS)" + echo " - configure NOPASSWD for the lab user on the target" +fi + +# --- os --- +# shellcheck disable=SC2016 # $ID and $VERSION_CODENAME are intentionally evaluated on the remote. +remote_os="$(lab_ssh '. /etc/os-release 2>/dev/null && printf "%s %s" "$ID" "${VERSION_CODENAME:-unknown}"' || echo unknown)" +case "$remote_os" in + "ubuntu "*) ok "os: $remote_os" ;; + *) fail "os: $remote_os (only ubuntu targets are implemented)" ;; +esac + +# --- kernel --- +live_kernel="$(lab_ssh 'uname -r' 2>/dev/null || echo unknown)" +ok "kernel: $live_kernel" + +cache_dir="$repo_root/.kernel-cache/$target" +cached_kernel="$(cat "$cache_dir/kernel.release" 2>/dev/null || echo)" +if [[ -n "$cached_kernel" && "$cached_kernel" != "$live_kernel" ]]; then + warn "cached kernel ($cached_kernel) differs from live; re-run scripts/02-setup-host-build.sh $target" +fi + +# --- headers --- +if lab_ssh "test -d /lib/modules/$live_kernel/build" 2>/dev/null; then + ok "headers: /lib/modules/$live_kernel/build present" +else + warn "headers: missing on target; run scripts/01-provision-target.sh $target " +fi + +# --- vmlinux (debug image) --- +if lab_ssh_sudo "test -r /usr/lib/debug/boot/vmlinux-$live_kernel" 2>/dev/null; then + ok "vmlinux: /usr/lib/debug/boot/vmlinux-$live_kernel readable (debug symbols installed)" +else + warn "vmlinux: missing on target; for source debugging, re-run scripts/01-... with --debug-symbols" +fi + +# --- kernel source --- +src_found="$(lab_ssh 'ls -1d /usr/src/linux-source-*/ 2>/dev/null | head -1 || true' 2>/dev/null)" +if [[ -n "$src_found" ]]; then + ok "kernel-source: ${src_found%/} present (step-into-kernel will resolve source)" +else + warn "kernel-source: missing on target; for step-into-kernel, re-run scripts/01-... with --debug-symbols" +fi + +# --- debug endpoint(s) reachable from this host --- +check_endpoint() { + local label="$1" key="$2" + local endpoint + endpoint="$(target_cfg "$target" "$key")" + if [[ -z "$endpoint" ]]; then + warn "$label: TARGET_${key}[$target] is unset (skip if you don't use this method)" + return + fi + local host="${endpoint%:*}" port="${endpoint##*:}" + if command -v nc >/dev/null 2>&1; then + if nc -z -w 3 "$host" "$port" 2>/dev/null; then + ok "$label: $endpoint reachable from this host" + else + warn "$label: $endpoint NOT reachable from this host (start your GDB stub / serial bridge)" + fi + else + warn "$label: cannot test (nc not installed); endpoint configured as $endpoint" + fi +} + +case "$debug_method" in + kgdb) check_endpoint "debug-kgdb" DEBUG_ENDPOINT_KGDB ;; + qemu) check_endpoint "debug-qemu" DEBUG_ENDPOINT_QEMU ;; + "") + check_endpoint "debug-kgdb" DEBUG_ENDPOINT_KGDB + check_endpoint "debug-qemu" DEBUG_ENDPOINT_QEMU + ;; +esac + +echo +if [[ $failures -eq 0 ]]; then + echo "all checks passed; ready to run: scripts/01-provision-target.sh $target [--debug-symbols]" +else + echo "$failures check(s) FAILED; resolve before running scripts/01." + exit 1 +fi diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index 87217c5..eeb709f 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -46,6 +46,7 @@ esac symbols_blurb="${symbols:+, with debug symbols}" echo "provisioning $target at $LAB_SSH_TARGET:$LAB_SSH_PORT (debug method: $debug_method$symbols_blurb)" +lab_check_connection SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" -t "$LAB_SSH_TARGET" \ "TARGET_OS='$target_os' DEBUG_METHOD='$debug_method' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$LAB_SUDO_PASS_B64' DEBUG_MAXCPUS='${DEBUG_MAXCPUS:-}' bash -s" <<'REMOTE' diff --git a/scripts/02-setup-host-build.sh b/scripts/02-setup-host-build.sh index 3d20fdd..8bcec0c 100755 --- a/scripts/02-setup-host-build.sh +++ b/scripts/02-setup-host-build.sh @@ -62,6 +62,7 @@ apt_get install -y \ # --- Discover the target's kernel + header layout ------------------------- echo "[2/5] discovering target kernel" +lab_check_connection kernel="$(lab_ssh 'uname -r')" echo " target $target is running $kernel" @@ -169,8 +170,12 @@ ln -s "usr-src/$build_name" "$build_dir" echo "[4/5] checking vmlinux debug image" if lab_ssh_sudo "test -r '$remote_vmlinux'"; then - echo " copying $remote_vmlinux" - lab_ssh_sudo "cat '$remote_vmlinux'" > "$cache_dir/vmlinux" + # Use rsync over sudo: skips when unchanged, restartable, checksummed. + # Without this the cat-based fetch would silently corrupt vmlinux any + # time sudo printed anything unexpected to stdout or the SSH connection + # dropped mid-transfer (the file is ~415MB so the window is non-trivial). + echo " rsync $remote_vmlinux -> ${cache_dir#"$repo_root"/}/vmlinux" + lab_rsync_from "$remote_vmlinux" "$cache_dir/vmlinux" --rsync-path='sudo rsync' else rm -f "$cache_dir/vmlinux" echo " no vmlinux on target ($remote_vmlinux is missing)" diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index 1e01e95..109e593 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -63,6 +63,21 @@ vmlinux="$cache_dir/vmlinux" scripts/02-setup-host-build.sh $target" kernel="$(<"$kernel_file")" + +# Preflight: connection + sudo work, and the target is still running the +# kernel we cached. If the target rebooted into a different kernel since +# scripts/02 ran, vmlinux symbols won't match runtime addresses and GDB +# would silently show wrong source lines. +lab_check_connection +live_kernel="$(lab_ssh 'uname -r')" +if [[ "$live_kernel" != "$kernel" ]]; then + die "kernel mismatch: target is now running '$live_kernel' but cache has '$kernel'. +- if the target was rebooted into a new kernel, re-sync: + scripts/01-provision-target.sh $target $debug_method --debug-symbols # if you also need the new vmlinux + scripts/02-setup-host-build.sh $target +- if you booted the wrong kernel, reboot back into $kernel" +fi + artifact_dir="$repo_root/build/artifacts/$target/$kernel" intermediate_dir="$repo_root/build/intermediate/$target/$kernel" module_dir="$repo_root/module" @@ -99,15 +114,14 @@ rm -f "$symbols_file" "$loader_log" # accept queue and blocks fresh attaches. We best-effort kill any local gdb # process that holds a connection to the endpoint. Errors here never fail # the deploy — at worst the user has to `killall gdb` manually. +# +# `lsof -ti` is more robust than parsing `ss -ntp` output: it gives one PID +# per line and won't break across iproute2 versions. free_debug_endpoint() { - command -v ss >/dev/null 2>&1 || return 0 + command -v lsof >/dev/null 2>&1 || return 0 local host="${debug_endpoint%:*}" port="${debug_endpoint##*:}" local pids - pids="$(ss -ntp 2>/dev/null \ - | grep -F "$host:$port" \ - | grep -oE 'pid=[0-9]+' \ - | cut -d= -f2 \ - | sort -u || true)" + pids="$(lsof -ti "@$host:$port" 2>/dev/null || true)" local gdb_pids=() local pid for pid in $pids; do @@ -129,6 +143,12 @@ remote_module="$LAB_REMOTE_DIR/$module_name.ko" echo "uploading $artifact -> $LAB_SSH_TARGET:$remote_module" lab_ssh "mkdir -p '$LAB_REMOTE_DIR'" lab_scp_to "$artifact" "$remote_module" +# Verify the upload — a truncated .ko would silently insmod-fail with cryptic +# "invalid module format" errors. +local_size="$(stat -c %s "$artifact")" +remote_size="$(lab_ssh "stat -c %s '$remote_module' 2>/dev/null" | tr -d '[:space:]')" +[[ "$remote_size" == "$local_size" ]] || + die "upload size mismatch: local $local_size bytes, remote ${remote_size:-missing} bytes; retry scripts/04 or check disk space on the target" # Load the module and discover the addresses /sys/module//sections/ # exposes once it's live. The insmod runs inside a single nohup'd sh -c so @@ -142,10 +162,15 @@ lab_scp_to "$artifact" "$remote_module" load_and_discover_symbols() { echo "loading $module_name on $target (debug_delay_ms=$debug_delay_ms)" lab_ssh_sudo "rmmod '$module_name' >/dev/null 2>&1 || true" - # nohup must wrap the sudo invocation so the load survives the SSH - # session closing; we keep the sudo prefix inline rather than going - # through lab_ssh_sudo here. - lab_ssh "nohup $(lab_remote_sudo_prefix)sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$LAB_REMOTE_DIR/insmod.log' 2>&1 &" + # Background insmod on the target so we can poll /sys/module/.../sections + # in parallel with the module's debug_delay_ms sleep. `& sleep 1` keeps + # the SSH session open just long enough for sudo to read the password + # from stdin and exec into nohup; without it, ssh can close before sudo + # finishes authenticating and the load silently aborts. The orphaned + # nohup+sh+insmod chain survives session exit (non-interactive bash + # doesn't huponexit). The `|| insmod ...` falls back when the module + # doesn't declare a debug_delay_ms parameter. + lab_ssh_sudo "nohup sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$LAB_REMOTE_DIR/insmod.log' 2>&1 & sleep 1" echo "polling /sys/module/$module_name/sections/ (timeout ${insmod_wait_secs}s)" @@ -183,7 +208,8 @@ load_and_discover_symbols() { return 0 fi fi - sleep 0.2 + # No explicit sleep — each SSH round-trip already takes 0.3-1s, which + # is the right polling cadence. done rm -f "$tmp" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index a16a084..373fe3c 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -6,9 +6,10 @@ # lab_* higher-level helpers; rely on globals # populated by `lab_load_target`. # LAB_* globals populated by `lab_load_target`. -# Scripts should call lab_ssh / lab_scp_to -# / lab_rsync_from rather than touching -# the underlying ssh/scp/sshpass binaries. +# Scripts should call lab_ssh / lab_ssh_sudo +# / lab_scp_to / lab_rsync_from rather than +# touching the underlying ssh/scp/sshpass +# binaries directly. # # Sourcing this file is side-effect-free. State is created when a script # calls `lab_load_target ` after sourcing `lab.local.env`. @@ -91,12 +92,12 @@ lab_env_file() { # LAB_TARGET target name # LAB_SSH_HOST/PORT/USER raw connection details # LAB_SSH_PASS ssh password ("" when using keys) -# LAB_SUDO_PASS_B64 base64-encoded sudo password (for piping into sudo -S) -# LAB_REMOTE_DIR target staging dir (with default) +# LAB_SUDO_PASS sudo password ("" when sudo is passwordless / NOPASSWD) +# LAB_REMOTE_DIR target staging dir (with per-user default) # LAB_SSH_TARGET "user@host" form for use in commands -# LAB_SSH_CMD array, ready to invoke (includes sshpass wrapping) -# LAB_SCP_CMD array, ready to invoke (includes sshpass wrapping) -# LAB_RSYNC_RSH rsync -e value (includes sshpass wrapping) +# LAB_SSH_CMD array, ready to invoke (includes sshpass + timeouts) +# LAB_SCP_CMD array, ready to invoke (includes sshpass + timeouts) +# LAB_RSYNC_RSH rsync -e value (includes sshpass + timeouts) lab_load_target() { local target="$1" target_is_configured "$target" @@ -107,15 +108,14 @@ lab_load_target() { LAB_SSH_PORT="$(target_cfg "$target" SSH_PORT)" LAB_SSH_USER="$(target_require_cfg "$target" SSH_USER)" LAB_SSH_PASS="$(target_cfg "$target" SSH_PASS)" + LAB_SUDO_PASS="$(target_cfg "$target" SUDO_PASS)" LAB_REMOTE_DIR="$(target_cfg "$target" REMOTE_DIR)" - local sudo_pass - sudo_pass="$(target_cfg "$target" SUDO_PASS)" - LAB_SSH_PORT="${LAB_SSH_PORT:-22}" - sudo_pass="${sudo_pass:-$LAB_SSH_PASS}" - LAB_SUDO_PASS_B64="$(printf '%s' "$sudo_pass" | base64 -w0)" - LAB_REMOTE_DIR="${LAB_REMOTE_DIR:-/tmp/kmod-debug-lab}" + # Sudo password falls back to ssh password — common single-user dev case. + LAB_SUDO_PASS="${LAB_SUDO_PASS:-$LAB_SSH_PASS}" + # Per-user default so two devs on the same host don't collide on /tmp. + LAB_REMOTE_DIR="${LAB_REMOTE_DIR:-/tmp/kmod-debug-lab-$LAB_SSH_USER}" local ssh_bin scp_bin sshpass_bin ssh_bin="${SSH_BIN:-ssh}" @@ -129,13 +129,22 @@ lab_load_target() { LAB_SSH_TARGET="$LAB_SSH_USER@$LAB_SSH_HOST" # LAB_SSH_CMD and LAB_SCP_CMD are flag-only — they do NOT include the - # target. Helpers (lab_ssh, lab_scp_to) and direct callers append the - # target themselves. This matters because ssh treats anything after the - # host as the remote command, so flags like `-t` must be inserted before - # the target. - LAB_SSH_CMD=("$ssh_bin" -o StrictHostKeyChecking=accept-new -p "$LAB_SSH_PORT") - LAB_SCP_CMD=("$scp_bin" -o StrictHostKeyChecking=accept-new -P "$LAB_SSH_PORT") - LAB_RSYNC_RSH="$ssh_bin -o StrictHostKeyChecking=accept-new -p $LAB_SSH_PORT" + # target. Helpers and direct callers append the target themselves so + # `ssh ... -t user@host cmd` works (anything after the host is treated + # as the remote command by ssh). + # + # ConnectTimeout fails fast (10s) if the target is unreachable instead + # of blocking for minutes. ServerAliveInterval keeps long-lived + # connections (e.g. the 30s symbol-discovery loop) from being dropped + # by NAT idle timers. + local ssh_opts=( + -o StrictHostKeyChecking=accept-new + -o ConnectTimeout=10 + -o ServerAliveInterval=15 + ) + LAB_SSH_CMD=("$ssh_bin" "${ssh_opts[@]}" -p "$LAB_SSH_PORT") + LAB_SCP_CMD=("$scp_bin" "${ssh_opts[@]}" -P "$LAB_SSH_PORT") + LAB_RSYNC_RSH="$ssh_bin ${ssh_opts[*]} -p $LAB_SSH_PORT" if [[ -n "$LAB_SSH_PASS" ]]; then LAB_SSH_CMD=("$sshpass_bin" -e "${LAB_SSH_CMD[@]}") LAB_SCP_CMD=("$sshpass_bin" -e "${LAB_SCP_CMD[@]}") @@ -147,28 +156,26 @@ lab_load_target() { # # All of these require a prior `lab_load_target` call. -# Emit a shell prefix that pipes the cached sudo password into `sudo -S`. Used -# inside remote command strings, e.g. -# lab_ssh "$(lab_remote_sudo_prefix)cat /usr/lib/debug/boot/vmlinux-*" -# When no password was configured, the prefix is plain `sudo `. -lab_remote_sudo_prefix() { - if [[ -n "$LAB_SUDO_PASS_B64" ]]; then - printf "printf '%%s\\n' \"\$(printf '%%s' '%s' | base64 -d)\" | sudo -S " "$LAB_SUDO_PASS_B64" - else - printf "sudo " - fi -} - -# Run a command on the target. Arguments after the function name are passed -# through to ssh as the remote command (typically one quoted shell string). +# Run a command on the target. Arguments are passed through to ssh as the +# remote command (typically one quoted shell string). Inherits stdin from +# the caller — handy when callers want to pipe data in (e.g. lab_ssh_sudo +# pipes the sudo password). lab_ssh() { SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "$@" } -# Run a command on the target with the sudo prefix prepended. Arguments are -# concatenated into a single shell string after the prefix. +# Run a command on the target as root. The sudo password reaches the remote +# `sudo -S` via ssh's stdin, which means it never appears in any process's +# argv on the target — `ps auxww` is clean. When LAB_SUDO_PASS is empty +# (NOPASSWD sudo or sudo configured for passwordless), the empty string is +# piped and sudo proceeds without prompting. +# +# Callers that need to feed their own stdin to the remote command (e.g. +# rsync) should use `lab_rsync_from` / scp helpers instead — those take a +# different path that doesn't compete for stdin. lab_ssh_sudo() { - lab_ssh "$(lab_remote_sudo_prefix)$*" + printf '%s\n' "$LAB_SUDO_PASS" | + SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "sudo -S -p '' $*" } # Copy a local file to the target. The remote path is absolute. @@ -179,7 +186,8 @@ lab_scp_to() { } # Rsync a remote path (file or dir) into a local destination. Extra rsync -# args can be appended. +# args can be appended — notably `--rsync-path='sudo rsync'` for paths only +# root can read (vmlinux), and `--delete` for tree mirroring. lab_rsync_from() { local remote_src="$1" local local_dst="$2" @@ -187,3 +195,23 @@ lab_rsync_from() { SSHPASS="$LAB_SSH_PASS" "${RSYNC_BIN:-rsync}" -a -e "$LAB_RSYNC_RSH" "$@" \ "$LAB_SSH_TARGET:$remote_src" "$local_dst" } + +# --- Preflight ------------------------------------------------------------ + +# Fast, non-mutating connection check. Verifies SSH reachable, sudo works. +# Called at the top of 01/02/04 so failures surface before any real work. +# `scripts/00-check-target.sh` calls a longer-form report on top of this. +lab_check_connection() { + if ! lab_ssh 'true' 2>/dev/null; then + die "cannot reach $LAB_SSH_TARGET over ssh (port $LAB_SSH_PORT). +- check TARGET_SSH_HOST/PORT/USER for '$LAB_TARGET' in lab.local.env +- if the VM is up, try by hand: ssh -p $LAB_SSH_PORT $LAB_SSH_TARGET +- if you're using passwords, confirm TARGET_SSH_PASS[$LAB_TARGET] is set" + fi + if ! lab_ssh_sudo 'true' 2>/dev/null; then + die "ssh reaches $LAB_SSH_TARGET but sudo doesn't work there. +- check TARGET_SUDO_PASS[$LAB_TARGET] in lab.local.env (defaults to TARGET_SSH_PASS) +- on the target, confirm: sudo -n -v (or run sudo by hand once to cache creds) +- consider configuring NOPASSWD for the lab user on long-lived dev targets" + fi +} From 9a074eb236dd73c91e9873212b212bbdf856d1dc Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 19:46:59 +0300 Subject: [PATCH 11/13] reame fixes, sudo pass issue fix in scripts --- .github/workflows/lint.yml | 66 ++++++++++- .gitignore | 5 +- .vscode/tasks.json | 6 + LICENSE | 2 +- Makefile | 43 ++++++- README.md | 87 +++++++++++--- examples/chuck_norise/README.md | 11 +- module/README.md | 24 +++- scripts/00-check-target.sh | 3 +- scripts/01-provision-target.sh | 181 ++++++++++++++++++++++-------- scripts/02-setup-host-build.sh | 17 ++- scripts/03-build-module.sh | 3 +- scripts/04-deploy-debug-vscode.sh | 29 +++-- scripts/lib/common.sh | 25 ++++- 14 files changed, 400 insertions(+), 102 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 32bf870..8c84bff 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: lint +name: ci on: push: @@ -48,3 +48,67 @@ jobs: - name: make help (dry-parse) run: make help + + build: + name: build example module against runner kernel (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + # Surface drift across LTS kernels. ubuntu-22.04 ships 5.15; 24.04 ships 6.8. + os: [ubuntu-22.04, ubuntu-24.04] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: install host kernel headers + build tools + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + bc bison flex libelf-dev libssl-dev \ + "linux-headers-$(uname -r)" + + - name: use the example module + run: make use-example NAME=chuck_norise + + - name: build via top-level Makefile (default target) + # KDIR defaults to /lib/modules/$(uname -r)/build when there's no + # .kernel-cache/current, so this exercises the full prepare-build + + # KCFLAGS pipeline against a real (the runner's) kernel. + run: make modules + + - name: assert a .ko was produced + run: | + set -e + shopt -s nullglob + kos=(build/artifacts/*/*/*.ko) + if [ "${#kos[@]}" -eq 0 ]; then + echo "::error::no .ko produced under build/artifacts/" + ls -R build/ || true + exit 1 + fi + for ko in "${kos[@]}"; do + echo "built $ko ($(stat -c %s "$ko") bytes)" + file "$ko" + done + + - name: verify DWARF prefix-map rewrote paths back to module/ + # The whole point of -ffile-prefix-map is that DWARF references your + # source under module/<...>.c, not build/intermediate/<...>.c. If a + # change to the Makefile breaks the injection, this catches it. + run: | + set -e + ko=$(find build/artifacts -name '*.ko' | head -1) + if objdump --dwarf=decodedline "$ko" 2>/dev/null \ + | grep -qE '/build/intermediate/'; then + echo "::error::DWARF still references build/intermediate/ — prefix-map broken" + objdump --dwarf=decodedline "$ko" 2>/dev/null | grep -E '/build/intermediate/' | head + exit 1 + fi + echo "ok: DWARF paths point at module/" + + - name: clean-module resets cleanly + run: | + make clean-module + ls module/ + [ "$(ls module/)" = "README.md" ] || { echo "::error::clean-module did not reduce module/ to just README.md"; exit 1; } diff --git a/.gitignore b/.gitignore index 16fdcb6..aa02f4b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ Module.symvers modules.order # IDE / editor noise. -compile_commands.json .vscode/ipch/ .codex .claude/ @@ -29,6 +28,4 @@ compile_commands.json # out of the public repo so workflow guidance stays separate from user docs). CLAUDE.md session.md - -# Stray scratch. -kernel.gdb +todo.md diff --git a/.vscode/tasks.json b/.vscode/tasks.json index b9190e9..69e19f9 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -19,6 +19,12 @@ } ], "tasks": [ + { + "label": "Kernel: Check Target", + "type": "shell", + "command": "./scripts/00-check-target.sh ${input:kernelTarget} ${input:debugMethod}", + "problemMatcher": [] + }, { "label": "Kernel: Provision Target", "type": "shell", diff --git a/LICENSE b/LICENSE index 4db60c3..b77f1ff 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 Dor Grosglik and contributors +Copyright (c) 2026 Dor Grosglick and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 3077963..7560971 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ MODULE_DIR ?= $(CURDIR)/module EXTRA_CCFLAGS ?= -g -DDEBUG KCFLAGS_INJECT := -ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR) $(EXTRA_CCFLAGS) -.PHONY: all modules prepare-build clean help +.PHONY: all modules prepare-build clean help use-example clean-module all: modules @@ -42,11 +42,15 @@ help: @echo "Normally invoked by scripts/03-build-module.sh , which sets" @echo "KDIR, BUILD_ID, INTERMEDIATE_DIR, and ARTIFACT_DIR to per-target paths." @echo "" - @echo "Targets:" + @echo "Build targets:" @echo " modules build module/ against KDIR (default)" @echo " clean remove intermediate and artifact dirs for BUILD_ID" @echo " prepare-build stage module/ into INTERMEDIATE_DIR (internal)" @echo "" + @echo "module/ ergonomics:" + @echo " use-example NAME=chuck_norise copy an example into module/" + @echo " clean-module reset module/ to just README.md" + @echo "" @echo "Vars (auto-resolved when .kernel-cache/current points at a target):" @echo " MODULE_DIR $(MODULE_DIR)" @echo " KDIR $(KDIR)" @@ -55,6 +59,39 @@ help: @echo " ARTIFACT_DIR $(ARTIFACT_DIR)" @echo " EXTRA_CCFLAGS $(EXTRA_CCFLAGS) (appended to KCFLAGS)" +# Copy examples/$(NAME)/ into module/. Refuses if module/ already contains a +# Makefile/Kbuild (use clean-module first). NAME=chuck_norise unless overridden. +NAME ?= chuck_norise +use-example: + @if [ ! -d "$(CURDIR)/examples/$(NAME)" ]; then \ + echo "Makefile: examples/$(NAME) does not exist;" >&2; \ + echo " available examples:" >&2; \ + ls $(CURDIR)/examples 2>/dev/null | sed 's/^/ /' >&2; \ + exit 1; \ + fi + @if [ -f "$(MODULE_DIR)/Makefile" ] || [ -f "$(MODULE_DIR)/Kbuild" ]; then \ + echo "Makefile: $(MODULE_DIR) already has a Makefile/Kbuild." >&2; \ + echo " Run 'make clean-module' first if you want to replace it." >&2; \ + exit 1; \ + fi + @echo "copying examples/$(NAME)/* -> $(MODULE_DIR)/ (keeping module/README.md placeholder)" + @find "$(CURDIR)/examples/$(NAME)" -mindepth 1 -maxdepth 1 -not -name README.md \ + -exec cp -r {} "$(MODULE_DIR)/" \; + @echo "done. next: scripts/03-build-module.sh " + @echo "(the example's docs stay at examples/$(NAME)/README.md)" + +# Remove everything from module/ except its placeholder README, so you can +# drop a fresh module project in. Refuses to touch tracked files outside +# module/. +clean-module: + @if [ ! -d "$(MODULE_DIR)" ]; then \ + echo "Makefile: $(MODULE_DIR) is missing — nothing to clean." >&2; \ + exit 1; \ + fi + @find "$(MODULE_DIR)" -mindepth 1 -maxdepth 1 -not -name README.md -print0 \ + | xargs -0 -r rm -rf + @echo "module/ reset (kept only README.md)" + modules: prepare-build $(MAKE) -C "$(KDIR)" M="$(INTERMEDIATE_DIR)" KCFLAGS="$(KCFLAGS_INJECT)" modules @mkdir -p "$(ARTIFACT_DIR)" @@ -80,7 +117,7 @@ prepare-build: @if [ ! -e "$(MODULE_DIR)/Makefile" ] && [ ! -e "$(MODULE_DIR)/Kbuild" ]; then \ echo "Makefile: no Makefile or Kbuild under $(MODULE_DIR);" >&2; \ echo " drop your kernel module project there (see $(MODULE_DIR)/README.md)" >&2; \ - echo " or try the example: cp -r examples/chuck_norise/. module/" >&2; \ + echo " or try the example: make use-example NAME=chuck_norise" >&2; \ exit 1; \ fi @cp --help 2>&1 | grep -q -- '--symbolic-link' || { \ diff --git a/README.md b/README.md index 1665d5e..538ee56 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ code. ## Quickstart You need a **Debian-based dev host** (Debian, Ubuntu, WSL Ubuntu, …) and -at least one **Ubuntu target VM** reachable over SSH. Five steps from a -fresh clone to a stopped breakpoint: +at least one **Ubuntu target VM** reachable over SSH. From fresh clone +to a stopped breakpoint: ```bash # 0. clone + configure @@ -37,6 +37,9 @@ git clone kmod-debug-lab && cd kmod-debug-lab cp lab.example.env lab.local.env $EDITOR lab.local.env # set TARGET_SSH_*, endpoints, etc. +# 0.5 (optional) sanity-check the target before changing anything on it +./scripts/00-check-target.sh server qemu + # 1. one-time target setup (installs headers, vmlinux dbg, kernel source) ./scripts/01-provision-target.sh server qemu --debug-symbols # ...reboot the target VM so new GRUB args take effect... @@ -45,7 +48,7 @@ $EDITOR lab.local.env # set TARGET_SSH_*, endpoints, etc. ./scripts/02-setup-host-build.sh server # 3. try the example module (replace with your own when ready) -cp -r examples/chuck_norise/. module/ +make use-example NAME=chuck_norise ./scripts/03-build-module.sh server # 4. deploy + load on the target @@ -63,22 +66,30 @@ bind. You're stepping through kernel code from your module. ## How the pieces fit ``` -module/ your module project (Makefile + sources) -examples/chuck_norise/ worked example you can copy into module/ - -scripts/01-provision-target.sh target-side: install headers, vmlinux dbg, source, GRUB args -scripts/02-setup-host-build.sh dev-host: sync headers + vmlinux + kernel source into .kernel-cache/ -scripts/03-build-module.sh dev-host: build module/ via Kbuild against the synced headers -scripts/04-deploy-debug-vscode.sh dev-host: upload, insmod, write .gdb/-.gdb - -.kernel-cache// per-target build/source/vmlinux cache (gitignored) -build/ per-target intermediate + final .ko (gitignored) -.gdb/ generated GDB init files (gitignored, regenerated on F5) - -host/ optional helpers for VMware-on-Windows users -lab.example.env -> lab.local.env per-machine config (gitignored copy) +module/ your module project (Makefile + sources) +examples/chuck_norise/ worked example; copy into module/ with `make use-example` + +scripts/00-check-target.sh non-mutating preflight: ssh / sudo / headers / vmlinux / endpoint +scripts/01-provision-target.sh target-side: install headers, vmlinux dbg, source, GRUB args + (also `--uninstall` to undo the GRUB args) +scripts/02-setup-host-build.sh dev-host: sync headers + vmlinux + kernel source into .kernel-cache/ +scripts/03-build-module.sh dev-host: build module/ via Kbuild against the synced headers +scripts/04-deploy-debug-vscode.sh dev-host: upload, insmod, write .gdb/-.gdb +scripts/lib/common.sh shared bash helpers (lab_ssh, lab_ssh_sudo, lab_rsync_from, …) + +.kernel-cache// per-target build/source/vmlinux cache (gitignored) +build/ per-target intermediate + final .ko (gitignored) +.gdb/ generated GDB init files (gitignored, regenerated on F5) + +host/ optional helpers for VMware-on-Windows users +lab.example.env -> lab.local.env per-machine config (gitignored copy) ``` +Top-level `Makefile` exposes a few non-build targets too: +`make use-example NAME=` (copy `examples//` into `module/`), +`make clean-module` (reset `module/` to just its placeholder README), +`make help` (full var/target listing). + The numbered scripts are designed to be run in order. After the one-time 01 + 02 against a target, the inner loop is **03 → 04 → F5** (or just F5, which runs 04 as a prelaunch task and rebuilds via 03 if you ran it @@ -263,6 +274,22 @@ disassembly when stepping into kernel functions. Check --- +## Undoing target-side changes + +Script 01 modifies the target's GRUB command line. To remove every boot +arg the lab added (`nokaslr`, `kgdboc=...`, `sysrq_always_enabled=1`, +`maxcpus=...`) and restore a clean kernel command line: + +```bash +./scripts/01-provision-target.sh server --uninstall +``` + +This leaves installed packages alone — only the GRUB args are reverted. +A timestamped backup of `/etc/default/grub` is left on the target +(`/etc/default/grub.kmod-debug-lab..bak`) for paranoid recovery. + +--- + ## Build outputs ``` @@ -326,6 +353,32 @@ module either failed to load, or it's slow to init on the target. The loader log will show `insmod` output. Bump the timeout with `INSMOD_WAIT_SECS=60 ./scripts/04-... ...` if needed. +**GDB shows `Cannot access memory at address …` when stepping into the +kernel.** Either `vmlinux` is missing (re-run `scripts/01-... --debug-symbols` +then `scripts/02-...`) or the cached `vmlinux` belongs to a different +kernel than the one running on the target. The latter is caught by +script 04's stale-kernel guard, but if you bypassed it, re-sync. + +**Spaces in your repo path.** Clone to a path WITHOUT spaces. GDB's +`set substitute-path FROM TO` splits on whitespace, so a `repo_root` like +`/home/me/path with spaces/...` produces a substitute-path that gets +parsed as four arguments and silently fails to remap module sources. +Kbuild is also notoriously fragile with spaces in paths. + +--- + +## Tool versions + +Tested with: + +- **Dev host:** Ubuntu 24.04, bash 5+, GNU coreutils, GDB ≥ 10, gcc 13. +- **Target VM:** Ubuntu 22.04 / 24.04, kernel 5.15 / 6.8. +- The `cp -as` recursive-symlink staging assumes GNU coreutils on the dev + host; macOS would need `brew install coreutils` and a `CP=gcp` override. +- `set substitute-path` for kernel-source remapping needs GDB ≥ 7.5 + (every supported distro ships much newer). +- `-ffile-prefix-map` (DWARF rewriting) requires GCC ≥ 8 or Clang ≥ 7. + --- ## Adding a new target OS diff --git a/examples/chuck_norise/README.md b/examples/chuck_norise/README.md index 0576a73..92b2a7d 100644 --- a/examples/chuck_norise/README.md +++ b/examples/chuck_norise/README.md @@ -13,15 +13,16 @@ userspace buffer" routine. ## Use it as a template The repo's build pipeline operates on whatever lives under `module/` at the -repo root. To try this example end-to-end, copy it into place: +repo root. To try this example end-to-end, copy it into place from the +repo root: ```bash -cp -r examples/chuck_norise/. module/ +make use-example NAME=chuck_norise ``` -Then run the normal lab flow from the repo root: `scripts/01-...` through -`scripts/04-...`, or press F5 in VS Code. After the module loads on the -target VM: +Then run the normal lab flow (`scripts/00-...` through `scripts/04-...`, +or press F5 in VS Code — see the top-level [README](../../README.md) for +the full quickstart). After the module loads on the target VM: ```bash head -c 10 /dev/chuck_norise # prints "chuck nori" diff --git a/module/README.md b/module/README.md index fcca4d9..3026dcd 100644 --- a/module/README.md +++ b/module/README.md @@ -49,13 +49,27 @@ path rewriting, uploading, loading, and symbol discovery on the target. ## Try the example ```bash -cp -r examples/chuck_norise/. module/ +make use-example NAME=chuck_norise ``` -Then build and debug as in the top-level `README.md`. +Then build and debug as in the top-level `README.md`. To reset: + +```bash +make clean-module +``` ## Replacing the example with your own module -Delete everything under `module/` except this README (or replace this -README too — the lab does not depend on it). Drop your project in. Edit -your `Makefile` to set `obj-m`, `-y`, and `ccflags-y`. Rebuild. +Run `make clean-module`, then drop your project in. Edit your `Makefile` +to set `obj-m`, `-y`, and `ccflags-y`. Rebuild. + +## One caveat: no symlinks inside `module/` + +The lab stages your sources into `build/intermediate///` +as a tree of absolute symlinks (`cp -as`) and then runs Kbuild from +there. If `module/` contains its OWN symlinks, the staged copy ends up +with symlinks pointing at the original symlinks — Kbuild will still find +the files, but the DWARF prefix-map remap (which assumes source files +live under `module/`) may not produce the paths your IDE expects. Keep +`module/` symlink-free, or copy in real files for any external sources +you depend on. diff --git a/scripts/00-check-target.sh b/scripts/00-check-target.sh index c4dc5e9..a286470 100755 --- a/scripts/00-check-target.sh +++ b/scripts/00-check-target.sh @@ -144,7 +144,8 @@ esac echo if [[ $failures -eq 0 ]]; then - echo "all checks passed; ready to run: scripts/01-provision-target.sh $target [--debug-symbols]" + echo "all checks passed." + echo "next: scripts/01-provision-target.sh $target [--debug-symbols]" else echo "$failures check(s) FAILED; resolve before running scripts/01." exit 1 diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index eeb709f..c01de37 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -6,6 +6,9 @@ # step-into-kernel, and update the guest's GRUB command line with the boot # args this lab needs (nokaslr always; kgdboc + sysrq for the kgdb method). # +# With --uninstall, remove every boot arg the lab added (nokaslr / kgdboc / +# sysrq_always_enabled / maxcpus) — but leave installed packages alone. +# # Reboot the target after this script completes so the new boot args take # effect. The script itself does not reboot — staying out of the user's way # matters for VMs that are reverted from snapshots and shouldn't be touched. @@ -13,14 +16,33 @@ set -euo pipefail usage() { echo "usage: $0 [--debug-symbols]" >&2 + echo " $0 --uninstall" >&2 exit 2 } target="${1:-}" -debug_method="${2:-}" -symbols="${3:-}" -case "$debug_method" in kgdb|qemu) ;; *) usage ;; esac -case "$symbols" in ""|--debug-symbols) ;; *) usage ;; esac +arg2="${2:-}" +arg3="${3:-}" + +# Two modes: normal provision ( [--debug-symbols]) and +# --uninstall (which doesn't need a debug method since it only undoes GRUB). +uninstall=0 +debug_method="" +symbols="" +case "$arg2" in + --uninstall) + uninstall=1 + [[ -z "$arg3" ]] || usage + ;; + kgdb|qemu) + debug_method="$arg2" + case "$arg3" in + ""|--debug-symbols) symbols="$arg3" ;; + *) usage ;; + esac + ;; + *) usage ;; +esac repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib/common.sh @@ -44,28 +66,62 @@ case "$target_os" in *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; esac -symbols_blurb="${symbols:+, with debug symbols}" -echo "provisioning $target at $LAB_SSH_TARGET:$LAB_SSH_PORT (debug method: $debug_method$symbols_blurb)" -lab_check_connection - -SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" -t "$LAB_SSH_TARGET" \ - "TARGET_OS='$target_os' DEBUG_METHOD='$debug_method' KGDB_TTY='$kgdb_tty' KGDB_BAUD='$kgdb_baud' INSTALL_DEBUG_SYMBOLS='$symbols' SUDO_PASS_B64='$LAB_SUDO_PASS_B64' DEBUG_MAXCPUS='${DEBUG_MAXCPUS:-}' bash -s" <<'REMOTE' -set -euo pipefail +# Preflight: ssh works. We don't insist sudo works yet because the user +# might be running 01 *to* configure sudo. The actual provisioning shell +# (a piped `sudo -S bash -s`) will exercise sudo and fail with apt's own +# diagnostics if the password is wrong. +lab_check_connection --no-sudo -sudo_run() { - sudo "$@" -} +if (( uninstall )); then + echo "uninstalling lab GRUB args from $target ($LAB_SSH_TARGET:$LAB_SSH_PORT)" +else + symbols_blurb="${symbols:+, with debug symbols}" + echo "provisioning $target ($LAB_SSH_TARGET:$LAB_SSH_PORT) for debug method: $debug_method$symbols_blurb" +fi -sudo_auth() { - if [[ -n "${SUDO_PASS_B64:-}" ]]; then - printf '%s\n' "$(printf '%s' "$SUDO_PASS_B64" | base64 -d)" | sudo -S -v - else - sudo -v - fi +# Run the remote script as root via SUDO_ASKPASS so stdin stays a clean +# pipe of "env-vars then script body" for bash -s. +# +# Why NOT `sudo -S` + piped password on stdin: sudo only reads stdin when +# policy requires it. With NOPASSWD configured, or cached creds, sudo +# skips the stdin read entirely and the password line we piped leaks into +# bash -s as the first script line — producing +# `bash: line 1: : command not found`. `sudo -k` doesn't help +# because NOPASSWD is a policy override, not a cache thing. +# +# SUDO_ASKPASS works regardless: sudo invokes the askpass helper to fetch +# the password (or doesn't, under NOPASSWD), and stdin is purely the +# bash -s script. We scp the helper + a chmod-600 password file to the +# target up front; a trap cleans both up on script exit. +askpass="$LAB_REMOTE_DIR/.kmod-askpass-$$" +pwfile="$LAB_REMOTE_DIR/.kmod-sudo-pw-$$" +cleanup_sudo_helpers() { + lab_ssh "rm -f '$askpass' '$pwfile' 2>/dev/null" || true } +trap cleanup_sudo_helpers EXIT + +lab_ssh "mkdir -p '$LAB_REMOTE_DIR'; umask 077; cat > '$pwfile'" <<<"$LAB_SUDO_PASS" +lab_ssh "umask 077; cat > '$askpass'; chmod 700 '$askpass'" <&2; exit 1; } -[[ "${ID:-}" == "ubuntu" ]] || { echo "target is not Ubuntu (os-release ID=$ID); only ubuntu targets are implemented" >&2; exit 1; } -sudo_auth +[[ "${TARGET_OS:-ubuntu}" == "ubuntu" ]] || { + echo "unsupported target OS: ${TARGET_OS:-}" >&2; exit 1; } +[[ "${ID:-}" == "ubuntu" ]] || { + echo "target is not Ubuntu (os-release ID=$ID); only ubuntu targets are implemented" >&2; exit 1; } + +grub_file=/etc/default/grub +grub_managed_keys='kgdboc=|maxcpus=|sysrq_always_enabled=|nokaslr' + +# Read current GRUB_CMDLINE_LINUX_DEFAULT, stripping the lab's managed keys +# so reruns replace rather than append. `nokaslr` is a bare token (no =), +# matched as a standalone word. +read_current_cmdline() { + local raw stripped + raw="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" + # Strip key=value forms and the bare "nokaslr" token; collapse spaces. + stripped="$(printf '%s' "$raw" | sed -E "s/(^| )($grub_managed_keys)([^ ]*)?/ /g; s/ */ /g; s/^ +//; s/ +$//")" + printf '%s' "$stripped" +} + +write_cmdline() { + local newline="$1" + cp "$grub_file" "$grub_file.kmod-debug-lab.$(date +%Y%m%d%H%M%S).bak" + if grep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' "$grub_file"; then + sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$newline\"|" "$grub_file" + else + printf 'GRUB_CMDLINE_LINUX_DEFAULT="%s"\n' "$newline" >> "$grub_file" + fi + update-grub +} + +if [[ "${UNINSTALL:-0}" == "1" ]]; then + echo "stripping lab boot args from $grub_file" + current="$(read_current_cmdline)" + write_cmdline "$current" + echo + echo "done. reboot to drop the lab boot args." + echo " boot args now: $current" + exit 0 +fi echo "[1/3] installing kernel headers for running kernel $kernel" apt_retry update @@ -90,11 +182,11 @@ if [[ "$INSTALL_DEBUG_SYMBOLS" == "--debug-symbols" ]]; then vmlinux="/usr/lib/debug/boot/vmlinux-$kernel" if [[ ! -r "$vmlinux" ]]; then apt_retry install -y ubuntu-dbgsym-keyring - cat </dev/null + cat > /etc/apt/sources.list.d/ddebs.list </dev/null | sort) +done < <(find "$usr_src_dir" -mindepth 1 -maxdepth 3 -name Makefile -path '*linux-source-*' 2>/dev/null | sort -Vr) +found_root="${matching_root:-$found_root}" if [[ -n "$found_root" ]]; then rm -rf "$source_tree" diff --git a/scripts/03-build-module.sh b/scripts/03-build-module.sh index 3c54c7f..2404f6b 100755 --- a/scripts/03-build-module.sh +++ b/scripts/03-build-module.sh @@ -23,7 +23,7 @@ kdir="$cache_dir/build" module_dir="$repo_root/module" [[ -e "$module_dir/Makefile" || -e "$module_dir/Kbuild" ]] || - die "no Makefile or Kbuild under $module_dir; drop your module project there (see $module_dir/README.md), or try the example: cp -r examples/chuck_norise/. module/" + die "no Makefile or Kbuild under $module_dir; drop your module project there (see $module_dir/README.md), or try the example: make use-example NAME=chuck_norise" kernel="$(<"$kernel_file")" intermediate_dir="$repo_root/build/intermediate/$target/$kernel" @@ -44,3 +44,4 @@ mapfile -t kos < <(find "$artifact_dir" -maxdepth 1 -name '*.ko' -type f | sort) for ko in "${kos[@]}"; do echo "built $ko" done +echo "next: scripts/04-deploy-debug-vscode.sh $target " diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index 109e593..dcf1c81 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -163,14 +163,24 @@ load_and_discover_symbols() { echo "loading $module_name on $target (debug_delay_ms=$debug_delay_ms)" lab_ssh_sudo "rmmod '$module_name' >/dev/null 2>&1 || true" # Background insmod on the target so we can poll /sys/module/.../sections - # in parallel with the module's debug_delay_ms sleep. `& sleep 1` keeps - # the SSH session open just long enough for sudo to read the password - # from stdin and exec into nohup; without it, ssh can close before sudo - # finishes authenticating and the load silently aborts. The orphaned - # nohup+sh+insmod chain survives session exit (non-interactive bash - # doesn't huponexit). The `|| insmod ...` falls back when the module - # doesn't declare a debug_delay_ms parameter. - lab_ssh_sudo "nohup sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$LAB_REMOTE_DIR/insmod.log' 2>&1 & sleep 1" + # in parallel with the module's debug_delay_ms sleep. + # + # Three subtleties: + # - `<&0` keeps sudo's stdin attached to the SSH-inherited pipe (where + # the password arrives). Non-job-control bash automatically redirects + # backgrounded processes' stdin to /dev/null UNLESS the command has + # its own stdin redirection — without `<&0`, sudo would see EOF + # instead of the password and silently fail to authenticate. + # - `& sleep 1` keeps the SSH session open long enough for sudo to + # read the password and exec into nohup. The sleep is short because + # sudo authentication is fast; if SSH closed earlier the in-flight + # password write could be lost. + # - The orphaned nohup+sh+insmod chain survives session exit because + # non-interactive bash doesn't huponexit. + # + # The `|| insmod ...` falls back when the module doesn't declare a + # debug_delay_ms parameter (the kv arg would otherwise reject with EINVAL). + lab_ssh_sudo "nohup sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$LAB_REMOTE_DIR/insmod.log' 2>&1 <&0 & sleep 1" echo "polling /sys/module/$module_name/sections/ (timeout ${insmod_wait_secs}s)" @@ -303,6 +313,7 @@ ln -sfn "$target-$debug_method-attached.gdb" "$gdb_dir/current-debug-attached.gd ln -sfn "../.kernel-cache/$target/vmlinux" "$gdb_dir/current-vmlinux" echo -echo "ready to attach GDB at $debug_endpoint" +echo "ready. GDB endpoint: $debug_endpoint" echo " gdb script: $gdb_file" echo " loader log: $loader_log" +echo "next: F5 in VS Code (Kernel: cppdbg) — or: gdb -tui $vmlinux -x $gdb_file" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 373fe3c..bb987d4 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -170,12 +170,21 @@ lab_ssh() { # (NOPASSWD sudo or sudo configured for passwordless), the empty string is # piped and sudo proceeds without prompting. # +# Why `-k`: without it, if sudo has cached credentials from an earlier +# invocation (e.g. `scripts/00-check-target.sh` just ran, or the user did +# `sudo` on the target within the last 5 min), `sudo -S` skips the stdin +# read entirely. The piped password then leaks into whatever inherits +# stdin from sudo — for `sudo bash -s`, the password line becomes bash's +# first script line and you get `bash: line 1: a: command not found`. +# `-k` ignores the cache for THIS invocation only; it does NOT invalidate +# the user's existing sudo timestamp. +# # Callers that need to feed their own stdin to the remote command (e.g. # rsync) should use `lab_rsync_from` / scp helpers instead — those take a # different path that doesn't compete for stdin. lab_ssh_sudo() { printf '%s\n' "$LAB_SUDO_PASS" | - SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "sudo -S -p '' $*" + SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "sudo -k -S -p '' $*" } # Copy a local file to the target. The remote path is absolute. @@ -198,17 +207,25 @@ lab_rsync_from() { # --- Preflight ------------------------------------------------------------ -# Fast, non-mutating connection check. Verifies SSH reachable, sudo works. +# Fast, non-mutating connection check. Verifies SSH reachable, and (by +# default) that sudo works. Pass `--no-sudo` to skip the sudo test — useful +# in `scripts/01-provision-target.sh`, which a fresh user might be running +# *to* configure sudo for the first time. +# # Called at the top of 01/02/04 so failures surface before any real work. -# `scripts/00-check-target.sh` calls a longer-form report on top of this. +# `scripts/00-check-target.sh` runs a longer-form report on top of this. +# shellcheck disable=SC2120 # arg is optional (--no-sudo); callers without it are intentional. lab_check_connection() { + local check_sudo=1 + [[ "${1:-}" == "--no-sudo" ]] && check_sudo=0 + if ! lab_ssh 'true' 2>/dev/null; then die "cannot reach $LAB_SSH_TARGET over ssh (port $LAB_SSH_PORT). - check TARGET_SSH_HOST/PORT/USER for '$LAB_TARGET' in lab.local.env - if the VM is up, try by hand: ssh -p $LAB_SSH_PORT $LAB_SSH_TARGET - if you're using passwords, confirm TARGET_SSH_PASS[$LAB_TARGET] is set" fi - if ! lab_ssh_sudo 'true' 2>/dev/null; then + if (( check_sudo )) && ! lab_ssh_sudo 'true' 2>/dev/null; then die "ssh reaches $LAB_SSH_TARGET but sudo doesn't work there. - check TARGET_SUDO_PASS[$LAB_TARGET] in lab.local.env (defaults to TARGET_SSH_PASS) - on the target, confirm: sudo -n -v (or run sudo by hand once to cache creds) From 7513cf5462d2edb99bbefc506bf48d1986a4578d Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 20:12:22 +0300 Subject: [PATCH 12/13] fix: removed .llseek = no_llseek as it was removed in new kernel versions --- examples/chuck_norise/src/chuck_device.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/chuck_norise/src/chuck_device.c b/examples/chuck_norise/src/chuck_device.c index 4705ac9..cb6758d 100644 --- a/examples/chuck_norise/src/chuck_device.c +++ b/examples/chuck_norise/src/chuck_device.c @@ -33,7 +33,13 @@ static const struct file_operations chuck_fops = { .owner = THIS_MODULE, .open = chuck_open, .read = chuck_read, - .llseek = no_llseek, + /* + * .llseek deliberately unset. The kernel installs default_llseek for + * char devices, which matches chuck_read's behavior (it advances and + * respects *position). Earlier versions of this file pinned + * .llseek = no_llseek, but that symbol was removed in Linux 6.12 + * (mainline commit 868941b14441 "fs: remove no_llseek"). + */ }; static struct class *chuck_class_create(void) From 3cd0008a4bbbef092ab8cb4ad75158196f8bc8d2 Mon Sep 17 00:00:00 2001 From: Dor Date: Sat, 23 May 2026 20:41:46 +0300 Subject: [PATCH 13/13] cleaner readmes and remove large comments --- .github/workflows/lint.yml | 3 +- .vscode/c_cpp_properties.json | 5 - CONTRIBUTING.md | 111 +++---- Makefile | 59 ++-- README.md | 366 ++++++++--------------- examples/chuck_norise/Makefile | 21 +- examples/chuck_norise/README.md | 59 ++-- examples/chuck_norise/src/chuck_device.c | 11 +- examples/chuck_norise/src/hello.c | 4 +- module/README.md | 78 +---- scripts/00-check-target.sh | 21 +- scripts/01-provision-target.sh | 102 +++---- scripts/02-setup-host-build.sh | 89 ++---- scripts/03-build-module.sh | 7 +- scripts/04-deploy-debug-vscode.sh | 151 ++++------ scripts/lib/common.sh | 130 +++----- 16 files changed, 428 insertions(+), 789 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8c84bff..75dcc27 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -54,8 +54,7 @@ jobs: strategy: fail-fast: false matrix: - # Surface drift across LTS kernels. ubuntu-22.04 ships 5.15; 24.04 ships 6.8. - os: [ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 248ab5d..9065b7f 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -41,10 +41,5 @@ "${workspaceFolder}/.kernel-cache/current/build/include/linux/compiler_types.h" ] } - ], - "_notes": [ - "KBUILD_MODNAME is a placeholder; the actual value at compile time is your module's real name. IntelliSense only needs the macro to resolve, not match exactly.", - "ubuntu/include is Ubuntu-kernel-specific (where the distro's extra headers live); on non-Ubuntu kernels VS Code silently ignores it.", - "All paths under .kernel-cache/current/ resolve only after scripts/02-setup-host-build.sh has run. Before that, expect include squiggles." ] } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e518573..d7a590b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,83 +1,68 @@ # Contributing -Bug reports, fixes, and improvements are welcome. This file describes what -that looks like for this repo. +Bug reports, fixes, and improvements welcome. -## What this repo is (and isn't) +## What belongs here -This is a **template** for building and source-debugging out-of-tree Linux -kernel modules. The lab tooling lives in `Makefile`, `scripts/`, -`.vscode/`, `host/`, and the docs. The example module under -`examples/chuck_norise/` exists to demonstrate the workflow end-to-end. +This is a **template** for building and source-debugging out-of-tree +Linux kernel modules. Lab tooling lives in `Makefile`, `scripts/`, +`.vscode/`, `host/`. The example under `examples/chuck_norise/` +demonstrates the workflow end-to-end. -PRs that **belong** here: +**Yes:** -- Bugs in the lab tooling — broken scripts, wrong assumptions about target - state, scripts that fail on re-run, unclear error messages. -- Support for a new target OS family (currently only Ubuntu). See the - `TARGET_OS` checks in scripts 01 and 02 for where to add a backend. -- New debug methods that fit the ` -> GDB` shape (anything - speaking the GDB remote serial protocol). -- Documentation that fixes misleading or missing information. -- Quality-of-life improvements: better error messages, more idempotent - re-runs, faster sync, smaller artifact dirs. +- Bugs in the lab tooling: broken scripts, wrong assumptions, scripts + that fail on re-run, unclear errors. +- New target-OS backends (currently only Ubuntu — see the `TARGET_OS` + checks in scripts 01 and 02). +- New debug methods that fit the ` → GDB` shape. +- Docs fixes. +- Quality-of-life: better errors, idempotent re-runs, faster sync. -PRs that **do not** belong here: +**No:** -- Changes to `module/` (that's the user's slot; we keep it as a placeholder - with just a README). -- New example modules under `examples/`. We keep one example focused and - small so it stays useful as a walkthrough. -- Project-management features (work logs, task tracking, etc.). The lab - stays focused on build + debug. +- Changes to `module/` (user's slot; stays as just a placeholder README). +- New examples under `examples/` (one focused example beats many). +- Project-management features (work logs, task tracking). ## Reporting bugs Open an issue with: -- Your **dev host** (distro, version, kernel) and **target VM** (distro, - version, kernel, hypervisor). -- The debug method you tried (`kgdb` or `qemu`) and the endpoint you - pointed it at. -- The command you ran and the full output. For deploy failures, attach +- Dev host (distro, version, kernel) and target VM (distro, version, + kernel, hypervisor). +- Debug method tried (`kgdb` or `qemu`) and endpoint configured. +- Command run and full output. For deploy failures, attach `.gdb/--loader.log`. -- What you expected, what happened instead. +- What you expected vs what happened. ## Submitting changes -1. Fork and branch from `main`. -2. Make focused commits — one logical change per commit. The git history - should read top-to-bottom as a small set of intentional steps. -3. Keep scripts shellcheck-clean (`shellcheck -x scripts/lib/common.sh - scripts/0*.sh`) and bash-syntax-clean (`bash -n`). -4. Update docs in the same PR. If you touched a script, check `README.md` - and `module/README.md` for anything the change makes inaccurate. -5. Open a PR with a description that explains *why* the change is needed, - not just what it does. The diff already says what. +1. Fork, branch from `main`. +2. Focused commits — one logical change per commit. +3. Keep scripts shellcheck-clean and `bash -n`-clean: + `shellcheck -x scripts/lib/common.sh scripts/0*.sh` and + `for f in scripts/lib/*.sh scripts/0*.sh; do bash -n "$f"; done`. +4. Update docs in the same PR — if you touched a script, check + `README.md` and `module/README.md` for anything the change made + inaccurate. +5. PR description explains *why*, not just what. ## Style -- **Shell:** tabs for indentation. `set -euo pipefail` at the top of every - script. Errors via the `die` helper. Use `lab_*` helpers from - `scripts/lib/common.sh` for SSH/SCP/rsync — do not call those binaries - directly from numbered scripts. -- **Makefile:** tabs for recipes. Explicit `.PHONY` declarations. -- **Markdown:** wrap prose at ~80 columns. Use fenced code blocks with - language tags. -- **JSON (VS Code config):** 2-space indent, trailing newline. - -The repo includes an `.editorconfig` that captures these — most editors -will apply it automatically. - -## Testing changes locally - -There is no automated test suite for the build/debug flow (it requires real -hardware-or-VM targets). Before sending a PR: - -- Run `bash -n scripts/0*.sh scripts/lib/*.sh` to catch syntax errors. -- If you have `shellcheck`, run it too: `shellcheck -x scripts/lib/common.sh - scripts/0*.sh`. -- Smoke-test the full chain against at least one target VM you have access - to: `01 --debug-symbols` → reboot → `02` → `03` → `04` → F5 in VS Code - → set breakpoints in both your module and a kernel function and confirm - both bind. +- **Shell:** tabs for indent. `set -euo pipefail` at the top. Errors via + the `die` helper. Use `lab_*` helpers from `scripts/lib/common.sh` — + don't call ssh/scp/sshpass binaries directly from numbered scripts. +- **Makefile:** tabs for recipes; explicit `.PHONY`. +- **Markdown:** wrap at ~80; fenced code blocks with language tags. +- **JSON / YAML:** 2-space indent, trailing newline. + +`.editorconfig` captures these — most editors apply it automatically. + +## Testing locally + +No automated test suite for the build/debug flow (it needs real VMs). +Before sending a PR, smoke-test the full chain against at least one +target VM: `01 --debug-symbols` → reboot → `02` → `03` → `04` → F5 in +VS Code → set breakpoints in both your module and a kernel function, +confirm both bind. diff --git a/Makefile b/Makefile index 7560971..7c48b92 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,15 @@ -# Top-level wrapper that drives the user's kernel module under module/. +# Top-level wrapper for the user's module under module/. # -# The user's module is built out-of-tree against a target-specific kernel -# header tree cached under .kernel-cache//. Build artifacts go under -# build/intermediate/// (Kbuild's M=) and the final .ko is -# moved to build/artifacts///. +# Builds out-of-tree against a target-specific kernel header tree under +# .kernel-cache//. Sources are staged into +# build/intermediate/// before Kbuild runs from there, +# and `-ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR)` is injected +# via KCFLAGS so DWARF paths and `__FILE__` resolve back to module/ — +# IDE breakpoints keyed by absolute path bind to the real sources. # -# Staging exists for two reasons: -# 1. Per-target out-of-tree artifacts. Building the same module/ tree against -# multiple target kernels needs separate intermediate dirs. -# 2. DWARF path rewriting. -ffile-prefix-map=$(INTERMEDIATE_DIR)=$(MODULE_DIR) -# rewrites paths in DWARF debug info (and __FILE__) from the staged -# intermediate dir back to module/, so IDEs that bind breakpoints by -# absolute path (VS Code, CLion) match the symtab. -# -# The user's module/Makefile sees only a normal Kbuild invocation rooted at -# the staging directory. KCFLAGS is the kernel build's official escape hatch -# for adding C flags from the outside, so the user's Makefile stays generic. +# Per-target staging also keeps each (target, kernel) pair's object +# files isolated. The user's module/Makefile stays a normal Kbuild file +# with no knowledge of the staging. HOST_KERNEL := $(shell uname -r) CURRENT_CACHE := $(CURDIR)/.kernel-cache/current @@ -40,16 +34,16 @@ help: @echo "Top-level lab Makefile." @echo "" @echo "Normally invoked by scripts/03-build-module.sh , which sets" - @echo "KDIR, BUILD_ID, INTERMEDIATE_DIR, and ARTIFACT_DIR to per-target paths." + @echo "KDIR, BUILD_ID, INTERMEDIATE_DIR, and ARTIFACT_DIR per-target." @echo "" @echo "Build targets:" - @echo " modules build module/ against KDIR (default)" - @echo " clean remove intermediate and artifact dirs for BUILD_ID" - @echo " prepare-build stage module/ into INTERMEDIATE_DIR (internal)" + @echo " modules build module/ against KDIR (default)" + @echo " clean remove BUILD_ID's intermediate + artifact dirs" + @echo " prepare-build stage module/ into INTERMEDIATE_DIR (internal)" @echo "" @echo "module/ ergonomics:" - @echo " use-example NAME=chuck_norise copy an example into module/" - @echo " clean-module reset module/ to just README.md" + @echo " use-example NAME=chuck_norise copy an example into module/" + @echo " clean-module reset module/ to just README.md" @echo "" @echo "Vars (auto-resolved when .kernel-cache/current points at a target):" @echo " MODULE_DIR $(MODULE_DIR)" @@ -59,8 +53,9 @@ help: @echo " ARTIFACT_DIR $(ARTIFACT_DIR)" @echo " EXTRA_CCFLAGS $(EXTRA_CCFLAGS) (appended to KCFLAGS)" -# Copy examples/$(NAME)/ into module/. Refuses if module/ already contains a -# Makefile/Kbuild (use clean-module first). NAME=chuck_norise unless overridden. +# Copy examples/$(NAME)/ into module/. Refuses if module/ already has a +# Makefile/Kbuild (run clean-module first). Preserves the placeholder +# module/README.md so the example's own README doesn't overwrite it. NAME ?= chuck_norise use-example: @if [ ! -d "$(CURDIR)/examples/$(NAME)" ]; then \ @@ -80,14 +75,8 @@ use-example: @echo "done. next: scripts/03-build-module.sh " @echo "(the example's docs stay at examples/$(NAME)/README.md)" -# Remove everything from module/ except its placeholder README, so you can -# drop a fresh module project in. Refuses to touch tracked files outside -# module/. clean-module: - @if [ ! -d "$(MODULE_DIR)" ]; then \ - echo "Makefile: $(MODULE_DIR) is missing — nothing to clean." >&2; \ - exit 1; \ - fi + @[ -d "$(MODULE_DIR)" ] || { echo "Makefile: $(MODULE_DIR) is missing." >&2; exit 1; } @find "$(MODULE_DIR)" -mindepth 1 -maxdepth 1 -not -name README.md -print0 \ | xargs -0 -r rm -rf @echo "module/ reset (kept only README.md)" @@ -108,11 +97,9 @@ modules: prepare-build echo "built $(ARTIFACT_DIR)/$$name"; \ done -# Mirror $(MODULE_DIR) into $(INTERMEDIATE_DIR) as a tree of absolute -# symlinks so Kbuild's `M=` sees the user's sources in a writeable scratch -# dir without polluting module/. `cp -as` is GNU coreutils; it recursively -# creates dirs and symlinks each file. Re-running is idempotent because we -# wipe the symlink scaffolding first. +# `cp -as` recursively creates dirs and absolute symlinks for each file. +# Re-runs are idempotent: we wipe the symlink scaffolding first so a +# removed source file doesn't linger as a stale symlink. prepare-build: @if [ ! -e "$(MODULE_DIR)/Makefile" ] && [ ! -e "$(MODULE_DIR)/Kbuild" ]; then \ echo "Makefile: no Makefile or Kbuild under $(MODULE_DIR);" >&2; \ diff --git a/README.md b/README.md index 538ee56..cdfdd3b 100644 --- a/README.md +++ b/README.md @@ -2,52 +2,49 @@ A template for building and **source-debugging** out-of-tree Linux kernel modules against one or more target VMs. Drop your module under `module/`, -point the lab at a target, press F5 — step through your code (and into -kernel code) in VS Code or `gdb -tui`. +point the lab at a target, press F5 — step through your code and into +kernel code from VS Code or `gdb -tui`. -What you get out of the box: +What you get: - **One slot, any module.** `module/` is your project; the lab is generic. - **Per-target builds.** Cross-build the same source against multiple target kernels without polluting your tree. -- **Breakpoints just work.** DWARF `-ffile-prefix-map` is injected for - you so IDE breakpoints by absolute path bind to your real source files. -- **Step into the kernel.** With one flag (`--debug-symbols`), the lab - pulls down `vmlinux` + kernel source and wires GDB `substitute-path` - so you can step from your module's read handler into `vfs_read`. +- **Breakpoints just work.** `-ffile-prefix-map` is injected via `KCFLAGS` + so IDE breakpoints by absolute path bind to your real source files. +- **Step into the kernel.** With `--debug-symbols`, the lab pulls down + `vmlinux` + kernel source and wires GDB `substitute-path` so you can + step from your module's read handler into `vfs_read`. - **Two debug methods.** In-kernel KGDB over serial (`kgdb`) and any hypervisor GDB stub (`qemu`, also VMware's `debugStub`). -- **One-command flow.** Four numbered scripts + a VS Code F5. -The repo ships with a small worked example (`examples/chuck_norise/`) so -you can run the whole flow end-to-end before writing a line of module -code. +The repo ships with a worked example (`examples/chuck_norise/`) so you +can run the full flow end-to-end before writing a line of module code. --- ## Quickstart -You need a **Debian-based dev host** (Debian, Ubuntu, WSL Ubuntu, …) and -at least one **Ubuntu target VM** reachable over SSH. From fresh clone -to a stopped breakpoint: +You need a **Debian-based dev host** (Debian, Ubuntu, WSL Ubuntu) and at +least one **Ubuntu target VM** reachable over SSH. ```bash # 0. clone + configure git clone kmod-debug-lab && cd kmod-debug-lab cp lab.example.env lab.local.env -$EDITOR lab.local.env # set TARGET_SSH_*, endpoints, etc. +$EDITOR lab.local.env # TARGET_SSH_*, endpoints, etc. -# 0.5 (optional) sanity-check the target before changing anything on it +# optional preflight (non-mutating) ./scripts/00-check-target.sh server qemu -# 1. one-time target setup (installs headers, vmlinux dbg, kernel source) +# 1. target setup: headers, vmlinux dbg, kernel source (one-time) ./scripts/01-provision-target.sh server qemu --debug-symbols # ...reboot the target VM so new GRUB args take effect... -# 2. one-time host setup (syncs everything to .kernel-cache/server/) +# 2. host setup: sync everything to .kernel-cache/server/ (one-time) ./scripts/02-setup-host-build.sh server -# 3. try the example module (replace with your own when ready) +# 3. try the example (or skip and drop your own module under module/) make use-example NAME=chuck_norise ./scripts/03-build-module.sh server @@ -61,49 +58,45 @@ code . Set a breakpoint in `module/src/hello.c` and one in `vfs_read` — both bind. You're stepping through kernel code from your module. +After the one-time `01 + 02`, the inner loop is **03 → 04 → F5** (or +just F5 — the launch's pre-task runs 04, and 03 if needed). + --- -## How the pieces fit +## Repo layout ``` module/ your module project (Makefile + sources) -examples/chuck_norise/ worked example; copy into module/ with `make use-example` +examples/chuck_norise/ worked example; `make use-example` copies it into module/ -scripts/00-check-target.sh non-mutating preflight: ssh / sudo / headers / vmlinux / endpoint -scripts/01-provision-target.sh target-side: install headers, vmlinux dbg, source, GRUB args - (also `--uninstall` to undo the GRUB args) -scripts/02-setup-host-build.sh dev-host: sync headers + vmlinux + kernel source into .kernel-cache/ -scripts/03-build-module.sh dev-host: build module/ via Kbuild against the synced headers +scripts/00-check-target.sh preflight: ssh / sudo / headers / vmlinux / endpoint +scripts/01-provision-target.sh target-side: headers, vmlinux dbg, kernel source, GRUB args + (`--uninstall` to undo the GRUB args) +scripts/02-setup-host-build.sh dev-host: sync headers + vmlinux + source into .kernel-cache/ +scripts/03-build-module.sh dev-host: build module/ against the synced headers scripts/04-deploy-debug-vscode.sh dev-host: upload, insmod, write .gdb/-.gdb -scripts/lib/common.sh shared bash helpers (lab_ssh, lab_ssh_sudo, lab_rsync_from, …) +scripts/lib/common.sh shared helpers (lab_ssh, lab_ssh_sudo, lab_rsync_from, ...) -.kernel-cache// per-target build/source/vmlinux cache (gitignored) +.kernel-cache// synced build/source/vmlinux (gitignored) build/ per-target intermediate + final .ko (gitignored) -.gdb/ generated GDB init files (gitignored, regenerated on F5) +.gdb/ generated GDB init files (gitignored, regenerated each F5) -host/ optional helpers for VMware-on-Windows users +host/ optional VMware-on-Windows helpers lab.example.env -> lab.local.env per-machine config (gitignored copy) ``` -Top-level `Makefile` exposes a few non-build targets too: -`make use-example NAME=` (copy `examples//` into `module/`), -`make clean-module` (reset `module/` to just its placeholder README), -`make help` (full var/target listing). - -The numbered scripts are designed to be run in order. After the one-time -01 + 02 against a target, the inner loop is **03 → 04 → F5** (or just F5, -which runs 04 as a prelaunch task and rebuilds via 03 if you ran it -manually first). +Useful `make` targets: `make use-example NAME=`, `make clean-module`, +`make help`. --- ## The `module/` contract -`module/` is the lab's only slot for your kernel module project. The lab -does not know your module's name, source layout, or behavior; it expects -exactly: +`module/` is the lab's only slot for your module. The lab does not know +its name, source layout, or behavior. Expectations: -- **A standard Kbuild file** at `module/Makefile` (or `module/Kbuild`): +- **`module/Makefile`** (or `Kbuild`) following standard out-of-tree + conventions: ```makefile obj-m += my_module.o @@ -111,54 +104,41 @@ exactly: ccflags-y := -I$(src)/include -g -DDEBUG ``` - See `examples/chuck_norise/Makefile` for the canonical shape, including - the optional `ifndef KERNELRELEASE` wrapper that lets you also run plain - `make` directly in `module/`. + `examples/chuck_norise/Makefile` shows the canonical shape, including + the optional `ifndef KERNELRELEASE` wrapper that lets `make` work + directly in `module/`. - **Exactly one `obj-m` entry per build.** The lab discovers the module - name from the produced `.ko` — you don't declare it anywhere - else. + name from the produced `.ko`. -- **Optional:** a `debug_delay_ms` module parameter: +- **Optional `debug_delay_ms` module parameter** so the host has time to + attach GDB before init runs: ```c static unsigned int debug_delay_ms = 5000; module_param(debug_delay_ms, uint, 0644); ``` - When present, scripts/04 passes the value from `DEBUG_LOAD_DELAY_MS` - (in `lab.local.env`, default 5000 ms) so your `module_init` sleeps - long enough for the host to read `/sys/module//sections/*` and - for you to break in GDB before init does anything. Modules without - the parameter would reject `insmod foo.ko debug_delay_ms=...` with - `-EINVAL`; the loader catches that and retries with a plain `insmod`, - so undeclared modules still load. + Script 04 passes `DEBUG_LOAD_DELAY_MS` (from `lab.local.env`) on + `insmod`. Modules that don't declare the parameter fall back to a + plain `insmod` automatically. -Source layout, headers, license — those are yours. The lab passes -through whatever your Kbuild file declares. +Source layout is yours. The lab passes through whatever your Kbuild file +declares. --- ## Configuring the lab -Copy the example env and edit it for your machines: - -```bash -cp lab.example.env lab.local.env -``` - Targets are **data, not variable prefixes**. Add a profile name to `TARGETS`, then add entries to the `TARGET_*` maps under that name: ```bash TARGETS=(desktop server) -declare -A TARGET_SSH_HOST=( - [desktop]=ubuntu-desktop.local - [server]=ubuntu-server.local -) +declare -A TARGET_SSH_HOST=([desktop]=ubuntu-desktop.local [server]=ubuntu-server.local) declare -A TARGET_SSH_USER=([desktop]=user [server]=user) -declare -A TARGET_SSH_PASS=([desktop]= [server]=) # empty when using SSH keys +declare -A TARGET_SSH_PASS=([desktop]= [server]=) # empty when using SSH keys declare -A TARGET_DEBUG_ENDPOINT_QEMU=( [desktop]=127.0.0.1:1234 @@ -166,233 +146,141 @@ declare -A TARGET_DEBUG_ENDPOINT_QEMU=( ) ``` -Endpoints are split per debug method -(`TARGET_DEBUG_ENDPOINT_KGDB` and `TARGET_DEBUG_ENDPOINT_QEMU`). Either -map may be left empty per target if you don't use that method. Script 04 -errors clearly if you pick a method without an endpoint. - -Target names accept `[A-Za-z][A-Za-z0-9_-]*`. The VS Code task picker -accepts any name you type — no editing of `.vscode/` needed when adding -a target. +Endpoints are split per debug method (`TARGET_DEBUG_ENDPOINT_KGDB`, +`TARGET_DEBUG_ENDPOINT_QEMU`); either map can be empty per target. The +VS Code task picker accepts any target name you type — no edits to +`.vscode/` when adding targets. --- ## Debug methods -Provisioning (script 01) and deploy (script 04) both take a -`` argument: - -- **`kgdb`** — in-kernel KGDB over the guest's serial port. Script 01 - adds `kgdboc=ttyS0,115200` and `sysrq_always_enabled=1` to the guest's - GRUB command line. Break with `echo g | sudo tee /proc/sysrq-trigger` - on the target. Useful when you cannot change the hypervisor's command - line (VMware Workstation is the common case). - - **`qemu`** — the hypervisor's built-in GDB stub - (QEMU's `-gdb tcp::PORT` / `-s`, VMware's `debugStub.listen.guest64`). + (QEMU's `-gdb tcp::PORT` or `-s`, VMware's `debugStub.listen.guest64`). No KGDB in the guest; the hypervisor halts the vCPU directly. +- **`kgdb`** — in-kernel KGDB over the guest's serial port. Script 01 + adds `kgdboc=ttyS0,115200` and `sysrq_always_enabled=1` to GRUB. Break + with `echo g | sudo tee /proc/sysrq-trigger` on the target. Useful + when you can't change the hypervisor's command line (VMware + Workstation is the common case). Neither method assumes any in-module `kgdb_breakpoint()` call. In both, -you break manually after script 04 finishes. +you break manually after 04 finishes. -### QEMU gdbstub +**QEMU stub:** `qemu-system-x86_64 ... -gdb tcp::1234` (`-S` to pause +at boot). -```text -qemu-system-x86_64 ... -gdb tcp::1234 -``` +**VMware debug stub:** in the VM's `.vmx`: -Use `-S` to pause the vCPU at boot. Without `-S`, the guest runs until -GDB connects. - -### VMware debug stub - -Power off the VM, edit the VM's `.vmx`: - -```text +``` debugStub.listen.guest64 = "TRUE" debugStub.port.guest64 = "8864" debugStub.listen.guest64.remote = "TRUE" debugStub.hideBreakpoints = "FALSE" ``` -Same shape as the QEMU stub — set -`TARGET_DEBUG_ENDPOINT_QEMU[server]=127.0.0.1:8864`. - -### KGDB over serial → TCP - -Set up a serial bridge from the guest's `/dev/ttyS0` to a TCP listener -GDB can connect to. How depends on the hypervisor: - -- **QEMU:** `-serial tcp:127.0.0.1:5520,server,nowait` -- **VMware Workstation on Windows:** named pipe in the VM's serial port - + `host/bridge-kgdb.ps1`: - - ```powershell - .\host\bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520 - ``` - -Provision the target with `kgdb`: - -```bash -./scripts/01-provision-target.sh server kgdb -``` - -`TARGET_KGDB_TTY` and `TARGET_KGDB_BAUD` in `lab.local.env` must match -the GRUB args added by provisioning (defaults `ttyS0` / `115200`). +Then set `TARGET_DEBUG_ENDPOINT_QEMU[server]=127.0.0.1:8864`. -Useful sanity check before F5: +**KGDB over serial → TCP:** bridge the guest's `/dev/ttyS0` to a TCP +listener. For QEMU: `-serial tcp:127.0.0.1:5520,server,nowait`. For +VMware Workstation on Windows: named-pipe serial port + +`host/bridge-kgdb.ps1 -PipeName kgdb-server -Port 5520`. `TARGET_KGDB_TTY` +and `TARGET_KGDB_BAUD` in `lab.local.env` must match the GRUB args +script 01 adds. -```bash -nc -vz 127.0.0.1 5520 -``` +Sanity check before F5: `nc -vz 127.0.0.1 `. --- ## Stepping into kernel code -Pass `--debug-symbols` to script 01 to install both the matching -`vmlinux` debug image and the `linux-source-X` package on the target. -Script 02 syncs both, extracts the source on the host, and creates a -stable `.kernel-cache//source` symlink. - -Script 04 then asks `addr2line` where `start_kernel` lives in `vmlinux` -(always `/init/main.c`) and emits the corresponding GDB -remap into the generated `.gdb` file: - -```text -set substitute-path -directory -``` +Pass `--debug-symbols` to script 01 (installs the matching `vmlinux` +debug image and the `linux-source-X` package). Script 02 syncs and +extracts both. Script 04 then asks `addr2line` where `start_kernel` +lives in `vmlinux` (always `/init/main.c`) and emits the +corresponding `set substitute-path` into the generated `.gdb` file. That's it — set a breakpoint in `vfs_read` and step through it. -This step gracefully degrades. If `vmlinux` is missing, script 04 errors -clearly. If the kernel source wasn't synced, script 04 prints a one-line -note and continues — module debugging still works; you just see -disassembly when stepping into kernel functions. Check -`.gdb/--loader.log` for details when something is off. +Gracefully degrades: missing `vmlinux` is fatal (script 04 errors with +remediation); missing kernel source prints a one-line note and module +debugging still works (kernel step-into shows disassembly instead of +source). --- ## Undoing target-side changes -Script 01 modifies the target's GRUB command line. To remove every boot -arg the lab added (`nokaslr`, `kgdboc=...`, `sysrq_always_enabled=1`, -`maxcpus=...`) and restore a clean kernel command line: - -```bash -./scripts/01-provision-target.sh server --uninstall -``` - -This leaves installed packages alone — only the GRUB args are reverted. -A timestamped backup of `/etc/default/grub` is left on the target -(`/etc/default/grub.kmod-debug-lab..bak`) for paranoid recovery. - ---- - -## Build outputs - -``` -build/intermediate/// staged Kbuild tree (recursive symlinks back to module/) -build/artifacts///.ko final module -.kernel-cache//build -> synced linux-headers -.kernel-cache//source -> synced linux-source (when --debug-symbols) -.kernel-cache//vmlinux debug-symbol vmlinux from the target -.kernel-cache/current -> drives bare `make` and IntelliSense -.gdb/-.gdb full GDB init: arch, vmlinux, target remote, sourced symbols -.gdb/--attached.gdb trimmed variant for IDEs that already attach themselves -.gdb/--loader.log output of remote insmod + section discovery -.gdb/current-debug{.gdb,-attached.gdb} symlinks the VS Code launch configs read (re-pointed each F5) -``` - -`build/`, `.kernel-cache/`, `.gdb/`, `lab.local.env`, and -`compile_commands.json` are gitignored. +`./scripts/01-provision-target.sh --uninstall` removes every +boot arg the lab added (`nokaslr`, `kgdboc=`, `sysrq_always_enabled=1`, +`maxcpus=`) and re-runs `update-grub`. Installed packages are left in +place. A timestamped GRUB backup is left on the target under +`/etc/default/grub.kmod-debug-lab..bak`. --- ## Troubleshooting -**`sshpass: command not found`.** You configured `TARGET_SSH_PASS`. On -the dev host: `sudo apt-get install -y sshpass`. Leave -`TARGET_SUDO_PASS` empty when the sudo password matches the SSH -password. - -**F5 fails with `target remote ... Connection timed out`.** The endpoint -in `.gdb/current-debug.gdb` isn't reachable from the dev host. Verify -with `nc -vz `. On WSL, the Windows host address is often -in `/etc/resolv.conf`: - -```bash -grep nameserver /etc/resolv.conf -``` +**`sshpass: command not found`.** `TARGET_SSH_PASS` is set. On the dev +host: `sudo apt-get install -y sshpass`. -After editing `lab.local.env`, re-run F5 so script 04 regenerates the +**F5 fails with `target remote ... Connection timed out`.** Endpoint +not reachable from the dev host. `nc -vz ` to confirm. On +WSL, the Windows-host address is in `/etc/resolv.conf`'s `nameserver` +line. Re-run F5 after fixing `lab.local.env` so 04 regenerates the `.gdb/` files. -**GDB connects but the module never loads.** Check -`.gdb/--loader.log` for the insmod error. You can also -SSH in and try the insmod by hand: +**GDB connects but the module never loads.** Inspect +`.gdb/--loader.log`. Reproduce by hand: +`ssh sudo insmod /tmp/kmod-debug-lab-/.ko` then +`ssh sudo dmesg | tail -40`. -```bash -ssh sudo insmod /tmp/kmod-debug-lab/.ko -ssh sudo dmesg | tail -40 -``` - -**Breakpoints bind to files under `build/intermediate/...`.** Re-run F5 -so `.gdb/current-debug.gdb` is regenerated with the correct -`substitute-path`. Don't edit `.gdb/` by hand — it's regenerated every -F5. +**Breakpoints bind under `build/intermediate/...`.** Re-run F5; 04 +regenerates `.gdb/current-debug.gdb` with the correct +`substitute-path`. Don't edit `.gdb/` by hand. **VS Code shows include squiggles for `linux/module.h`.** Script 02 -hasn't synced the headers yet, or you switched targets and IntelliSense -is caching old paths. Re-run `scripts/02-setup-host-build.sh ` -and `C/C++: Reset IntelliSense Database` from the command palette. +hasn't synced the headers yet, or you switched targets and +IntelliSense is caching old paths. Re-run script 02 and run +`C/C++: Reset IntelliSense Database`. **`/sys/module//sections/.text` didn't appear within 30s.** Your -module either failed to load, or it's slow to init on the target. The -loader log will show `insmod` output. Bump the timeout with -`INSMOD_WAIT_SECS=60 ./scripts/04-... ...` if needed. - -**GDB shows `Cannot access memory at address …` when stepping into the -kernel.** Either `vmlinux` is missing (re-run `scripts/01-... --debug-symbols` -then `scripts/02-...`) or the cached `vmlinux` belongs to a different -kernel than the one running on the target. The latter is caught by -script 04's stale-kernel guard, but if you bypassed it, re-sync. - -**Spaces in your repo path.** Clone to a path WITHOUT spaces. GDB's -`set substitute-path FROM TO` splits on whitespace, so a `repo_root` like -`/home/me/path with spaces/...` produces a substitute-path that gets -parsed as four arguments and silently fails to remap module sources. -Kbuild is also notoriously fragile with spaces in paths. +module failed to load, or it's slow to init. Loader log shows insmod +output. Bump with `INSMOD_WAIT_SECS=60 ./scripts/04-... ...` if needed. + +**`Cannot access memory at address …` when stepping into the kernel.** +Either vmlinux is missing (re-run `01 --debug-symbols && 02`) or the +cached vmlinux is for a different kernel than the target now runs +(script 04 normally catches this — if you bypassed the guard, re-sync). + +**Spaces in your repo path.** Clone to a path without spaces. GDB's +`set substitute-path FROM TO` splits on whitespace; Kbuild is also +fragile with spaces. --- ## Tool versions -Tested with: - -- **Dev host:** Ubuntu 24.04, bash 5+, GNU coreutils, GDB ≥ 10, gcc 13. -- **Target VM:** Ubuntu 22.04 / 24.04, kernel 5.15 / 6.8. -- The `cp -as` recursive-symlink staging assumes GNU coreutils on the dev - host; macOS would need `brew install coreutils` and a `CP=gcp` override. -- `set substitute-path` for kernel-source remapping needs GDB ≥ 7.5 - (every supported distro ships much newer). -- `-ffile-prefix-map` (DWARF rewriting) requires GCC ≥ 8 or Clang ≥ 7. +Dev host: bash 5+, GNU coreutils, GDB ≥ 10, gcc ≥ 8 (for +`-ffile-prefix-map`). Target VM: Ubuntu 22.04 / 24.04 (kernel 5.15 / +6.8 tested; CI also exercises ubuntu-24.04 runners with 6.17). macOS +dev hosts need `brew install coreutils` and `make CP=gcp` for the +recursive-symlink staging. --- ## Adding a new target OS -Currently only `TARGET_OS[*]=ubuntu` is implemented in scripts 01 and 02. -Adding (say) Fedora means teaching script 01 how to install headers / -dbgsym / source via `dnf`, and teaching script 02 how to discover the -header tree paths under `/usr/src/`. Both scripts check `target_os` near -the top and fail fast on unknown values — that's where to add a backend. -PRs welcome. +Only `TARGET_OS[*]=ubuntu` is implemented in scripts 01 and 02. Adding +(say) Fedora means teaching script 01 to install headers / dbgsym / +source via `dnf`, and teaching script 02 to discover the header tree +paths under `/usr/src/`. Both scripts check `target_os` near the top +and fail fast on unknown values — that's where to add a backend. PRs +welcome. --- ## License -MIT — see [LICENSE](LICENSE). Your module under `module/` is yours; the -lab's license does not impose terms on what you build. +MIT — see [LICENSE](LICENSE). The lab's license does not impose terms +on your module under `module/`. diff --git a/examples/chuck_norise/Makefile b/examples/chuck_norise/Makefile index 5aa9583..c196f49 100644 --- a/examples/chuck_norise/Makefile +++ b/examples/chuck_norise/Makefile @@ -1,18 +1,13 @@ # Standard out-of-tree Linux kernel module Makefile. # -# This file is read in two different modes: -# -# 1. As a Kbuild fragment, when the kernel build system pulls it in via -# `make -C $KDIR M=$THIS_DIR modules`. In that mode KERNELRELEASE is set, -# so the Kbuild assignments below run and the wrapper recipe is skipped. -# -# 2. As a standalone Makefile, when a developer runs `make` in this directory -# directly. In that mode KERNELRELEASE is unset, so the wrapper recipe runs -# and re-invokes mode 1 against the host kernel's build tree. -# -# The lab's outer build (scripts/03-build-module.sh) always drives mode 1 -# against a target-specific kernel cache, but mode 2 is useful for ad-hoc -# builds against the host kernel. +# Read in two modes: +# 1. As a Kbuild fragment when the kernel build system pulls it in via +# `make -C $KDIR M=$THIS_DIR modules` (KERNELRELEASE is set; Kbuild +# assignments run, wrapper recipe is skipped). This is the mode the +# lab's outer build uses. +# 2. As a standalone Makefile when you run `make` in this dir directly +# (KERNELRELEASE is unset; the wrapper recipe re-invokes mode 1 +# against the host kernel). obj-m += chuck_norise.o chuck_norise-y := src/hello.o src/chuck_device.o src/chuck_message.o diff --git a/examples/chuck_norise/README.md b/examples/chuck_norise/README.md index 92b2a7d..c3f8e7b 100644 --- a/examples/chuck_norise/README.md +++ b/examples/chuck_norise/README.md @@ -1,53 +1,38 @@ # chuck_norise — example module -A small multi-file char device module used as the worked example for this lab. -Builds as `chuck_norise.ko` and exposes `/dev/chuck_norise`. Reads return the -exact string `chuck norise!` repeated indefinitely, preserving the file offset -for each open file descriptor. +Worked example for the lab. Builds as `chuck_norise.ko` and exposes +`/dev/chuck_norise`. Reads return `chuck norise!` repeated, preserving +the file offset. -It is intentionally tiny so it stays useful as a template walk-through: an init -that sleeps long enough to give you time to attach GDB, a cdev/class lifecycle -split into its own file, and a separately-testable "produce bytes into a -userspace buffer" routine. +## Run it through the lab -## Use it as a template - -The repo's build pipeline operates on whatever lives under `module/` at the -repo root. To try this example end-to-end, copy it into place from the -repo root: +From the repo root: ```bash make use-example NAME=chuck_norise ``` -Then run the normal lab flow (`scripts/00-...` through `scripts/04-...`, -or press F5 in VS Code — see the top-level [README](../../README.md) for -the full quickstart). After the module loads on the target VM: +Then follow the [top-level Quickstart](../../README.md#quickstart). After +the module loads on the target VM: ```bash -head -c 10 /dev/chuck_norise # prints "chuck nori" -``` - -To see offset-preserving reads from a single open fd: +head -c 10 /dev/chuck_norise # chuck nori -```bash +# offset-preserving reads from a single open fd exec 9/dev/null # chu -dd bs=1 count=5 <&9 2>/dev/null # ck no -dd bs=1 count=7 <&9 2>/dev/null # rise!ch +dd bs=1 count=3 <&9 2>/dev/null # chu +dd bs=1 count=5 <&9 2>/dev/null # ck no +dd bs=1 count=7 <&9 2>/dev/null # rise!ch exec 9<&- ``` -## What this example demonstrates - -- A standard out-of-tree Linux kernel module project layout (`src/`, `include/`, - `Makefile`). -- A `Makefile` that doubles as a Kbuild fragment and as a standalone wrapper — - the conventional shape for kernel modules. The lab's outer build invokes it - the Kbuild way; running `make` in this directory directly invokes the wrapper. -- A module init that sleeps for `debug_delay_ms` (default 5000 ms, exposed as - a module parameter) before doing anything observable. The delay gives the - host time to attach GDB and set breakpoints before init runs. -- A `LINUX_VERSION_CODE` shim for the 6.4 `class_create()` signature change, - as a real-world example of one of the things out-of-tree modules have to - carry. +## What this example shows + +- The conventional out-of-tree layout (`src/`, `include/`, `Makefile`). +- A `Makefile` that doubles as a Kbuild fragment AND a standalone + wrapper (`ifndef KERNELRELEASE` clause). +- A `debug_delay_ms` module parameter so the host has time to attach + GDB before init runs. +- A `LINUX_VERSION_CODE` shim for the 6.4 `class_create()` signature + change — a real-world example of the compat carrying out-of-tree + modules have to do. diff --git a/examples/chuck_norise/src/chuck_device.c b/examples/chuck_norise/src/chuck_device.c index cb6758d..d0d3dcf 100644 --- a/examples/chuck_norise/src/chuck_device.c +++ b/examples/chuck_norise/src/chuck_device.c @@ -29,17 +29,14 @@ static ssize_t chuck_read(struct file *file, char __user *buffer, size_t count, return chuck_message_read(buffer, count, position); } +/* .llseek deliberately unset — kernel installs default_llseek for char + * devices, which matches chuck_read's behavior (it respects *position). + * Don't reach for no_llseek; it was removed in 6.12 (commit 868941b14441). + */ static const struct file_operations chuck_fops = { .owner = THIS_MODULE, .open = chuck_open, .read = chuck_read, - /* - * .llseek deliberately unset. The kernel installs default_llseek for - * char devices, which matches chuck_read's behavior (it advances and - * respects *position). Earlier versions of this file pinned - * .llseek = no_llseek, but that symbol was removed in Linux 6.12 - * (mainline commit 868941b14441 "fs: remove no_llseek"). - */ }; static struct class *chuck_class_create(void) diff --git a/examples/chuck_norise/src/hello.c b/examples/chuck_norise/src/hello.c index 406b46d..d29c98e 100644 --- a/examples/chuck_norise/src/hello.c +++ b/examples/chuck_norise/src/hello.c @@ -38,5 +38,5 @@ module_init(hello_init); module_exit(hello_exit); MODULE_LICENSE("GPL"); -MODULE_AUTHOR("small-ko lab"); -MODULE_DESCRIPTION("Multi-file char device module for kernel source debugging"); +MODULE_AUTHOR("Linux Kernel Module Debug Lab"); +MODULE_DESCRIPTION("Multi-file char device example for kernel source debugging"); diff --git a/module/README.md b/module/README.md index 3026dcd..397d41e 100644 --- a/module/README.md +++ b/module/README.md @@ -1,75 +1,27 @@ # module/ — your kernel module goes here -This directory is the lab's only slot for the kernel module being built and -debugged. The repo is otherwise generic — it does not know your module's -name, source layout, or what it does. +The lab's only slot for the module being built and debugged. The repo is +otherwise generic — it doesn't know your module's name, source layout, +or what it does. -## Contract - -Drop a standard out-of-tree Linux kernel module project here. The lab -expects: - -- **`module/Makefile`** (or `module/Kbuild`) following standard Kbuild - conventions for out-of-tree modules: - - ```makefile - obj-m += my_module.o - my_module-y := src/main.o src/util.o - ccflags-y := -I$(src)/include -g -DDEBUG - ``` - - See `examples/chuck_norise/Makefile` for the canonical shape, including - the optional `ifndef KERNELRELEASE` wrapper that lets the same file be - driven by `make` directly. - -- **Sources anywhere you like.** The lab does not impose a `src/` or - `include/` layout — your Makefile's `*-y` line picks the source files - and `ccflags-y` picks the include paths. - -- **Exactly one `obj-m` entry per build.** The lab discovers the module - name from the produced `.ko`, so you don't declare it anywhere - else. - -- **Optionally a `debug_delay_ms` module parameter** that sleeps in - `module_init` so the host has time to attach GDB before init runs. - Declared as: - - ```c - static unsigned int debug_delay_ms = 5000; - module_param(debug_delay_ms, uint, 0644); - ``` - - When declared, scripts/04 passes `DEBUG_LOAD_DELAY_MS` from - `lab.local.env` to `insmod`. When not declared, the module loads - without it. - -That's the entire contract. The lab handles per-target staging, DWARF -path rewriting, uploading, loading, and symbol discovery on the target. +**Contract:** see the [top-level README](../README.md#the-module-contract). +TL;DR: a standard out-of-tree `Makefile` (or `Kbuild`) with one `obj-m` +entry; sources anywhere you like; an optional `debug_delay_ms` module +parameter so the host has time to attach GDB. ## Try the example ```bash -make use-example NAME=chuck_norise -``` - -Then build and debug as in the top-level `README.md`. To reset: - -```bash -make clean-module +make use-example NAME=chuck_norise # copy examples/chuck_norise/ into module/ +make clean-module # reset module/ to just this README ``` -## Replacing the example with your own module - -Run `make clean-module`, then drop your project in. Edit your `Makefile` -to set `obj-m`, `-y`, and `ccflags-y`. Rebuild. - ## One caveat: no symlinks inside `module/` The lab stages your sources into `build/intermediate///` -as a tree of absolute symlinks (`cp -as`) and then runs Kbuild from -there. If `module/` contains its OWN symlinks, the staged copy ends up -with symlinks pointing at the original symlinks — Kbuild will still find -the files, but the DWARF prefix-map remap (which assumes source files -live under `module/`) may not produce the paths your IDE expects. Keep -`module/` symlink-free, or copy in real files for any external sources -you depend on. +as a tree of absolute symlinks (`cp -as`) and runs Kbuild there. If +`module/` contains its OWN symlinks, the staged copy ends up with +symlinks-to-symlinks; the DWARF prefix-map remap (which assumes sources +live under `module/`) won't produce the paths your IDE expects. Keep +`module/` symlink-free, or copy in real files for external sources you +need. diff --git a/scripts/00-check-target.sh b/scripts/00-check-target.sh index a286470..23551d5 100755 --- a/scripts/00-check-target.sh +++ b/scripts/00-check-target.sh @@ -1,19 +1,10 @@ #!/usr/bin/env bash -# Non-mutating health check for a target. Run this before scripts/01 to -# catch configuration mistakes before any state on the target changes. +# Non-mutating health check for a target. Run before scripts/01 to catch +# configuration mistakes before any state on the target changes. # -# Reports, one line each: -# ssh is the target reachable over ssh? -# sudo does the configured (or fallback) sudo work? -# os distro and codename (must be ubuntu for now) -# kernel running kernel release -# headers /lib/modules/$kernel/build present? -# vmlinux debug-symbol vmlinux available? -# kernel-source linux-source-* package extracted? -# debug endpoint TCP port for the picked debug method reachable from the dev host? -# -# A "debug method" argument is optional; if omitted, both kgdb and qemu -# endpoints are checked when configured. +# Reports ssh / sudo / os / kernel / headers / vmlinux / kernel-source / +# debug endpoint. The debug-method arg is optional — both kgdb and qemu +# endpoints are checked when omitted. set -euo pipefail usage() { @@ -35,7 +26,7 @@ env_file="$(lab_env_file "$repo_root")" source "$env_file" lab_load_target "$target" -# Colored OK / FAIL / WARN, falling back to plain text when stdout isn't a TTY. +# Colored OK / WARN / FAIL on a TTY; plain text when piped. if [[ -t 1 ]]; then GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[0;33m'; RESET='\033[0m' else diff --git a/scripts/01-provision-target.sh b/scripts/01-provision-target.sh index c01de37..6802ed7 100755 --- a/scripts/01-provision-target.sh +++ b/scripts/01-provision-target.sh @@ -1,17 +1,16 @@ #!/usr/bin/env bash # Provision a target VM for kernel module debugging. # -# Target-side setup: install kernel headers (so the host build can sync them), -# optionally install matching vmlinux debug image + kernel source for -# step-into-kernel, and update the guest's GRUB command line with the boot -# args this lab needs (nokaslr always; kgdboc + sysrq for the kgdb method). +# Installs kernel headers (so the host build can sync them), optionally the +# matching vmlinux debug image + kernel source for step-into-kernel, and +# adds the boot args this lab needs to the guest's GRUB command line: +# - nokaslr (both methods; lets vmlinux symbols line up) +# - kgdboc + sysrq (kgdb method only) +# - maxcpus (only if DEBUG_MAXCPUS is set in the env) # -# With --uninstall, remove every boot arg the lab added (nokaslr / kgdboc / -# sysrq_always_enabled / maxcpus) — but leave installed packages alone. -# -# Reboot the target after this script completes so the new boot args take -# effect. The script itself does not reboot — staying out of the user's way -# matters for VMs that are reverted from snapshots and shouldn't be touched. +# `--uninstall` removes every boot arg the lab added and leaves installed +# packages alone. Reboot the target after either mode for boot args to +# take effect — this script never reboots on its own. set -euo pipefail usage() { @@ -24,8 +23,6 @@ target="${1:-}" arg2="${2:-}" arg3="${3:-}" -# Two modes: normal provision ( [--debug-symbols]) and -# --uninstall (which doesn't need a debug method since it only undoes GRUB). uninstall=0 debug_method="" symbols="" @@ -66,9 +63,8 @@ case "$target_os" in *) die "unsupported TARGET_OS[$target]='$target_os'; only ubuntu targets are implemented" ;; esac -# Preflight: ssh works. We don't insist sudo works yet because the user -# might be running 01 *to* configure sudo. The actual provisioning shell -# (a piped `sudo -S bash -s`) will exercise sudo and fail with apt's own +# Skip the sudo check — the user might be running 01 *to* configure sudo. +# The provisioning shell below will exercise it and fail with apt's own # diagnostics if the password is wrong. lab_check_connection --no-sudo @@ -79,26 +75,16 @@ else echo "provisioning $target ($LAB_SSH_TARGET:$LAB_SSH_PORT) for debug method: $debug_method$symbols_blurb" fi -# Run the remote script as root via SUDO_ASKPASS so stdin stays a clean -# pipe of "env-vars then script body" for bash -s. -# -# Why NOT `sudo -S` + piped password on stdin: sudo only reads stdin when -# policy requires it. With NOPASSWD configured, or cached creds, sudo -# skips the stdin read entirely and the password line we piped leaks into -# bash -s as the first script line — producing -# `bash: line 1: : command not found`. `sudo -k` doesn't help -# because NOPASSWD is a policy override, not a cache thing. -# -# SUDO_ASKPASS works regardless: sudo invokes the askpass helper to fetch -# the password (or doesn't, under NOPASSWD), and stdin is purely the -# bash -s script. We scp the helper + a chmod-600 password file to the -# target up front; a trap cleans both up on script exit. +# We run the remote script as root via `sudo -A bash -s` so stdin stays a +# clean pipe of env-var-assignments + script body for bash to read. The +# straightforward "pipe password to sudo -S" pattern doesn't work here +# because sudo skips the stdin read whenever policy doesn't require auth +# (NOPASSWD, cached creds), and the password line then leaks into bash as +# its first command. SUDO_ASKPASS routes the password through a helper +# script so stdin is never contended. askpass="$LAB_REMOTE_DIR/.kmod-askpass-$$" pwfile="$LAB_REMOTE_DIR/.kmod-sudo-pw-$$" -cleanup_sudo_helpers() { - lab_ssh "rm -f '$askpass' '$pwfile' 2>/dev/null" || true -} -trap cleanup_sudo_helpers EXIT +trap 'lab_ssh "rm -f $askpass $pwfile 2>/dev/null" || true' EXIT lab_ssh "mkdir -p '$LAB_REMOTE_DIR'; umask 077; cat > '$pwfile'" <<<"$LAB_SUDO_PASS" lab_ssh "umask 077; cat > '$askpass'; chmod 700 '$askpass'" <&2; exit 1; } -[[ "${ID:-}" == "ubuntu" ]] || { - echo "target is not Ubuntu (os-release ID=$ID); only ubuntu targets are implemented" >&2; exit 1; } +[[ "${TARGET_OS:-ubuntu}" == "ubuntu" ]] || + { echo "unsupported target OS: ${TARGET_OS:-}" >&2; exit 1; } +[[ "${ID:-}" == "ubuntu" ]] || + { echo "target is not Ubuntu (os-release ID=$ID); only ubuntu targets are implemented" >&2; exit 1; } grub_file=/etc/default/grub +# Lab-managed boot args; stripped on every run so reruns replace rather +# than append, and `--uninstall` cleans them all out. grub_managed_keys='kgdboc=|maxcpus=|sysrq_always_enabled=|nokaslr' -# Read current GRUB_CMDLINE_LINUX_DEFAULT, stripping the lab's managed keys -# so reruns replace rather than append. `nokaslr` is a bare token (no =), -# matched as a standalone word. read_current_cmdline() { local raw stripped raw="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT="\{0,1\}\([^"]*\)"\{0,1\}/\1/p' "$grub_file" | head -1)" - # Strip key=value forms and the bare "nokaslr" token; collapse spaces. stripped="$(printf '%s' "$raw" | sed -E "s/(^| )($grub_managed_keys)([^ ]*)?/ /g; s/ */ /g; s/^ +//; s/ +$//")" printf '%s' "$stripped" } write_cmdline() { - local newline="$1" cp "$grub_file" "$grub_file.kmod-debug-lab.$(date +%Y%m%d%H%M%S).bak" if grep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' "$grub_file"; then - sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$newline\"|" "$grub_file" + sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$1\"|" "$grub_file" else - printf 'GRUB_CMDLINE_LINUX_DEFAULT="%s"\n' "$newline" >> "$grub_file" + printf 'GRUB_CMDLINE_LINUX_DEFAULT="%s"\n' "$1" >> "$grub_file" fi update-grub } @@ -190,27 +173,23 @@ DDEBS apt_retry update apt_retry install -y "linux-image-${kernel}-dbgsym" || apt_retry install -y "linux-image-unsigned-${kernel}-dbgsym" || - { echo "failed to install debug symbols for $kernel; retry later if ddebs.ubuntu.com is returning 503" >&2; exit 1; } + { echo "failed to install dbgsym for $kernel; retry later if ddebs.ubuntu.com is returning 503" >&2; exit 1; } fi [[ -r "$vmlinux" ]] || echo "warning: $vmlinux still missing after install; kernel source debugging will be unavailable" - # Try the major.minor.patch-suffixed package first (e.g. linux-source-6.8.0); - # fall back to the unversioned meta-package. Extraction is deferred to - # script 02 — the host has more incentive to extract (it's where GDB lives) - # and doing it there avoids needing tar / sudo on the target. + # Try the major.minor.patch-suffixed package first (linux-source-6.8.0), + # falling back to the meta-package. We don't extract here — script 02 + # does it host-side to avoid needing tar on the target. short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" src_pkg="" for candidate in "linux-source-$short_kver" "linux-source"; do [[ -z "$candidate" || "$candidate" == "linux-source-" ]] && continue - if apt_retry install -y "$candidate"; then - src_pkg="$candidate" - break - fi + apt_retry install -y "$candidate" && { src_pkg="$candidate"; break; } done if [[ -z "$src_pkg" ]]; then echo "warning: could not install a linux-source package; step-into-kernel will only show disassembly" >&2 elif ls -1 /usr/src/linux-source-*.tar.* /usr/src/linux-source-*/linux-source-*.tar.* 2>/dev/null | head -1 >/dev/null; then - echo " kernel source tarball is in /usr/src on the target; script 02 will sync and extract it" + echo " kernel source tarball is in /usr/src; script 02 will sync and extract it" else echo "warning: $src_pkg installed but no linux-source tarball found under /usr/src/" >&2 fi @@ -221,18 +200,9 @@ fi echo "[3/3] updating GRUB command line" current="$(read_current_cmdline)" -# nokaslr is required in both modes so vmlinux symbols line up with running -# addresses. kgdb mode also needs an in-kernel debugger channel (kgdboc) and -# sysrq enabled so users can trigger a halt with `echo g > /proc/sysrq-trigger` -# — there is no kgdb_breakpoint() call in any user module to halt on insmod. -# qemu mode skips both: the hypervisor stub halts the vCPU directly. args=("nokaslr") -if [[ "$DEBUG_METHOD" == "kgdb" ]]; then - args+=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "sysrq_always_enabled=1") -fi -if [[ -n "${DEBUG_MAXCPUS:-}" ]]; then - args+=("maxcpus=${DEBUG_MAXCPUS}") -fi +[[ "$DEBUG_METHOD" == "kgdb" ]] && args+=("kgdboc=${KGDB_TTY},${KGDB_BAUD}" "sysrq_always_enabled=1") +[[ -n "${DEBUG_MAXCPUS:-}" ]] && args+=("maxcpus=${DEBUG_MAXCPUS}") for arg in "${args[@]}"; do case " $current " in *" $arg "*) ;; *) current="${current:+$current }$arg" ;; esac done diff --git a/scripts/02-setup-host-build.sh b/scripts/02-setup-host-build.sh index bb63dbd..c064a41 100755 --- a/scripts/02-setup-host-build.sh +++ b/scripts/02-setup-host-build.sh @@ -1,15 +1,13 @@ #!/usr/bin/env bash # Sync the target's kernel build tree to the host so an out-of-tree module -# can be cross-built against the target's exact kernel without copying the -# whole source tree per build. +# can be cross-built against the target's exact kernel. # -# Result: .kernel-cache// populated with: -# build/ -> symlink into the synced linux-headers tree -# source/ -> symlink into the synced linux-source tree (if --debug-symbols was used in 01) +# Populates .kernel-cache// with: +# build/ -> synced linux-headers tree +# source/ -> synced linux-source tree (if --debug-symbols was used in 01) # vmlinux -> debug-symbol vmlinux from the target (if available) -# kernel.release -> running kernel release (e.g. 6.8.0-117-generic) -# remote.build.path -> where headers came from on the target -# remote.header.paths -> list of all paths synced from /usr/src/ +# kernel.release -> running kernel release +# remote.{build,header}.* -> bookkeeping # Also updates .kernel-cache/current -> so plain `make` and the # IntelliSense config track the most recently synced target. set -euo pipefail @@ -47,17 +45,8 @@ apt_get() { echo "[1/5] installing host build prerequisites" apt_get update apt_get install -y \ - bc \ - bison \ - build-essential \ - dwarves \ - flex \ - gdb \ - libelf-dev \ - libssl-dev \ - openssh-client \ - rsync \ - sshpass + bc bison build-essential dwarves flex gdb \ + libelf-dev libssl-dev openssh-client rsync sshpass # --- Discover the target's kernel + header layout ------------------------- @@ -99,10 +88,9 @@ done REMOTE ) -# Kernel source is discovered separately: it can be a tarball (a single file -# at /usr/src/linux-source-X.tar.bz2) OR a directory (possibly containing -# the tarball inside it, on some Ubuntu releases). One round-trip classifies -# each path as file|dir so the sync loop stays trivial. +# Kernel source is found separately: it can be a tarball-file or a dir +# (sometimes with the tarball inside it). Classify each path as file|dir so +# the sync loop below stays trivial. mapfile -t remote_source_entries < <(lab_ssh "bash -s" <<'REMOTE' set -euo pipefail shopt -s nullglob @@ -123,20 +111,16 @@ REMOTE # --- Sync ---------------------------------------------------------------- sync_header_dir() { - local remote_dir="$1" - local dest + local remote_dir="$1" dest dest="$usr_src_dir/$(basename "$remote_dir")" echo " $remote_dir -> ${dest#"$repo_root"/}" - # Preserve symlinks inside Ubuntu's kernel header trees. Some optional - # symlinks (e.g. rust support) may be dangling and are harmless for - # external C module builds. + # Preserve symlinks inside Ubuntu's header trees (e.g. dangling rust + # links — harmless for external C builds). lab_rsync_from "$remote_dir/" "$dest/" --delete } sync_source_path() { - local remote_path="$1" - local kind="$2" - local dest + local remote_path="$1" kind="$2" dest dest="$usr_src_dir/$(basename "$remote_path")" echo " $remote_path -> ${dest#"$repo_root"/}" if [[ "$kind" == "dir" ]]; then @@ -147,10 +131,8 @@ sync_source_path() { } echo "[3/5] syncing kernel headers and source" -# Dedup the discovered header paths; the discovery on the target can -# legitimately list the same canonical path twice (e.g. when -# /lib/modules/$kernel/source and /usr/src/linux-headers-$base resolve to -# the same dir via symlink chains). +# Dedup discovered paths — two map entries can canonicalize to the same +# dir via symlink chains. printf '%s\n' "${remote_header_dirs[@]}" | awk 'NF && !seen[$0]++' | while IFS= read -r remote_dir; do sync_header_dir "$remote_dir" @@ -170,34 +152,30 @@ ln -s "usr-src/$build_name" "$build_dir" echo "[4/5] checking vmlinux debug image" if lab_ssh_sudo "test -r '$remote_vmlinux'"; then - # Use rsync over sudo: skips when unchanged, restartable, checksummed. - # Without this the cat-based fetch would silently corrupt vmlinux any - # time sudo printed anything unexpected to stdout or the SSH connection - # dropped mid-transfer (the file is ~415MB so the window is non-trivial). + # rsync-over-sudo: skips unchanged, restartable, checksummed. Replaces + # an earlier cat-over-ssh approach that silently corrupted vmlinux on + # any sudo banner or connection drop (file is ~415MB). echo " rsync $remote_vmlinux -> ${cache_dir#"$repo_root"/}/vmlinux" lab_rsync_from "$remote_vmlinux" "$cache_dir/vmlinux" --rsync-path='sudo rsync' else rm -f "$cache_dir/vmlinux" echo " no vmlinux on target ($remote_vmlinux is missing)" echo " module debugging will work; kernel source debugging will not" - echo " to enable, run: scripts/01-provision-target.sh $target --debug-symbols" + echo " to enable: scripts/01-provision-target.sh $target --debug-symbols" fi # --- Kernel source resolution -------------------------------------------- # -# Find a usable kernel-source root under usr-src/. Ubuntu has shipped three -# layouts over time: +# Ubuntu has shipped three linux-source layouts: # 1. /usr/src/linux-source-X.tar.bz2 (tarball, no enclosing dir) -# 2. /usr/src/linux-source-X/ (pre-extracted, files at top) -# 3. /usr/src/linux-source-X/linux-source-X/ (pre-extracted, nested one deep) -# Layout 1 also occurs nested as /usr/src/linux-source-X/linux-source-X.tar.bz2. +# 2. /usr/src/linux-source-X/ (extracted, files at top) +# 3. /usr/src/linux-source-X/linux-source-X/ (extracted, nested one deep) +# Plus the variant where the tarball lives inside the same-named dir. # -# Strategy: look for a Makefile + init/main.c (the kernel's top-level -# markers) at depth ≤ 3 under usr-src/. If found, symlink directly to it; -# otherwise look for a tarball and extract on the host into source-tree/. -# Extraction on the host (not the target) avoids a sudo+tar dependency on -# the target for what's purely a host-side debug resource, and makes the -# step self-healing: re-running 02 fixes partial/broken trees. +# Strategy: look for a Makefile + init/main.c (the kernel-source markers) +# at depth ≤ 3 under usr-src/. If found, symlink to it. Otherwise extract +# a tarball into source-tree/ on the host (self-healing on rerun, and +# avoids needing tar on the target). echo "[5/5] resolving kernel source for step-into-kernel" source_link="$cache_dir/source" @@ -208,17 +186,14 @@ is_kernel_source_root() { [[ -f "$1/Makefile" && -d "$1/init" && -f "$1/init/main.c" ]] } -# Pick the source dir matching the live kernel's major.minor.patch when -# possible (handles the case where multiple linux-source-X packages are -# installed on the target). Fall back to the highest-versioned root. +# Prefer the linux-source-X dir whose X matches the live kernel; fall +# back to the highest version installed. short_kver="$(printf '%s' "$kernel" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+' || true)" found_root="" matching_root="" while IFS= read -r mf; do candidate="$(dirname "$mf")" if is_kernel_source_root "$candidate"; then - # `sort -V` below means "first match" is also the highest version; - # remember the first hit, then prefer a kver-matching one if seen. [[ -z "$found_root" ]] && found_root="$candidate" if [[ -n "$short_kver" && "$candidate" == *"linux-source-$short_kver"* ]]; then matching_root="$candidate" @@ -252,7 +227,7 @@ else else rm -rf "$source_tree" echo " no kernel source on target; kernel step-into will show disassembly only" - echo " to enable, run: scripts/01-provision-target.sh $target --debug-symbols" + echo " to enable: scripts/01-provision-target.sh $target --debug-symbols" fi fi diff --git a/scripts/03-build-module.sh b/scripts/03-build-module.sh index 2404f6b..d34d096 100755 --- a/scripts/03-build-module.sh +++ b/scripts/03-build-module.sh @@ -1,11 +1,10 @@ #!/usr/bin/env bash # Build module/ against the synced headers for . # -# Driven by the top-level Makefile, which stages module/ into +# Delegates to the top-level Makefile, which stages module/ into # build/intermediate/// and runs Kbuild from there with -# -ffile-prefix-map injected via KCFLAGS. Result is one .ko under -# build/artifacts///. The module name is whatever the user's -# obj-m declares; the lab does not need to know it. +# -ffile-prefix-map injected via KCFLAGS. Result: one .ko under +# build/artifacts///. set -euo pipefail target="${1:-}" diff --git a/scripts/04-deploy-debug-vscode.sh b/scripts/04-deploy-debug-vscode.sh index dcf1c81..976f0b0 100755 --- a/scripts/04-deploy-debug-vscode.sh +++ b/scripts/04-deploy-debug-vscode.sh @@ -2,25 +2,18 @@ # Deploy the built module to a target, load it, and write the GDB init files # VS Code (or `gdb -tui`) needs to attach and source-debug it. # -# Steps: -# 1. Free the debug-endpoint TCP port from any stale GDB clients. Both -# KGDB's serial bridge and QEMU's gdbstub serve one client at a time, -# so any leftover gdb owning that socket blocks fresh attaches. This -# script kills any *local* gdb process holding the endpoint — be aware -# if you have unrelated gdb sessions on the same host:port. -# 2. Upload module/.ko to the target. -# 3. Insmod it (with debug_delay_ms if the module declares the param, -# falling back to a plain insmod otherwise). -# 4. Poll /sys/module//sections/ for the module's runtime load -# addresses; convert to an `add-symbol-file ... -s .name addr ...` -# line saved as the per-(target,method) symbols file. -# 5. Emit .gdb/-.gdb and -attached.gdb variants, plus -# current-debug{,attached}.gdb and current-vmlinux symlinks the IDE -# launch configs point at. +# Phases: +# 1. Best-effort kill of any local gdb holding the debug-endpoint TCP +# port (KGDB serial bridge and QEMU gdbstub each serve one client). +# 2. Upload module/.ko, verify size matches. +# 3. Insmod in the background; poll /sys/module//sections/ for the +# runtime load addresses and emit an `add-symbol-file ...` line. +# 4. Emit .gdb/-.gdb (+ -attached variant) and the +# current-* symlinks the IDE launch configs point at. # -# Env knobs (default values in parens): +# Env knobs: # DEBUG_LOAD_DELAY_MS (5000) passed to the module if it declares debug_delay_ms -# INSMOD_WAIT_SECS (30) how long to wait for /sys/module/.../sections/.text +# INSMOD_WAIT_SECS (30) poll deadline for /sys/module/.../sections/.text set -euo pipefail usage() { @@ -42,12 +35,11 @@ env_file="$(lab_env_file "$repo_root")" source "$env_file" lab_load_target "$target" -endpoint_key="DEBUG_ENDPOINT_${debug_method^^}" -debug_endpoint="$(target_require_cfg "$target" "$endpoint_key")" +debug_endpoint="$(target_require_cfg "$target" "DEBUG_ENDPOINT_${debug_method^^}")" debug_delay_ms="${DEBUG_LOAD_DELAY_MS:-5000}" insmod_wait_secs="${INSMOD_WAIT_SECS:-30}" -# --- Locate built artifact + vmlinux -------------------------------------- +# --- Locate artifact + vmlinux, verify target is on the cached kernel ---- cache_dir="$repo_root/.kernel-cache/$target" kernel_file="$cache_dir/kernel.release" @@ -55,7 +47,6 @@ vmlinux="$cache_dir/vmlinux" [[ -f "$kernel_file" ]] || die "missing kernel cache for $target; run: scripts/02-setup-host-build.sh $target" - [[ -f "$vmlinux" ]] || die "missing $vmlinux — source debugging needs the matching vmlinux from the target. - install + sync it with: @@ -64,18 +55,14 @@ vmlinux="$cache_dir/vmlinux" kernel="$(<"$kernel_file")" -# Preflight: connection + sudo work, and the target is still running the -# kernel we cached. If the target rebooted into a different kernel since -# scripts/02 ran, vmlinux symbols won't match runtime addresses and GDB -# would silently show wrong source lines. lab_check_connection live_kernel="$(lab_ssh 'uname -r')" if [[ "$live_kernel" != "$kernel" ]]; then - die "kernel mismatch: target is now running '$live_kernel' but cache has '$kernel'. -- if the target was rebooted into a new kernel, re-sync: - scripts/01-provision-target.sh $target $debug_method --debug-symbols # if you also need the new vmlinux + die "kernel mismatch: target now runs '$live_kernel' but cache has '$kernel'. +- if the target rebooted into a new kernel, re-sync: + scripts/01-provision-target.sh $target $debug_method --debug-symbols # for a new vmlinux scripts/02-setup-host-build.sh $target -- if you booted the wrong kernel, reboot back into $kernel" +- or reboot the target back into $kernel" fi artifact_dir="$repo_root/build/artifacts/$target/$kernel" @@ -89,13 +76,13 @@ case ${#artifacts[@]} in *) echo "expected one .ko under $artifact_dir; found:" >&2 printf ' %s\n' "${artifacts[@]}" >&2 - die "the lab assumes a single obj-m per build. Merge sources into one module (obj-m += foo.o; foo-y := a.o b.o ...) or split into separate module/ trees." + die "the lab assumes a single obj-m per build. Merge into one module (obj-m += foo.o; foo-y := a.o b.o ...) or split into separate module/ trees." ;; esac artifact="${artifacts[0]}" module_name="$(basename "$artifact" .ko)" -# --- GDB output paths ----------------------------------------------------- +# --- Output paths --------------------------------------------------------- gdb_dir="$repo_root/.gdb" mkdir -p "$gdb_dir" @@ -107,23 +94,16 @@ loader_log="$gdb_dir/$target-$debug_method-loader.log" rm -f "$symbols_file" "$loader_log" -# --- Free the debug endpoint from stale GDB clients ----------------------- -# -# Both KGDB's serial bridge and QEMU's gdbstub serve one client at a time. -# A zombie gdb left over from a previous session sits in the endpoint's -# accept queue and blocks fresh attaches. We best-effort kill any local gdb -# process that holds a connection to the endpoint. Errors here never fail -# the deploy — at worst the user has to `killall gdb` manually. +# --- Free the debug endpoint from stale GDB clients ---------------------- # -# `lsof -ti` is more robust than parsing `ss -ntp` output: it gives one PID -# per line and won't break across iproute2 versions. +# Best-effort: lsof to find sockets on the endpoint, ps to confirm comm=gdb, +# kill -9. Errors here never fail the deploy. free_debug_endpoint() { command -v lsof >/dev/null 2>&1 || return 0 local host="${debug_endpoint%:*}" port="${debug_endpoint##*:}" local pids pids="$(lsof -ti "@$host:$port" 2>/dev/null || true)" - local gdb_pids=() - local pid + local gdb_pids=() pid for pid in $pids; do [[ "$(ps -p "$pid" -o comm= 2>/dev/null || true)" == "gdb" ]] && gdb_pids+=("$pid") done @@ -136,71 +116,57 @@ free_debug_endpoint() { } free_debug_endpoint || true -# --- Upload + load on target ---------------------------------------------- +# --- Upload + load ------------------------------------------------------- remote_module="$LAB_REMOTE_DIR/$module_name.ko" echo "uploading $artifact -> $LAB_SSH_TARGET:$remote_module" lab_ssh "mkdir -p '$LAB_REMOTE_DIR'" lab_scp_to "$artifact" "$remote_module" -# Verify the upload — a truncated .ko would silently insmod-fail with cryptic -# "invalid module format" errors. +# Verify size — a truncated .ko produces cryptic "invalid module format" errors. local_size="$(stat -c %s "$artifact")" remote_size="$(lab_ssh "stat -c %s '$remote_module' 2>/dev/null" | tr -d '[:space:]')" [[ "$remote_size" == "$local_size" ]] || die "upload size mismatch: local $local_size bytes, remote ${remote_size:-missing} bytes; retry scripts/04 or check disk space on the target" -# Load the module and discover the addresses /sys/module//sections/ -# exposes once it's live. The insmod runs inside a single nohup'd sh -c so -# the debug_delay_ms-then-plain-insmod fallback chains correctly: if we -# instead ran two `nohup ... &` invocations the shell would background them -# independently and the `||` between them would be a no-op (it'd just check -# the success of the backgrounding, not the insmod itself). +# Background insmod on the target so we can poll for sections in parallel +# with the module's debug_delay_ms sleep. +# +# Three load-bearing details: +# - `<&0` keeps sudo's stdin attached to the SSH-inherited pipe (where +# the password arrives). Non-job-control bash auto-redirects async +# stdin to /dev/null without an explicit redirection — sudo would +# see EOF and silently fail to authenticate. +# - `& sleep 1` keeps the SSH session open long enough for sudo to +# authenticate and exec nohup before the channel closes. +# - `nohup` + non-interactive bash → the orphaned chain survives +# session exit (bash doesn't huponexit non-interactively). # -# Output is captured to $loader_log; we surface either the success tail -# line or the full log on failure. +# `|| insmod ...` falls back when the module doesn't declare a +# debug_delay_ms parameter (kv arg would otherwise reject with EINVAL). load_and_discover_symbols() { echo "loading $module_name on $target (debug_delay_ms=$debug_delay_ms)" lab_ssh_sudo "rmmod '$module_name' >/dev/null 2>&1 || true" - # Background insmod on the target so we can poll /sys/module/.../sections - # in parallel with the module's debug_delay_ms sleep. - # - # Three subtleties: - # - `<&0` keeps sudo's stdin attached to the SSH-inherited pipe (where - # the password arrives). Non-job-control bash automatically redirects - # backgrounded processes' stdin to /dev/null UNLESS the command has - # its own stdin redirection — without `<&0`, sudo would see EOF - # instead of the password and silently fail to authenticate. - # - `& sleep 1` keeps the SSH session open long enough for sudo to - # read the password and exec into nohup. The sleep is short because - # sudo authentication is fast; if SSH closed earlier the in-flight - # password write could be lost. - # - The orphaned nohup+sh+insmod chain survives session exit because - # non-interactive bash doesn't huponexit. - # - # The `|| insmod ...` falls back when the module doesn't declare a - # debug_delay_ms parameter (the kv arg would otherwise reject with EINVAL). lab_ssh_sudo "nohup sh -c 'insmod \"$remote_module\" debug_delay_ms=\"$debug_delay_ms\" 2>/dev/null || insmod \"$remote_module\"' > '$LAB_REMOTE_DIR/insmod.log' 2>&1 <&0 & sleep 1" echo "polling /sys/module/$module_name/sections/ (timeout ${insmod_wait_secs}s)" - # Dump every readable file under /sys/module//sections/. Discovery - # is dynamic so modules with non-standard sections (.text.hot, custom - # __ksymtab subsections, ...) get their addresses picked up too. + # Dump every readable file under sections/. Dynamic discovery handles + # non-standard sections (.text.hot, custom __ksymtab subsections, ...). # # The sh -c body MUST be single-quoted in the SSH command so the outer - # remote bash doesn't expand $(ls -A) and $f before sh -c sees them. - # With double quotes, the outer bash would evaluate the substitution in - # its own CWD; the for loop would iterate over the wrong filenames and - # we'd never find .text — even though the module had already loaded. + # remote bash doesn't expand $(ls -A) and $f before sh -c sees them — + # with double quotes the outer bash would expand in its own CWD and + # the for loop would iterate over the wrong filenames. local dump_body - # shellcheck disable=SC2016 # $(...) is intentionally not expanded locally — see comment above. + # shellcheck disable=SC2016 # $(...) is intentionally deferred to sh -c. dump_body='cd "/sys/module/'"$module_name"'/sections" 2>/dev/null && for f in $(ls -A 2>/dev/null); do [ -r "$f" ] && printf "%s %s\n" "$f" "$(cat "$f")"; done' - local deadline=$((SECONDS + insmod_wait_secs)) - local tmp + local deadline=$((SECONDS + insmod_wait_secs)) tmp tmp="$(mktemp)" while ((SECONDS < deadline)); do + # No sleep — each SSH round-trip already takes 0.3-1s, which is + # the right polling cadence. if lab_ssh_sudo "sh -c '$dump_body'" > "$tmp" 2>/dev/null; then local text_addr text_addr="$(awk '$1 == ".text" { print $2 }' "$tmp")" @@ -218,8 +184,6 @@ load_and_discover_symbols() { return 0 fi fi - # No explicit sleep — each SSH round-trip already takes 0.3-1s, which - # is the right polling cadence. done rm -f "$tmp" @@ -241,12 +205,12 @@ else exit 1 fi -# --- Build the GDB init files -------------------------------------------- +# --- GDB init files ------------------------------------------------------ # -# If the kernel source has been synced (scripts/01 --debug-symbols + scripts/02), -# discover Ubuntu's build-time source prefix from vmlinux so GDB can remap -# DWARF references to our local copy. We ask addr2line where `start_kernel` -# lives — the path is always /init/main.c, so stripping the +# If the kernel source has been synced (scripts/01 --debug-symbols + +# scripts/02), discover Ubuntu's build-time source prefix from vmlinux so +# GDB can remap DWARF references to our local copy. addr2line on +# `start_kernel` returns `/init/main.c`; stripping the # known suffix yields the prefix. kernel_substitute_line="" kernel_directory_line="" @@ -255,9 +219,8 @@ if [[ -L "$kernel_src_link" || -d "$kernel_src_link" ]]; then kernel_src_root="$(readlink -f "$kernel_src_link")" kernel_directory_line="directory $kernel_src_root" if command -v nm >/dev/null 2>&1 && command -v addr2line >/dev/null 2>&1; then - # `|| true` because pipefail + SIGPIPE: nm dumps every vmlinux - # symbol, awk's `exit` after the first match closes the pipe and nm - # gets SIGPIPE on its next write. We still capture awk's output. + # `|| true` because awk's `exit` SIGPIPEs nm, and pipefail would + # kill the script. We still capture awk's output before exit. sym_addr="$(nm "$vmlinux" 2>/dev/null | awk 'NF==3 && $3=="start_kernel" {print $1; exit}' || true)" if [[ -n "$sym_addr" ]]; then sym_loc="$(addr2line -e "$vmlinux" "$sym_addr" 2>/dev/null | head -1 | cut -d: -f1 || true)" @@ -301,9 +264,9 @@ source $symbols_file EOF } > "$gdb_file" -# Native Debug ("type": "gdb") and cppdbg with miDebuggerServerAddress have -# already called `target remote` and loaded the executable by the time they -# source our script, so commands that touch global gdb state error with +# Native Debug and cppdbg-with-miDebuggerServerAddress already called +# `target remote` and loaded the executable by the time they source this +# script — so commands that change global gdb state error with # "Cannot change this setting while the inferior is running". Strip them. grep -vE '^(target remote |set mi-async |set target-async |set tcp connect-timeout |set remotetimeout |set architecture |symbol-file )' \ "$gdb_file" > "$gdb_attached_file" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index bb987d4..220c1b0 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -1,18 +1,10 @@ #!/usr/bin/env bash -# scripts/lib/common.sh — shared helpers sourced by every numbered script. +# Shared helpers sourced by every numbered script. # -# Naming: -# die, validate_target, target_* bare functions; no global state required. -# lab_* higher-level helpers; rely on globals -# populated by `lab_load_target`. -# LAB_* globals populated by `lab_load_target`. -# Scripts should call lab_ssh / lab_ssh_sudo -# / lab_scp_to / lab_rsync_from rather than -# touching the underlying ssh/scp/sshpass -# binaries directly. -# -# Sourcing this file is side-effect-free. State is created when a script -# calls `lab_load_target ` after sourcing `lab.local.env`. +# `die`, `validate_target`, `target_*` are state-free. +# `lab_*` helpers rely on globals set by `lab_load_target`. +# `LAB_*` globals are set by `lab_load_target`. Use the helpers, not the +# raw ssh/scp/sshpass binaries. die() { echo "$*" >&2 @@ -66,38 +58,30 @@ target_require_cfg() { require_debian_host() { command -v apt-get >/dev/null 2>&1 || - die "missing apt-get; the development host must be Debian-based (Debian, Ubuntu, WSL Ubuntu, ...)" + die "the development host must be Debian-based (apt-get not found)" } # --- Lab env loading ------------------------------------------------------ -# Validate that lab.local.env exists under repo_root and return its path on -# stdout. Caller is expected to `source` the returned path directly so the -# `declare -A TARGET_*=...` entries land in the script's global scope (a -# function-internal source would scope them to the function). +# Callers `source` the returned path directly so TARGET_* assignments land +# in script-global scope (sourcing from inside a function would scope them +# to the function). lab_env_file() { - local repo_root="$1" - local env_file="$repo_root/lab.local.env" + local env_file="$1/lab.local.env" [[ -f "$env_file" ]] || die "missing $env_file; copy lab.example.env to lab.local.env and edit it for your machines" printf '%s' "$env_file" } # --- Per-target connection setup ------------------------------------------ -# Read TARGET_* config for the given target and populate LAB_* globals used -# by every remote helper below. Validates that ssh password tooling is -# available if a password is configured. +# Read TARGET_* config and populate LAB_* globals: LAB_TARGET, LAB_SSH_HOST, +# LAB_SSH_PORT, LAB_SSH_USER, LAB_SSH_PASS, LAB_SUDO_PASS, LAB_REMOTE_DIR, +# LAB_SSH_TARGET, LAB_SSH_CMD[], LAB_SCP_CMD[], LAB_RSYNC_RSH. # -# After this returns, the following globals are set: -# LAB_TARGET target name -# LAB_SSH_HOST/PORT/USER raw connection details -# LAB_SSH_PASS ssh password ("" when using keys) -# LAB_SUDO_PASS sudo password ("" when sudo is passwordless / NOPASSWD) -# LAB_REMOTE_DIR target staging dir (with per-user default) -# LAB_SSH_TARGET "user@host" form for use in commands -# LAB_SSH_CMD array, ready to invoke (includes sshpass + timeouts) -# LAB_SCP_CMD array, ready to invoke (includes sshpass + timeouts) -# LAB_RSYNC_RSH rsync -e value (includes sshpass + timeouts) +# LAB_SSH_CMD / LAB_SCP_CMD are flag-only — they do NOT include the host. +# Helpers (and the few direct callers) append it themselves, so options +# like `-t` can be inserted before the host (ssh treats anything after +# the host as the remote command). lab_load_target() { local target="$1" target_is_configured "$target" @@ -112,9 +96,7 @@ lab_load_target() { LAB_REMOTE_DIR="$(target_cfg "$target" REMOTE_DIR)" LAB_SSH_PORT="${LAB_SSH_PORT:-22}" - # Sudo password falls back to ssh password — common single-user dev case. LAB_SUDO_PASS="${LAB_SUDO_PASS:-$LAB_SSH_PASS}" - # Per-user default so two devs on the same host don't collide on /tmp. LAB_REMOTE_DIR="${LAB_REMOTE_DIR:-/tmp/kmod-debug-lab-$LAB_SSH_USER}" local ssh_bin scp_bin sshpass_bin @@ -123,20 +105,14 @@ lab_load_target() { sshpass_bin="${SSHPASS_BIN:-sshpass}" if [[ -n "$LAB_SSH_PASS" ]] && ! command -v "$sshpass_bin" >/dev/null 2>&1; then - die "TARGET_SSH_PASS[$target] is set but '$sshpass_bin' is not installed; install it with: sudo apt-get install -y sshpass" + die "TARGET_SSH_PASS[$target] is set but '$sshpass_bin' isn't installed; install with: sudo apt-get install -y sshpass" fi LAB_SSH_TARGET="$LAB_SSH_USER@$LAB_SSH_HOST" - # LAB_SSH_CMD and LAB_SCP_CMD are flag-only — they do NOT include the - # target. Helpers and direct callers append the target themselves so - # `ssh ... -t user@host cmd` works (anything after the host is treated - # as the remote command by ssh). - # - # ConnectTimeout fails fast (10s) if the target is unreachable instead - # of blocking for minutes. ServerAliveInterval keeps long-lived - # connections (e.g. the 30s symbol-discovery loop) from being dropped - # by NAT idle timers. + # ConnectTimeout: fail fast on dead targets instead of multi-minute TCP wait. + # ServerAliveInterval: keep long-lived connections (e.g. the polling loop) + # alive through NAT idle timers. local ssh_opts=( -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 @@ -153,50 +129,37 @@ lab_load_target() { } # --- Remote command helpers ----------------------------------------------- -# -# All of these require a prior `lab_load_target` call. +# All require a prior `lab_load_target` call. -# Run a command on the target. Arguments are passed through to ssh as the -# remote command (typically one quoted shell string). Inherits stdin from -# the caller — handy when callers want to pipe data in (e.g. lab_ssh_sudo -# pipes the sudo password). +# Inherits stdin from the caller (so a caller can pipe data into the remote +# command — see lab_ssh_sudo). lab_ssh() { SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "$@" } -# Run a command on the target as root. The sudo password reaches the remote -# `sudo -S` via ssh's stdin, which means it never appears in any process's -# argv on the target — `ps auxww` is clean. When LAB_SUDO_PASS is empty -# (NOPASSWD sudo or sudo configured for passwordless), the empty string is -# piped and sudo proceeds without prompting. +# Run a command on the target as root. The password reaches `sudo -S` via +# ssh's stdin, so it never appears in argv on either host. # -# Why `-k`: without it, if sudo has cached credentials from an earlier -# invocation (e.g. `scripts/00-check-target.sh` just ran, or the user did -# `sudo` on the target within the last 5 min), `sudo -S` skips the stdin -# read entirely. The piped password then leaks into whatever inherits -# stdin from sudo — for `sudo bash -s`, the password line becomes bash's -# first script line and you get `bash: line 1: a: command not found`. -# `-k` ignores the cache for THIS invocation only; it does NOT invalidate -# the user's existing sudo timestamp. +# `-k` ignores any cached sudo timestamp (e.g. left by `scripts/00-check-target.sh`) +# for THIS invocation only — without it, sudo would skip the stdin read and +# the password would leak into the next reader. It does NOT invalidate the +# user's existing sudo cache. # -# Callers that need to feed their own stdin to the remote command (e.g. -# rsync) should use `lab_rsync_from` / scp helpers instead — those take a -# different path that doesn't compete for stdin. +# For callers where the remote command itself reads stdin (e.g. `bash -s`), +# this helper is not enough — NOPASSWD policy also skips the stdin read, +# leaking the password. Use SUDO_ASKPASS instead (see scripts/01 for the +# pattern). lab_ssh_sudo() { printf '%s\n' "$LAB_SUDO_PASS" | SSHPASS="$LAB_SSH_PASS" "${LAB_SSH_CMD[@]}" "$LAB_SSH_TARGET" "sudo -k -S -p '' $*" } -# Copy a local file to the target. The remote path is absolute. lab_scp_to() { - local local_path="$1" - local remote_path="$2" - SSHPASS="$LAB_SSH_PASS" "${LAB_SCP_CMD[@]}" "$local_path" "$LAB_SSH_TARGET:$remote_path" + SSHPASS="$LAB_SSH_PASS" "${LAB_SCP_CMD[@]}" "$1" "$LAB_SSH_TARGET:$2" } -# Rsync a remote path (file or dir) into a local destination. Extra rsync -# args can be appended — notably `--rsync-path='sudo rsync'` for paths only -# root can read (vmlinux), and `--delete` for tree mirroring. +# Extra rsync flags can be appended — notably `--rsync-path='sudo rsync'` +# for paths only root can read (vmlinux), and `--delete` for tree mirroring. lab_rsync_from() { local remote_src="$1" local local_dst="$2" @@ -207,14 +170,9 @@ lab_rsync_from() { # --- Preflight ------------------------------------------------------------ -# Fast, non-mutating connection check. Verifies SSH reachable, and (by -# default) that sudo works. Pass `--no-sudo` to skip the sudo test — useful -# in `scripts/01-provision-target.sh`, which a fresh user might be running -# *to* configure sudo for the first time. -# -# Called at the top of 01/02/04 so failures surface before any real work. -# `scripts/00-check-target.sh` runs a longer-form report on top of this. -# shellcheck disable=SC2120 # arg is optional (--no-sudo); callers without it are intentional. +# `--no-sudo` skips the sudo test, for scripts/01 which may be running TO +# configure sudo for the first time. +# shellcheck disable=SC2120 # --no-sudo arg is optional. lab_check_connection() { local check_sudo=1 [[ "${1:-}" == "--no-sudo" ]] && check_sudo=0 @@ -222,13 +180,13 @@ lab_check_connection() { if ! lab_ssh 'true' 2>/dev/null; then die "cannot reach $LAB_SSH_TARGET over ssh (port $LAB_SSH_PORT). - check TARGET_SSH_HOST/PORT/USER for '$LAB_TARGET' in lab.local.env -- if the VM is up, try by hand: ssh -p $LAB_SSH_PORT $LAB_SSH_TARGET -- if you're using passwords, confirm TARGET_SSH_PASS[$LAB_TARGET] is set" +- try by hand: ssh -p $LAB_SSH_PORT $LAB_SSH_TARGET +- if using passwords, confirm TARGET_SSH_PASS[$LAB_TARGET] is set" fi if (( check_sudo )) && ! lab_ssh_sudo 'true' 2>/dev/null; then die "ssh reaches $LAB_SSH_TARGET but sudo doesn't work there. - check TARGET_SUDO_PASS[$LAB_TARGET] in lab.local.env (defaults to TARGET_SSH_PASS) -- on the target, confirm: sudo -n -v (or run sudo by hand once to cache creds) -- consider configuring NOPASSWD for the lab user on long-lived dev targets" +- on the target, confirm: sudo -n -v +- consider NOPASSWD for the lab user on long-lived dev targets" fi }