From 83f36fd3f9d67524c9abe837cf2d5183b86abaf6 Mon Sep 17 00:00:00 2001 From: Antoine Bernardeau Date: Tue, 15 Sep 2026 15:16:20 +0200 Subject: [PATCH] vk-driver: show live host memory usage in VM listings Show process-tree memory beside the configured size in vk list and vk dev list, with byte counts in JSON and a separate detail field. --- CHANGELOG.md | 4 + README.md | 36 ++++-- docs/dev.md | 10 +- vk-driver/src/dev/cli.rs | 10 +- vk-driver/src/dev/list.rs | 109 +++++++++++++--- vk-driver/src/main.rs | 21 +-- vk-driver/src/usage.rs | 126 ++++++++++++++++++ vk-driver/src/vms.rs | 265 ++++++++++++++++++++++++++++++++++---- 8 files changed, 512 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b3c2fbb..95f2a202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ All notable changes to virtkit will be documented in this file. `vk run --numa off|auto|interleave|N` decides for one VM. Single-node hosts, which are most of them, are unaffected. +- **`vk list` and `vk dev list` show live host memory usage beside the VM's configured + memory.** The detailed VM record adds `MEM USED`, and JSON output includes + `mem_used_bytes` for scripts. + ## [0.71.0] - 2026-09-14 ### Changed diff --git a/README.md b/README.md index 26fbdfae..5312a8d8 100644 --- a/README.md +++ b/README.md @@ -435,12 +435,20 @@ vk list ``` ``` -PID UPTIME NAME SERVICES PROJECT PUBLISHED -41230 2h14m app/Dockerfile:dev - ~/app 127.0.0.1:8443->localhost:443 -41877 35m shop db, redis, web, +4 ~/shop 127.0.0.1:5432->127.0.0.1:5432@db +PID UPTIME MEM NAME SERVICES PROJECT PUBLISHED +41230 2h14m 1.2G/8G app/Dockerfile:dev - ~/app 127.0.0.1:8443->localhost:443 +41877 35m 5.9G/16G shop db, redis, web, +4 ~/shop 127.0.0.1:5432->127.0.0.1:5432@db ``` -NAME is the built Dockerfile with its target stage, the compose primary, or the image ref. +MEM is what the VM is costing the host now over the size it booted with: the resident +memory of its whole process tree — the guest, its service VMs, the switch, the virtiofsds +and the forwards — counted proportionally so a page several of them map is charged once, +over the `--mem` token as the run recorded it. When proportional usage is unavailable, +resident usage is used instead and may count shared pages more than once. The total includes +service VMs and helpers, so it is not the primary guest's memory utilization. +Either half is `-` on its own when unknown, +and the cell is a bare `-` when neither is. NAME is the built Dockerfile with its target +stage, the compose primary, or the image ref. SERVICES lists the compose services running beside the primary, or every declared one when the VM cannot be asked; `-` for none. Past three names, the rest are counted (`+4`). PROJECT is the run's `--workspace`, then its `--workdir`, then its launch directory, with @@ -482,6 +490,7 @@ GUEST IP 10.0.0.2 VMM libkrun (pid 41902) CPUS 4 MEM 8G +MEM USED 5.9 GiB NESTED no ATOP LOG - SERVICES db running 10.0.0.3 vsock-auto:///home/me/shop/.vk/svc-db/vsock.sock:4444 @@ -496,15 +505,16 @@ PUBLISHED pg 127.0.0.1:5432->127.0.0.1:5432@db pid 42011 ``` `--json` gives an array of objects, one per VM, with `pid`, `label`, `project_dir`, -`exec_addr`, `state_dir`, `vmm`, `vmm_pid`, `cpus`, `mem`, `nested`, `guest_ip` (the eth0 -address on a `--net` LAN), `ssh_addr`, `atop_log`, `created_secs`, `uptime_secs`, -`services` (every declared compose service with its `name`, `exec_addr`, `state` and LAN -`ip`), and `published` (each publisher's `name`, `listen`, `to` and `pid`, plus `via` when -a compose sibling dials — the one `vk publish ensure --via` named — and `"unconfirmed": -true` when its liveness could not be checked). `--field` picks fields without jq, one -`--field` per field: one line per VM and tab-separated in flag order, or with `--json` -objects holding only those fields; a dotted path reaches into nested values, and a key a -record omits reads `null`: +`exec_addr`, `state_dir`, `vmm`, `vmm_pid`, `cpus`, `mem`, `mem_used_bytes` (what the VM's +process tree holds on the host now, in bytes — `null` when it could not be read), +`nested`, `guest_ip` (the eth0 address on a `--net` LAN), `ssh_addr`, `atop_log`, +`created_secs`, `uptime_secs`, `services` (every declared compose service with its +`name`, `exec_addr`, `state` and LAN `ip`), and `published` (each publisher's `name`, +`listen`, `to` and `pid`, plus `via` when a compose sibling dials — the one +`vk publish ensure --via` named — and `"unconfirmed": true` when its liveness could not +be checked). `--field` picks fields without jq, one `--field` per field: one line per VM +and tab-separated in flag order, or with `--json` objects holding only those fields; a +dotted path reaches into nested values, and a key a record omits reads `null`: ```sh vk list . --field pid # the pid to hand to vk stop diff --git a/docs/dev.md b/docs/dev.md index c9d036d6..c91f5dbe 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -474,9 +474,13 @@ vk dev gc ENVIRONMENT_NAME --yes `list` and `gc` work from anywhere without a project config. Copy names from `list`; they identify state directories, not just the config's `dev` or `hook` -selector. `gc` refuses running environments. `--all-stale` selects stopped state -whose workspace is gone or which never recorded a boot, including leftovers -from throwaway tasks. It does not mean every stopped development environment. +selector. The `MEM` column shows what a running environment's VM holds on the +host now over the size it booted with (`1.2G/8G`), the same figure `vk list` +reports; a stopped environment reads `-`, and `mem_used_bytes` in `--json` is +null for it. `gc` refuses running environments. `--all-stale` selects stopped +state whose workspace is gone or which never recorded a boot, including +leftovers from throwaway tasks. It does not mean every stopped development +environment. Without `--yes`, GC asks on a terminal; without a terminal it only lists what would be removed. GC deletes state directories, including managed data inside diff --git a/vk-driver/src/dev/cli.rs b/vk-driver/src/dev/cli.rs index 64ad8776..e26d0935 100644 --- a/vk-driver/src/dev/cli.rs +++ b/vk-driver/src/dev/cli.rs @@ -356,10 +356,12 @@ enum DevAction { /// /// Host-wide, and needs no config in the current directory: one row per state directory /// under `$XDG_STATE_HOME/virtkit/dev` — which workspace and environment it belongs to, - /// whether it is running, which vk created it, how long ago it last booted and what it - /// holds on disk (`--no-sizes` skips the measure). Flagged when its workspace is gone, or - /// when it recorded no boot at all — the shape a task run in a throwaway environment leaves. - /// Reads only. + /// whether it is running, which vk created it, how long ago it last booted, what its VM + /// holds in memory and what it holds on disk (`--no-sizes` skips the disk measure). MEM + /// is the running VM's whole process tree over the size it booted with (`1.2G/8G`), as + /// `vk list` reports it, and `-` for an environment that is not running. Flagged when its + /// workspace is gone, or when it recorded no boot at all — the shape a task run in a + /// throwaway environment leaves. Reads only. List { /// print the same facts as JSON #[arg(long)] diff --git a/vk-driver/src/dev/list.rs b/vk-driver/src/dev/list.rs index 6d17a5b3..7662f349 100644 --- a/vk-driver/src/dev/list.rs +++ b/vk-driver/src/dev/list.rs @@ -17,7 +17,8 @@ //! `vk dev list --json` is an array of [`Row`], and its field names are the interface: they //! are added to, never renamed or repurposed. `size_bytes` is the exception that is absent //! rather than null — measuring a state directory walks all of it, and `--no-sizes` skips -//! the default measurement. +//! the default measurement. `mem_used_bytes` is always requested; null means the environment +//! is stopped or its memory could not be read. use std::os::unix::fs::FileTypeExt; use std::path::{Path, PathBuf}; @@ -72,6 +73,20 @@ impl Flag { } } +/// A running VM as [`scan`] needs it: the state directory that ties it to a row, plus the +/// memory facts that row reports. Carried as its own type rather than a [`crate::vms::VmEntry`] +/// so `scan` stays a pure function over facts a test can state outright — measuring a live +/// process tree is the caller's job, not the scan's. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Running { + /// the VM's `--state-dir`, as the registry recorded it + pub state_dir: PathBuf, + /// what its whole process tree holds on the host now (`crate::usage::tree_resident`); + /// `None` when the tree could not be read + pub mem_used: Option, + /// the memory size it booted with, the `--mem` token verbatim + pub mem: Option, +} /// One state directory, as `vk dev list` reports it. #[derive(Debug, Clone, PartialEq, Serialize)] pub struct Row { @@ -87,6 +102,14 @@ pub struct Row { pub booted_secs: Option, /// how long ago that boot was, at the time of the scan pub age_secs: Option, + /// what the environment's VM holds on the host right now, in bytes: its whole process + /// tree, counted proportionally, the same figure `vk list` reports. `null` for an + /// environment that is not running — a stopped one holds nothing, which is a fact, not a + /// missing measurement — and for a running one whose tree could not be read. + pub mem_used_bytes: Option, + /// the memory size its VM booted with, the `--mem` token as recorded; `null` when it is + /// not running, or the run recorded none + pub mem: Option, /// what the directory holds; measured by default, omitted with `--no-sizes` #[serde(skip_serializing_if = "Option::is_none")] pub size_bytes: Option, @@ -115,7 +138,7 @@ pub struct Entry { /// List state directories under `base`, using `running` to identify active VMs. Read-only. /// Keep rows with absent or unreadable `dev.json` so `gc` can collect them. /// `sizes` measures each directory, walking its root images and server trees. -pub fn scan(base: &Path, running: &[PathBuf], sizes: bool) -> Vec { +pub fn scan(base: &Path, running: &[Running], sizes: bool) -> Vec { let Ok(entries) = std::fs::read_dir(base) else { return Vec::new(); }; @@ -128,7 +151,7 @@ pub fn scan(base: &Path, running: &[PathBuf], sizes: bool) -> Vec { rows } -fn row(dir: &Path, running: &[PathBuf], sizes: bool) -> Row { +fn row(dir: &Path, running: &[Running], sizes: bool) -> Row { let identity = std::fs::read(dir.join("dev.json")) .ok() .and_then(|b| serde_json::from_slice::(&b).ok()); @@ -139,7 +162,9 @@ fn row(dir: &Path, running: &[PathBuf], sizes: bool) -> Row { // The registry records canonical state dirs, so compare against both forms: the base // itself reaches us through `$HOME`, which is a symlink on some hosts. let canonical = std::fs::canonicalize(dir).unwrap_or_else(|_| dir.to_path_buf()); - let is_running = running.iter().any(|r| r == dir || r == &canonical); + let live = running + .iter() + .find(|r| r.state_dir == dir || r.state_dir == canonical); let mut flags = Vec::new(); // Require the workspace's parent to exist: an unmounted share or unplugged disk must // not make its environments stale and let `gc --all-stale --yes` destroy their storage. @@ -160,7 +185,7 @@ fn row(dir: &Path, running: &[PathBuf], sizes: bool) -> Row { dir: dir.to_path_buf(), workspace, environment: manifest("environment"), - status: match (is_running, identity.is_some()) { + status: match (live.is_some(), identity.is_some()) { (true, _) => Status::Running, (false, true) => Status::Stopped, (false, false) => Status::NeverBooted, @@ -171,6 +196,8 @@ fn row(dir: &Path, running: &[PathBuf], sizes: bool) -> Row { .filter(|by| !by.is_empty()), booted_secs, age_secs: booted_secs.map(|s| crate::vms::unix_now().saturating_sub(s)), + mem_used_bytes: live.and_then(|r| r.mem_used), + mem: live.and_then(|r| r.mem.clone()), size_bytes: sizes.then(|| crate::dev::storage::dir_size(dir)), flags, } @@ -253,6 +280,7 @@ pub fn render(rows: &[Row]) -> String { .map(short_creator) .unwrap_or_default(), r.age_secs.map(crate::vms::fmt_uptime).unwrap_or_default(), + crate::vms::mem_cell(r.mem_used_bytes, r.mem.as_deref()), fmt_size(r.size_bytes), r.flags .iter() @@ -270,6 +298,7 @@ pub fn render(rows: &[Row]) -> String { "STATUS", "CREATED BY", "LAST BOOT", + "MEM", "ON DISK", "FLAGS", ], @@ -409,13 +438,30 @@ pub fn remove(selected: &[Row]) -> Result { Ok(out) } -/// The state dirs VMs are currently up on. +/// Running VMs' state directories. `remove` needs only liveness, so this skips the memory +/// walk in [`running_vms`]. fn running_dirs() -> Vec { crate::vms::running() .into_iter() .map(|e| e.state_dir) .collect() } +/// The running VMs, each with the live memory reading its row reports. One `/proc` walk per +/// VM, which is cheap beside the stat walk of every file that `sizes` does by default — and +/// unlike that one it has no opt-out, since a row with no memory figure would not say +/// whether the VM holds nothing or was never asked. +fn running_vms() -> Vec { + crate::vms::running() + .into_iter() + .map(|e| Running { + mem_used: i32::try_from(e.pid) + .ok() + .and_then(crate::usage::tree_resident), + mem: e.mem, + state_dir: e.state_dir, + }) + .collect() +} /// Every environment this host keeps state for, as `vk dev list` and `vk dev gc` see it: /// the state base scanned against what is running. Measuring what each holds on disk is a stat @@ -423,7 +469,7 @@ fn running_dirs() -> Vec { pub fn state(sizes: bool) -> Result> { Ok(scan( &crate::dev::plan::dev_state_base()?, - &running_dirs(), + &running_vms(), sizes, )) } @@ -612,8 +658,17 @@ mod tests { std::fs::create_dir_all(&base).unwrap(); let dir = booted(&base, "repo-aaaa", &workspace, "vk 0.62.0 (abcdef)"); - let rows = scan(&base, &[dir], true); + let live = Running { + state_dir: dir, + mem_used: Some(1_288_490_189), + mem: Some("8G".into()), + }; + let rows = scan(&base, std::slice::from_ref(&live), true); assert_eq!(rows[0].status, Status::Running); + // A running row carries the live figure and the size it booted with, so the MEM + // column has both halves; a stopped one has neither (see the render test). + assert_eq!(rows[0].mem_used_bytes, Some(1_288_490_189)); + assert_eq!(rows[0].mem.as_deref(), Some("8G")); assert!(!rows[0].stale()); let e = select_gc(rows, &["repo-aaaa".into()], false).unwrap_err(); assert!(format!("{e:#}").contains("repo-aaaa is running"), "{e:#}"); @@ -659,7 +714,10 @@ mod tests { fn render_aligns_the_columns_and_names_what_is_missing() { let tmp = scratch("render"); let base = tmp.0.join("state"); + let workspace = tmp.0.join("repo"); std::fs::create_dir_all(&base).unwrap(); + std::fs::create_dir_all(&workspace).unwrap(); + let up = booted(&base, "a-live-dddd", &workspace, "vk 0.62.0 (abcdef)"); booted( &base, "gone-bbbb", @@ -667,29 +725,48 @@ mod tests { "vk 0.62.0 (abcdef)", ); ephemeral(&base, "repo-hook-cccc"); - let mut rows = scan(&base, &[], true); - // Fixed, so the column reads the same on every run. - rows[0].age_secs = Some(7200); + let live = Running { + state_dir: up, + mem_used: Some(1_288_490_189), + mem: Some("8G".into()), + }; + let mut rows = scan(&base, std::slice::from_ref(&live), true); + // Fixed, so the columns read the same on every run. + for row in &mut rows { + row.age_secs = row.age_secs.map(|_| 7200); + } let out = render(&rows); let lines: Vec<&str> = out.lines().collect(); assert!(lines[0].starts_with("NAME"), "{out}"); assert!( - lines[0].contains("CREATED BY LAST BOOT ON DISK FLAGS"), + lines[0].contains("CREATED BY LAST BOOT MEM ON DISK FLAGS"), + "{out}" + ); + // The running row is the only one holding memory: what its tree holds now over the + // size it booted with, the same cell `vk list` prints. + assert!( + lines[1].contains("running") && lines[1].contains("1.2G/8G"), "{out}" ); assert!( - lines[1].contains("stopped") && lines[1].contains("vk 0.62.0"), + lines[2].contains("stopped") && lines[2].contains("vk 0.62.0"), "{out}" ); + // A stopped environment holds nothing, and the column says so rather than guessing + // from the `--mem` its last boot used. assert!( - lines[1].contains("2h0m") && lines[1].ends_with("workspace missing"), + lines[2].contains("2h0m") && lines[2].ends_with("workspace missing"), "{out}" ); + assert!( + lines[2].contains(" - ") || lines[2].contains(" - "), + "a stopped row dashes MEM: {out}" + ); // Nothing was recorded, so the workspace and environment columns say so. - assert!(lines[2].starts_with("repo-hook-cccc ?"), "{out}"); + assert!(lines[3].starts_with("repo-hook-cccc ?"), "{out}"); assert!( - lines[2].contains("never booted") && lines[2].ends_with("ephemeral"), + lines[3].contains("never booted") && lines[3].ends_with("ephemeral"), "{out}" ); assert_eq!(render(&[]), "no dev environment state on this host\n"); diff --git a/vk-driver/src/main.rs b/vk-driver/src/main.rs index 7619ba33..f522c167 100644 --- a/vk-driver/src/main.rs +++ b/vk-driver/src/main.rs @@ -1909,15 +1909,18 @@ enum Cmd { }, /// List the running vk VMs /// - /// VMs started with `--state-dir`, with their pid, uptime, name, compose services, - /// project directory (`--workspace`, then `--workdir`, then launch directory) and - /// published ports (`listen->to`; `@service` when a compose sibling dials). The table - /// folds `$HOME` to `~` and names at most three services; `--wide` shows every service, - /// the project directory in full and the exec-channel address. With PID or DIR, only the - /// VM with that pid, or the VMs whose project is DIR or below it (or whose state dir is - /// DIR); a selector that names exactly one VM prints its full record instead of a table - /// row. The record folds nothing, so `--wide` has nothing to add there. Use `--json` or - /// `--field` for scripts; neither takes `--wide`, since both already report every field. + /// VMs started with `--state-dir`, with their pid, uptime, memory, name, compose + /// services, project directory (`--workspace`, then `--workdir`, then launch directory) + /// and published ports (`listen->to`; `@service` when a compose sibling dials). MEM is + /// what the VM's whole process tree holds on the host right now over the size it booted + /// with (`1.2G/8G`) — the guest, its service VMs and their helpers, counted + /// proportionally so pages they share are charged once. The table folds `$HOME` to `~` + /// and names at most three services; `--wide` shows every service, the project directory + /// in full and the exec-channel address. With PID or DIR, only the VM with that pid, or + /// the VMs whose project is DIR or below it (or whose state dir is DIR); a selector that + /// names exactly one VM prints its full record instead of a table row. The record folds + /// nothing, so `--wide` has nothing to add there. Use `--json` or `--field` for scripts; + /// neither takes `--wide`, since both already report every field. #[command(display_order = 6)] List { /// which VMs: a PID, or those whose project is DIR or below it (default: all) diff --git a/vk-driver/src/usage.rs b/vk-driver/src/usage.rs index 7f5ad888..44eeea10 100644 --- a/vk-driver/src/usage.rs +++ b/vk-driver/src/usage.rs @@ -643,6 +643,54 @@ pub(crate) fn fmt_bytes(bytes: u64) -> String { } } +/// The memory a process tree holds *now*, in bytes — `root` and every process descending +/// from it, which for a VM is the guest, its compose service VMs, the switch, the +/// virtiofsds and the forwards. The live figure [`Usage::peak_rss`] deliberately is not: +/// `vk list` reports what a VM is costing the host at this moment, not the demand it once +/// passed through. +/// +/// Proportional (`Pss`), because the tree shares pages with itself: the driver, the libkrun +/// keeper and the boot child it forked all map this binary's text, and summing their `VmRSS` +/// would charge it three times. Reading Pss requires a page-table walk per process. +/// +/// Processes that exit during the walk are skipped. Returns `None` when `root` is gone +/// or no process has a readable memory measurement. +pub(crate) fn tree_resident(root: i32) -> Option { + let pids = descendants(root, &HashSet::new()); + if pids.is_empty() { + return None; + } + // Saturate to avoid wrapping or a debug-build panic if /proc reports an impossible total. + pids.into_iter() + .filter_map(resident) + .reduce(u64::saturating_add) +} + +/// One process's proportional share of resident memory, in bytes. `smaps_rollup` where the +/// kernel publishes it, else `VmRSS` — which over-counts that process's share of anything +/// shared rather than dropping it from the tree, the safer way to be wrong about a figure +/// the caller is summing. `None` for a process that is gone. +fn resident(pid: i32) -> Option { + std::fs::read_to_string(format!("/proc/{pid}/smaps_rollup")) + .ok() + .and_then(|rollup| parse_pss(&rollup)) + .or_else(|| mem(pid).map(|(rss, _)| rss)) +} + +/// `Pss` from `smaps_rollup`, in bytes. Match the colon to exclude the `Pss_Anon` and +/// `Pss_Dirty` subtotals. +fn parse_pss(rollup: &str) -> Option { + rollup.lines().find_map(|l| { + let kb: u64 = l + .strip_prefix("Pss:")? + .split_whitespace() + .next()? + .parse() + .ok()?; + kb.checked_mul(1024) + }) +} + #[cfg(test)] mod tests { use super::*; @@ -1260,4 +1308,82 @@ mod tests { "virtkit: job resource usage: cpu 2m14s, peak memory 1.6 GiB" ); } + + #[test] + fn reads_the_proportional_share_and_not_the_subtotals_beside_it() { + let rollup = "55d0-7ffc ---p 00000000 00:00 0 [rollup]\n\ + Rss: 8148 kB\n\ + Pss: 7736 kB\n\ + Pss_Dirty: 128 kB\n\ + Pss_Anon: 128 kB\n"; + // Pss, not the Pss_Dirty/Pss_Anon subsets printed under it. + assert_eq!(parse_pss(rollup), Some(7736 * 1024)); + assert_eq!(parse_pss("Rss: 8148 kB\n"), None); + assert_eq!(parse_pss("Pss: notanumber\n"), None); + assert_eq!(parse_pss("Pss: 18446744073709551615 kB\n"), None); + } + + /// A shell that holds nothing itself and waits on a child that holds `mib`. The root is + /// dedicated, so unlike a reading rooted at the test process this one cannot be moved by + /// the sibling tests' children coming and going. + struct NestedHog(Reap); + + impl NestedHog { + fn pid(&self) -> i32 { + self.0.pid() + } + } + + impl Drop for NestedHog { + fn drop(&mut self) { + // The child owns a dedicated process group; kill its descendants too. + // Reap then waits for the direct child. + unsafe { libc::kill(-self.pid(), libc::SIGKILL) }; + } + } + + fn nested_hog(mib: usize) -> NestedHog { + use std::os::unix::process::CommandExt; + + let inner = format!( + "s=$(head -c {} /dev/zero | tr \"\\0\" x); while true; do sleep 1; done", + mib * 1024 * 1024 + ); + NestedHog(Reap( + std::process::Command::new("sh") + .args(["-c", &format!("sh -c '{inner}' & wait")]) + .process_group(0) + .spawn() + .expect("spawning a shell whose child holds memory"), + )) + } + + #[test] + fn tree_resident_descends_to_a_child_and_reports_nothing_for_a_dead_root() { + // A pid that cannot exist has no tree, which is not the same as a tree of zero: + // `vk list` dashes the cell rather than claiming the VM holds nothing. + assert_eq!(tree_resident(-1), None); + + let root = nested_hog(64); + let grown = Instant::now(); + while tree_resident(root.pid()).is_none_or(|t| t < 32 * 1024 * 1024) { + assert!( + grown.elapsed() < Duration::from_secs(30), + "the child never grew" + ); + std::thread::sleep(Duration::from_millis(50)); + } + // The memory is all in the child, so a reading that stopped at the root would see + // almost none of it — which is exactly how `vmm_pid` reads for a VM booted with + // reboot-in-place, where the libkrun keeper holds nothing and its forked child holds + // the guest. + let (total, own) = ( + tree_resident(root.pid()).unwrap(), + resident(root.pid()).unwrap(), + ); + assert!( + own * 4 < total, + "the root holds {own} of the tree's {total}; the walk did not descend" + ); + } } diff --git a/vk-driver/src/vms.rs b/vk-driver/src/vms.rs index 01d356ce..8447813b 100644 --- a/vk-driver/src/vms.rs +++ b/vk-driver/src/vms.rs @@ -513,6 +513,10 @@ struct VmView<'a> { guest_ip: Option, cpus: Option, mem: Option<&'a str>, + /// Live host memory for the guest, service VMs and helpers (`usage::tree_resident`), + /// beside the boot-time size in `mem`. `null` when the managing `vk run` is gone or + /// the tree's memory could not be read from `/proc`. + mem_used_bytes: Option, nested: Option, atop_log: Option<&'a Path>, created_secs: u64, @@ -573,6 +577,7 @@ fn view<'a>( published: &'a [Published], freshness: Freshness, stale: bool, + mem_used: Option, ) -> VmView<'a> { VmView { state_dir: &entry.state_dir, @@ -586,6 +591,7 @@ fn view<'a>( guest_ip: entry.guest_ip, cpus: entry.cpus, mem: entry.mem.as_deref(), + mem_used_bytes: mem_used, nested: entry.nested, atop_log: entry.atop_log.as_deref(), created_secs: entry.created_secs, @@ -804,6 +810,42 @@ fn services_cell(entry: &VmEntry, units: Option<&[UnitStatus]>, wide: bool) -> S ) } +/// The `MEM` cell pairs live process-tree usage with the boot-time size: `1.2G/8G`. +/// Usage includes service VMs and helpers, so this is not guest used/total memory. +/// Each unknown half reads `-` (an unreadable tree or an older `vk run` that did not record +/// `--mem`); when both are unknown, the cell is a single `-`. +pub(crate) fn mem_cell(used: Option, configured: Option<&str>) -> String { + if used.is_none() && configured.is_none() { + return "-".to_string(); + } + format!( + "{}/{}", + used.map_or_else(|| "-".to_string(), compact_bytes), + configured.unwrap_or("-") + ) +} + +/// A memory figure in the narrowest form that still reads — `1.2G`, `780M`, `64K`. The +/// spacious `usage::fmt_bytes` (`1.6 GiB`) is what the detail view uses; the table holds two +/// figures and a slash in one column, so this drops the space and the `iB` and keeps the +/// unit letter of the `--mem` token it sits beside. +fn compact_bytes(bytes: u64) -> String { + let (gib, mib, kib) = ( + bytes as f64 / (1024.0 * 1024.0 * 1024.0), + bytes as f64 / (1024.0 * 1024.0), + bytes as f64 / 1024.0, + ); + // Each arm branches on the rounded figure it is about to print rather than a coarser + // one, for the reason `usage::fmt_bytes` spells out: branching on the untruncated MiB + // while printing the rounded one renders 1023.7 MiB as "1024M". + match bytes { + _ if mib.round() >= 1024.0 => format!("{gib:.1}G"), + _ if kib.round() >= 1024.0 => format!("{mib:.0}M"), + 1024.. => format!("{kib:.0}K"), + _ => format!("{bytes}B"), + } +} + /// The PROJECT column: the directory as recorded, or with `$HOME` folded to `~` unless /// `wide`. `-` when the run recorded none. fn project_cell(project_dir: Option<&Path>, home: Option<&Path>, wide: bool) -> String { @@ -961,6 +1003,15 @@ pub fn list_report( .collect(); let units_by_vm: Vec>> = vms.iter().map(service_units).collect(); let published_by_vm: Vec> = vms.iter().map(published).collect(); + // Measure once per VM for all output forms: table, detail record and JSON. + let mem_used: Vec> = vms + .iter() + .map(|e| { + i32::try_from(e.pid) + .ok() + .and_then(crate::usage::tree_resident) + }) + .collect(); if json || !fields.is_empty() { let views: Vec = vms @@ -968,8 +1019,9 @@ pub fn list_report( .zip(&units_by_vm) .zip(&published_by_vm) .zip(&fresh) - .map(|(((entry, units), published), freshness)| { - view(entry, units.as_deref(), published, *freshness, stale) + .zip(&mem_used) + .map(|((((entry, units), published), freshness), used)| { + view(entry, units.as_deref(), published, *freshness, stale, *used) }) .collect(); if !fields.is_empty() { @@ -989,14 +1041,15 @@ pub fn list_report( }); } if full_record(target.as_ref(), vms.len()) - && let ([e], [units], [published], [f]) = ( + && let ([e], [units], [published], [f], [used]) = ( vms.as_slice(), units_by_vm.as_slice(), published_by_vm.as_slice(), fresh.as_slice(), + mem_used.as_slice(), ) { - return Ok(detail(e, units.as_deref(), published, *f, stale)); + return Ok(detail(e, units.as_deref(), published, *f, stale, *used)); } // `tilde` matches against a canonical `project_dir`, so canonicalize `$HOME` as well; // only the narrow table folds it. @@ -1008,6 +1061,7 @@ pub fn list_report( &units_by_vm, &published_by_vm, &fresh, + &mem_used, home.as_deref(), stale, wide, @@ -1020,16 +1074,18 @@ pub fn list_report( /// because padding uses column positions. /// /// Per-VM slices come from `list_report`, in `vms` order. +#[allow(clippy::too_many_arguments)] fn table( vms: &[VmEntry], units_by_vm: &[Option>], published_by_vm: &[Vec], fresh: &[Freshness], + mem_used: &[Option], home: Option<&Path>, stale: bool, wide: bool, ) -> String { - let mut headers: Vec<&str> = vec!["PID", "UPTIME", "NAME", "SERVICES", "PROJECT"]; + let mut headers: Vec<&str> = vec!["PID", "UPTIME", "MEM", "NAME", "SERVICES", "PROJECT"]; if wide { headers.push("EXEC ADDRESS"); } @@ -1042,10 +1098,12 @@ fn table( .zip(units_by_vm) .zip(published_by_vm) .zip(fresh) - .map(|(((e, units), published), f)| { + .zip(mem_used) + .map(|((((e, units), published), f), used)| { let mut row = vec![ e.pid.to_string(), uptime(e.created_secs), + mem_cell(*used, e.mem.as_deref()), e.label.clone(), services_cell(e, units.as_deref(), wide), project_cell(e.project_dir.as_deref(), home, wide), @@ -1096,12 +1154,17 @@ fn table( /// folded or left out here, so `--wide` has nothing to add. A field the run did not record /// (an older `vk`, no `--ssh`, no `--net`) reads `-`; a service's state reads `-` when the /// VM could not be asked, or did not report it. +/// +/// `MEM USED` comes from `/proc` at report time; `MEM` is the recorded boot-time token. +/// The table combines them, but the detail record keeps them separate. Unreadable usage +/// reads `-`. fn detail( e: &VmEntry, units: Option<&[UnitStatus]>, published: &[Published], freshness: Freshness, stale: bool, + mem_used: Option, ) -> String { let dash = || "-".to_string(); let opt = |v: Option| v.unwrap_or_else(dash); @@ -1128,6 +1191,7 @@ fn detail( ("VMM", vmm), ("CPUS", opt(e.cpus.map(|n| n.to_string()))), ("MEM", opt(e.mem.clone())), + ("MEM USED", opt(mem_used.map(crate::usage::fmt_bytes))), ("NESTED", yes_no(e.nested)), ("ATOP LOG", path(e.atop_log.as_deref())), ]; @@ -1844,7 +1908,7 @@ mod tests { } fn services_json(e: &VmEntry, units: Option<&[UnitStatus]>) -> serde_json::Value { - serde_json::to_value(view(e, units, &[], Freshness::Unknown, false)).unwrap()["services"] + serde_json::to_value(view(e, units, &[], Freshness::Unknown, false, None)).unwrap()["services"] .take() } @@ -1953,13 +2017,24 @@ mod tests { let _ = std::fs::remove_dir_all(&real); } - /// One VM per row, with no publishers and unknown freshness — enough to check which - /// columns the table emits. + /// One VM per row, with no publishers, unknown freshness and no memory reading — enough + /// to check which columns the table emits. `table_of_mem` supplies the figures. fn table_of(vms: &[VmEntry], home: Option<&Path>, stale: bool, wide: bool) -> String { + let none: Vec> = vms.iter().map(|_| None).collect(); + table_of_mem(vms, &none, home, stale, wide) + } + + fn table_of_mem( + vms: &[VmEntry], + mem_used: &[Option], + home: Option<&Path>, + stale: bool, + wide: bool, + ) -> String { let units: Vec>> = vms.iter().map(|_| None).collect(); let published: Vec> = vms.iter().map(|_| Vec::new()).collect(); let fresh: Vec = vms.iter().map(|_| Freshness::Unknown).collect(); - table(vms, &units, &published, &fresh, home, stale, wide) + table(vms, &units, &published, &fresh, mem_used, home, stale, wide) } /// The cells of one table line. No cell holds two consecutive spaces, so the padding @@ -1991,7 +2066,7 @@ mod tests { ]; for (stale, wide) in [(false, false), (false, true), (true, false), (true, true)] { let text = table_of(&vms, Some(Path::new("/home/me")), stale, wide); - let mut expected = vec!["PID", "UPTIME", "NAME", "SERVICES", "PROJECT"]; + let mut expected = vec!["PID", "UPTIME", "MEM", "NAME", "SERVICES", "PROJECT"]; if wide { expected.push("EXEC ADDRESS"); } @@ -2068,6 +2143,7 @@ GUEST IP 10.0.0.2 VMM libkrun (pid 4242) CPUS 4 MEM 8G +MEM USED 1.6 GiB NESTED yes ATOP LOG /state/app/atop.log STALE yes @@ -2078,7 +2154,14 @@ PUBLISHED pg 127.0.0.1:5432->127.0.0.1:5432@db pid 4242 up = uptime(e.created_secs) ); assert_eq!( - detail(&e, Some(&units), &published, Freshness::Stale, true), + detail( + &e, + Some(&units), + &published, + Freshness::Stale, + true, + Some(1_717_986_918) + ), expected ); } @@ -2101,6 +2184,7 @@ GUEST IP - VMM - CPUS - MEM - +MEM USED - NESTED - ATOP LOG - SERVICES - @@ -2110,7 +2194,7 @@ PUBLISHED - ); // Without --stale the STALE row is absent entirely, not reported as unknown. assert_eq!( - detail(&plain, None, &[], Freshness::Unknown, false), + detail(&plain, None, &[], Freshness::Unknown, false, None), expected ); } @@ -2121,7 +2205,7 @@ PUBLISHED - e.nested = Some(false); // A VMM the run named but whose pid it did not record prints bare. e.vmm = Some("cloud-hypervisor".into()); - let text = detail(&e, None, &[], Freshness::Fresh, true); + let text = detail(&e, None, &[], Freshness::Fresh, true, None); assert!( text.contains("\nVMM cloud-hypervisor\n"), "{text}" @@ -2133,7 +2217,14 @@ PUBLISHED - #[test] fn detail_dashes_a_service_the_vm_did_not_report_but_keeps_its_exec_address() { // The VM answered and knows of neither service, so only the recorded facts remain. - let text = detail(&compose_entry(), Some(&[]), &[], Freshness::Unknown, true); + let text = detail( + &compose_entry(), + Some(&[]), + &[], + Freshness::Unknown, + true, + None, + ); assert!( text.contains( "SERVICES db - - vsock-auto:///state/app/svc-db/vsock.sock:4444\n" @@ -2168,6 +2259,7 @@ PUBLISHED - &published, Freshness::Unknown, false, + None, ); let lines: Vec<&str> = text.lines().collect(); let published_at = lines @@ -2247,8 +2339,9 @@ PUBLISHED - plain.label = "plain".into(); plain.pid = 7; vec![ - serde_json::to_value(view(&e, Some(&units), &[], Freshness::Unknown, false)).unwrap(), - serde_json::to_value(view(&plain, None, &[], Freshness::Unknown, false)).unwrap(), + serde_json::to_value(view(&e, Some(&units), &[], Freshness::Unknown, false, None)) + .unwrap(), + serde_json::to_value(view(&plain, None, &[], Freshness::Unknown, false, None)).unwrap(), ] } @@ -2314,7 +2407,10 @@ PUBLISHED - assert!(err.contains("(pass --stale)"), "{err}"); let plain = entry(PathBuf::from("/state/plain"), None); let with_stale = - [serde_json::to_value(view(&plain, None, &[], Freshness::Unknown, true)).unwrap()]; + [ + serde_json::to_value(view(&plain, None, &[], Freshness::Unknown, true, None)) + .unwrap(), + ]; assert_eq!(fields(&with_stale, &["stale"], false).unwrap(), "null\n"); // Malformed paths and repeats are rejected up front, even with nothing running. @@ -2496,7 +2592,8 @@ PUBLISHED - ), ]; let json = - serde_json::to_value(view(&e, None, &published, Freshness::Unknown, false)).unwrap(); + serde_json::to_value(view(&e, None, &published, Freshness::Unknown, false, None)) + .unwrap(); assert_eq!( json["published"], serde_json::json!([ @@ -2518,7 +2615,8 @@ PUBLISHED - ); // Nothing published is an empty array, not an absent key, so `--field` into it reads // null rather than failing. - let none = serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false)).unwrap(); + let none = + serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false, None)).unwrap(); assert_eq!(none["published"], serde_json::json!([])); assert_eq!( fields(&[json.clone(), none], &["published.0.listen"], false).unwrap(), @@ -2560,7 +2658,8 @@ PUBLISHED - e.mem = Some("8G".into()); e.nested = Some(true); e.guest_ip = Some(std::net::Ipv4Addr::new(10, 42, 0, 2)); - let json = serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false)).unwrap(); + let json = + serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false, None)).unwrap(); assert_eq!(json["vmm"], "libkrun"); assert_eq!(json["vmm_pid"], 4242); assert_eq!(json["cpus"], 4); @@ -2571,7 +2670,8 @@ PUBLISHED - // A run without `--net` has no address: an explicit null, not an absent key, so // scripts can tell it from a field this `vk` never emitted. let e = entry(PathBuf::from("/state/app"), None); - let json = serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false)).unwrap(); + let json = + serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false, None)).unwrap(); assert_eq!(json.get("guest_ip"), Some(&serde_json::Value::Null)); } @@ -2590,6 +2690,7 @@ PUBLISHED - guest_ip: None, cpus: None, mem: None, + mem_used_bytes: None, nested: None, atop_log: None, created_secs: 0, @@ -2630,7 +2731,8 @@ PUBLISHED - assert_eq!(e.mem, None); assert_eq!(e.nested, None); assert_eq!(e.guest_ip, None); - let json = serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false)).unwrap(); + let json = + serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false, None)).unwrap(); for key in [ "atop_log", "vmm", "vmm_pid", "cpus", "mem", "nested", "guest_ip", ] { @@ -2642,7 +2744,8 @@ PUBLISHED - fn list_view_reports_the_atop_log_path() { let mut e = entry(PathBuf::from("/state/app"), None); e.atop_log = Some(PathBuf::from("/state/app/atop/atop.log")); - let json = serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false)).unwrap(); + let json = + serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false, None)).unwrap(); assert_eq!(json["atop_log"], "/state/app/atop/atop.log"); } @@ -2665,4 +2768,118 @@ PUBLISHED - assert!(uptime(now.saturating_sub(7200)).contains('h')); assert!(uptime(now.saturating_sub(200_000)).contains('d')); } + + #[test] + fn mem_cell_pairs_what_is_held_with_what_was_asked_for() { + // Both known: the live figure over the boot token, as recorded. + assert_eq!(mem_cell(Some(1_288_490_189), Some("8G")), "1.2G/8G"); + // Preserve the VMM token's spelling and units, even when the usage units differ. + assert_eq!(mem_cell(Some(817_889_280), Some("2048m")), "780M/2048m"); + // Either half alone still prints, so a row says which of the two it is missing. + assert_eq!(mem_cell(None, Some("512M")), "-/512M"); + assert_eq!(mem_cell(Some(1024), None), "1K/-"); + // Neither: one dash, not "-/-", which reads as two known-absent facts. + assert_eq!(mem_cell(None, None), "-"); + } + + #[test] + fn compact_bytes_keeps_a_table_cell_narrow_without_lying_at_a_boundary() { + assert_eq!(compact_bytes(0), "0B"); + assert_eq!(compact_bytes(1023), "1023B"); + assert_eq!(compact_bytes(1024), "1K"); + assert_eq!(compact_bytes(1024 * 1024), "1M"); + assert_eq!(compact_bytes(1024 * 1024 * 1024), "1.0G"); + assert_eq!(compact_bytes(1_717_986_918), "1.6G"); + // Round into the larger unit instead of printing "1024M" or "1024K" by branching + // on the rounded figure, as `usage::fmt_bytes` does. + assert_eq!(compact_bytes(1024 * 1024 * 1024 - 1), "1.0G"); + assert_eq!(compact_bytes(1024 * 1024 - 1), "1M"); + } + + #[test] + fn the_table_shows_memory_after_uptime() { + let vms = [entry(PathBuf::from("/state/solo"), None)]; + let mut sized = vms[0].clone(); + sized.mem = Some("8G".into()); + let text = table_of_mem( + std::slice::from_ref(&sized), + &[Some(1_288_490_189)], + None, + false, + false, + ); + let mut lines = text.lines(); + let header = cells(lines.next().unwrap()); + let row = cells(lines.next().unwrap()); + // The column sits third, between UPTIME and NAME, in the header and the row alike. + assert_eq!(header[1..3], ["UPTIME", "MEM"], "{text}"); + assert_eq!(row[2], "1.2G/8G", "{text}"); + // A VM with neither figure keeps the column and dashes it, so the row stays aligned. + let bare = table_of(&vms, None, false, false); + assert_eq!(cells(bare.lines().nth(1).unwrap())[2], "-", "{bare}"); + } + + #[test] + fn json_view_reports_memory_used_in_bytes_beside_the_boot_token() { + let mut e = entry(PathBuf::from("/state/x"), None); + e.mem = Some("8G".into()); + let json = serde_json::to_value(view( + &e, + None, + &[], + Freshness::Unknown, + false, + Some(1_288_490_189), + )) + .unwrap(); + // JSON gives scripts bytes without parsing "1.2G", beside the unchanged boot token. + assert_eq!(json["mem_used_bytes"], 1_288_490_189u64); + assert_eq!(json["mem"], "8G"); + // Unreadable tree: an explicit null, distinct from a zero-byte tree. + let unknown = + serde_json::to_value(view(&e, None, &[], Freshness::Unknown, false, None)).unwrap(); + assert_eq!(unknown["mem_used_bytes"], serde_json::Value::Null); + } + + /// Renders the exact table the README shows, so the sample there cannot drift out of + /// alignment with the formatter. + #[test] + fn readme_table_sample_is_what_the_formatter_emits() { + let vm = |label: &str, pid: u32, mem: &str, services: &[&str], project: &str| { + let mut e = entry(PathBuf::from("/state/x"), None); + e.services = compose_entry_with(services).services[2..].to_vec(); + e.label = label.to_string(); + e.pid = pid; + e.mem = Some(mem.to_string()); + e.project_dir = Some(PathBuf::from(project)); + e.created_secs = unix_now(); + e + }; + let vms = [ + vm("app/Dockerfile:dev", 41230, "8G", &[], "/home/me/app"), + vm( + "shop", + 41877, + "16G", + &["db", "redis", "web", "worker", "mailer", "queue", "search"], + "/home/me/shop", + ), + ]; + let text = table_of_mem( + &vms, + &[Some(1_288_490_189), Some(6_335_076_761)], + Some(Path::new("/home/me")), + false, + false, + ); + let header = text.lines().next().unwrap(); + assert_eq!( + header, + "PID UPTIME MEM NAME SERVICES PROJECT PUBLISHED", + "{text}" + ); + // The two figures as the README prints them, in the cells it puts them in. + assert_eq!(cells(text.lines().nth(1).unwrap())[2], "1.2G/8G", "{text}"); + assert_eq!(cells(text.lines().nth(2).unwrap())[2], "5.9G/16G", "{text}"); + } }