From 53654a81a8bf7eb71ba096eeffe40c93f6bf4328 Mon Sep 17 00:00:00 2001 From: Stackrats Date: Wed, 14 Oct 2026 21:35:51 +0700 Subject: [PATCH 1/2] Stop a command even when something it started outlives the TERM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stopping a command sends SIGTERM to its process group and waits for the direct child, escalating to SIGKILL only if that child is still there after the grace. A descendant that outlived the TERM — one that ignores it, or one caught mid-fork when it landed — was never killed, and it still held the command's stdout and stderr. The operation ends only when its output forwarder does, which waited for every holder of those pipes to close them, so the stop waited for as long as the survivor ran. macOS runner image 20261013 makes the mid-fork case routine: in project_removal_blocks_new_commands_until_shutdown_finishes the test script's background `sleep 60` survived the TERM, the shell exited on release, and the command's operation stayed open past the test's thirty seconds — failing v0.7.7's release gate on all three runs there. A traced run showed the shell gone, the sleep alive in the command's process group with init as its parent, and no event from the command at all. The cancellation test in mast-docker leaked a sleeper the same way; it only checked that cancel returned promptly. Two fixes, each with a test that fails without it: - Once the direct child has exited, the rest of its group gets what is left of the grace to follow it, then SIGKILL. - When a command exits, its output is forwarded for at most two more seconds. A background process a command leaves running on purpose also inherits the pipes, and the operation used to stay open until that process ended. --- engine/mast-docker/src/command.rs | 66 +++++++++++++++++++++++- engine/mast-engine/src/ops.rs | 11 +++- engine/mast-engine/tests/engine_tests.rs | 40 ++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) diff --git a/engine/mast-docker/src/command.rs b/engine/mast-docker/src/command.rs index 6e165aa..a6a1312 100644 --- a/engine/mast-docker/src/command.rs +++ b/engine/mast-docker/src/command.rs @@ -295,6 +295,13 @@ async fn wait_for_killed_child(child: &mut tokio::process::Child) -> std::io::Re .map(|_| ()) } +/// Whether any process is still in the group — `kill` with signal 0 checks +/// without signalling. +#[cfg(unix)] +fn group_alive(pid: i32) -> bool { + unsafe { libc::kill(-pid, 0) == 0 } +} + #[cfg(unix)] async fn terminate_streaming_child( child: &mut tokio::process::Child, @@ -302,9 +309,22 @@ async fn terminate_streaming_child( grace: Option, ) -> std::io::Result<()> { if let Some(grace) = grace { + let deadline = tokio::time::Instant::now() + grace; kill_group(pid as i32, SIG_TERM); if let Ok(status) = tokio::time::timeout(grace, child.wait()).await { - return status.map(|_| ()); + status?; + // The direct child is gone; whatever else it started in the group + // gets the rest of the grace to follow, then SIGKILL. A descendant + // that outlives the TERM — one that ignores it, or one caught + // mid-fork when it arrived — keeps the output pipes open, and a + // stop would otherwise wait on it for as long as it runs. + while group_alive(pid as i32) && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } + if group_alive(pid as i32) { + kill_group(pid as i32, SIG_KILL); + } + return Ok(()); } } kill_group(pid as i32, SIG_KILL); @@ -703,6 +723,50 @@ mod tests { assert!(started.elapsed() < Duration::from_secs(5), "cancel was not prompt"); } + /// A descendant can outlive the group's SIGTERM — by ignoring it, as + /// here, or by being mid-fork when it lands, as a loaded macOS runner + /// showed — and it holds the output pipes. Once the direct child is gone, + /// the rest of the group must still be stopped within the grace. + #[cfg(unix)] + #[tokio::test(flavor = "multi_thread")] + async fn cancellation_stops_a_descendant_that_outlives_the_term() { + let (tx, mut rx) = mpsc::channel(64); + // An ignored signal stays ignored across exec. + let argv: Vec = ["sh", "-c", "(trap '' TERM; exec sleep 300) & echo $!; wait"] + .map(String::from) + .to_vec(); + let cancel = CancellationToken::new(); + let handle = { + let cancel = cancel.clone(); + tokio::spawn(async move { + run_streaming( + &argv, + None, + &[], + tx, + cancel, + Duration::from_secs(60), + Duration::from_secs(1), + ) + .await + }) + }; + let sleeper: i32 = rx.recv().await.unwrap().line.trim().parse().unwrap(); + let started = std::time::Instant::now(); + cancel.cancel(); + assert_eq!(handle.await.unwrap().unwrap(), CommandOutcome::Cancelled); + assert!(started.elapsed() < Duration::from_secs(5), "cancel was not prompt"); + // Gone, not merely signalled: give init a moment to reap it. + let deadline = std::time::Instant::now() + Duration::from_secs(3); + while unsafe { libc::kill(sleeper, 0) } == 0 { + assert!( + std::time::Instant::now() < deadline, + "the descendant that ignored SIGTERM outlived the cancel" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + #[cfg(windows)] struct WindowsTestTree(Vec); diff --git a/engine/mast-engine/src/ops.rs b/engine/mast-engine/src/ops.rs index f454ebb..2c4cb1c 100644 --- a/engine/mast-engine/src/ops.rs +++ b/engine/mast-engine/src/ops.rs @@ -328,7 +328,16 @@ impl Engine { Duration::from_secs(8), ) .await; - let _ = forwarder.await; + // The process has exited: forward what it wrote, but only for a + // moment. The forwarder ends when every holder of the output pipes + // has closed them, and a background process the command left behind + // inherits them — waiting on it would hold the operation open for as + // long as that process runs. Its later output belongs to no + // operation, so it is dropped. + let mut forwarder = forwarder; + if tokio::time::timeout(Duration::from_secs(2), &mut forwarder).await.is_err() { + forwarder.abort(); + } result.map_err(|e| ErrorInfo::Internal { message: redactor.redact(&e.to_string()) }) } } diff --git a/engine/mast-engine/tests/engine_tests.rs b/engine/mast-engine/tests/engine_tests.rs index 0720e8b..5e7a89b 100644 --- a/engine/mast-engine/tests/engine_tests.rs +++ b/engine/mast-engine/tests/engine_tests.rs @@ -3485,6 +3485,46 @@ async fn removing_project_cancels_its_running_command_before_forgetting_it() { assert!(matches!(terminal.kind, OperationEventKind::Cancelled)); } +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread")] +async fn a_command_that_leaves_a_background_process_still_finishes() { + use std::os::unix::fs::PermissionsExt; + + let tmp = tempfile::tempdir().unwrap(); + let project = make_project(tmp.path(), "leaves-a-daemon"); + // The background sleeper inherits stdout and stderr, so the output pipes + // stay open long after the script itself has exited. + let script = project.join("start.sh"); + std::fs::write(&script, "#!/bin/sh\nsleep 300 &\necho $! > daemon.pid\necho started\n").unwrap(); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); + let engine = test_engine(tmp.path(), Arc::new(FakeConnector(FakeAdapter::new()))); + run_action(&engine, Action::ImportProject { path: project.to_string_lossy().into() }).await; + let id = engine.snapshot().projects[0].id.clone(); + run_action(&engine, Action::SetProjectCommands { id: id.clone(), commands: vec![ + mast_contract::ProjectCommand { + name: "start".into(), command: "./start.sh".into(), + auto_start: false, cwd: None, after: None, ready_when: None, + auto_restart: false, restart_when_changed: Vec::new(), from_manifest: false, + }, + ] }).await; + let op = engine.dispatch(Action::RunProjectCommand { id, name: "start".into() }).unwrap(); + let mut events = engine.operation_events(op).unwrap(); + let ended = tokio::time::timeout(Duration::from_secs(15), async { + while let Some(event) = events.next().await { + if event.kind.is_terminal() { + return event.kind; + } + } + panic!("operation ended without a terminal event"); + }) + .await; + if let Ok(pid) = std::fs::read_to_string(project.join("daemon.pid")) { + let _ = std::process::Command::new("kill").arg(pid.trim()).status(); + } + let ended = ended.expect("the operation must end with the command, not with what it left running"); + assert!(matches!(ended, OperationEventKind::Completed), "{ended:?}"); +} + #[cfg(unix)] #[tokio::test(flavor = "multi_thread")] async fn project_removal_blocks_new_commands_until_shutdown_finishes() { From dfe19fc5077d0cab9c6a6cedea1ba5d16671e26b Mon Sep 17 00:00:00 2001 From: Stackrats Date: Wed, 14 Oct 2026 21:36:16 +0700 Subject: [PATCH 2/2] Mast v0.7.8 --- Cargo.lock | 30 +++++++++---------- Cargo.toml | 2 +- clients/desktop-vue/src-tauri/tauri.conf.json | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d82738..285dc45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2464,7 +2464,7 @@ dependencies = [ [[package]] name = "mast-cli" -version = "0.7.7" +version = "0.7.8" dependencies = [ "clap", "futures", @@ -2481,7 +2481,7 @@ dependencies = [ [[package]] name = "mast-client" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-trait", "futures", @@ -2491,7 +2491,7 @@ dependencies = [ [[package]] name = "mast-client-ipc" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-trait", "futures", @@ -2505,7 +2505,7 @@ dependencies = [ [[package]] name = "mast-client-local" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-trait", "futures", @@ -2520,7 +2520,7 @@ dependencies = [ [[package]] name = "mast-compose" -version = "0.7.7" +version = "0.7.8" dependencies = [ "mast-docker", "mast-yaml-edit", @@ -2536,7 +2536,7 @@ dependencies = [ [[package]] name = "mast-contract" -version = "0.7.7" +version = "0.7.8" dependencies = [ "serde", "serde_json", @@ -2545,7 +2545,7 @@ dependencies = [ [[package]] name = "mast-daemon" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-trait", "futures", @@ -2565,7 +2565,7 @@ dependencies = [ [[package]] name = "mast-desktop" -version = "0.7.7" +version = "0.7.8" dependencies = [ "futures", "gdkx11", @@ -2595,7 +2595,7 @@ dependencies = [ [[package]] name = "mast-diagnostics" -version = "0.7.7" +version = "0.7.8" dependencies = [ "mast-laravel", "rusqlite", @@ -2605,7 +2605,7 @@ dependencies = [ [[package]] name = "mast-docker" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-trait", "bollard", @@ -2619,7 +2619,7 @@ dependencies = [ [[package]] name = "mast-engine" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-trait", "futures", @@ -2647,7 +2647,7 @@ dependencies = [ [[package]] name = "mast-laravel" -version = "0.7.7" +version = "0.7.8" dependencies = [ "mast-docker", "tempfile", @@ -2657,7 +2657,7 @@ dependencies = [ [[package]] name = "mast-project" -version = "0.7.7" +version = "0.7.8" dependencies = [ "dirs 6.0.0", "serde", @@ -2670,7 +2670,7 @@ dependencies = [ [[package]] name = "mast-registry" -version = "0.7.7" +version = "0.7.8" dependencies = [ "reqwest 0.12.28", "serde", @@ -2681,7 +2681,7 @@ dependencies = [ [[package]] name = "mast-yaml-edit" -version = "0.7.7" +version = "0.7.8" dependencies = [ "saphyr", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 6b1d602..5636269 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ members = [ ] [workspace.package] -version = "0.7.7" +version = "0.7.8" edition = "2024" [workspace.dependencies] diff --git a/clients/desktop-vue/src-tauri/tauri.conf.json b/clients/desktop-vue/src-tauri/tauri.conf.json index 92689c4..38098a8 100644 --- a/clients/desktop-vue/src-tauri/tauri.conf.json +++ b/clients/desktop-vue/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Mast", - "version": "0.7.7", + "version": "0.7.8", "identifier": "dev.mast.desktop", "build": { "beforeDevCommand": "vp run --filter mast-desktop-ui dev",