Skip to content

fix(runtime): 回收分离子进程，并按在线状态/退避收敛常驻轮询 #801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
coder-hhx merged 1 commit into from
Sep 11, 2026
Merged

fix(runtime): 回收分离子进程，并按在线状态/退避收敛常驻轮询 #801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions crates/agent-gui/src-tauri/src/commands/integration/mcp.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -903,7 +903,14 @@ impl SseTransport {
let thread_server_id = config.id.trim().to_string();
let thread_server_url = url.to_string();

let handle = std::thread::spawn(move || loop {
// 失败重连用退避：固定 1s 会在上游不可达时以每秒一次的频率反复建连
//（DNS + TCP + TLS 握手），而"配置了 SSE server 却连不上"时这个循环是
// 常驻的。连上一次即复位，避免把瞬时抖动放大成持续退避。
const SSE_RECONNECT_MIN: Duration = Duration::from_secs(1);
const SSE_RECONNECT_MAX: Duration = Duration::from_secs(30);
let handle = std::thread::spawn(move || {
let mut backoff = SSE_RECONNECT_MIN;
loop {
if thread_stop.load(Ordering::Relaxed) {
break;
}
Expand All @@ -921,15 +928,22 @@ impl SseTransport {
builder = builder.header(ACCEPT, "text/event-stream");

let resp = match builder.send() {
Ok(r) => r,
Ok(r) => {
// 建连成功即复位退避：下一次失败重新从最小间隔起，避免把
// 瞬时抖动累积成持续 30s 才重连一次。
backoff = SSE_RECONNECT_MIN;
r
}
Err(_) => {
std::thread::sleep(Duration::from_secs(1));
std::thread::sleep(backoff);
backoff = (backoff * 2).min(SSE_RECONNECT_MAX);
continue;
}
};

if !resp.status().is_success() {
std::thread::sleep(Duration::from_secs(1));
std::thread::sleep(backoff);
backoff = (backoff * 2).min(SSE_RECONNECT_MAX);
continue;
}

Expand Down Expand Up @@ -996,6 +1010,7 @@ impl SseTransport {
continue;
}
}
}
});

Ok(Self {
Expand Down
21 changes: 15 additions & 6 deletions crates/agent-gui/src-tauri/src/commands/workspace/fs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3425,8 +3425,11 @@ fn workspace_open_command(target: &Path, mode: &str) -> Command {

#[cfg(target_os = "macos")]
pub(crate) fn spawn_workspace_open_command(target: &Path, mode: &str) -> Result<(), String> {
workspace_open_command(target, mode)
.spawn()
// 启动器进程不等，但必须收尸：Child 直接丢掉的话，子进程退出后没人
// wait()，每次在 Finder/资源管理器里打开或显示文件都会在本进程下留一个
// <defunct>（实测报告里长会话的僵尸累积就是这条路径）。
let mut command = workspace_open_command(target, mode);
crate::runtime::process::spawn_and_reap(&mut command)
.map(|_| ())
.map_err(|e| format!("Failed to open path with macOS open: {e}"))
}
Expand All @@ -3452,8 +3455,11 @@ fn workspace_open_command(target: &Path, mode: &str) -> Command {

#[cfg(target_os = "windows")]
pub(crate) fn spawn_workspace_open_command(target: &Path, mode: &str) -> Result<(), String> {
workspace_open_command(target, mode)
.spawn()
// 启动器进程不等，但必须收尸：Child 直接丢掉的话，子进程退出后没人
// wait()，每次在 Finder/资源管理器里打开或显示文件都会在本进程下留一个
// <defunct>（实测报告里长会话的僵尸累积就是这条路径）。
let mut command = workspace_open_command(target, mode);
crate::runtime::process::spawn_and_reap(&mut command)
.map(|_| ())
.map_err(|e| format!("Failed to open path with Windows Explorer: {e}"))
}
Expand All @@ -3472,8 +3478,11 @@ fn workspace_open_command(target: &Path, mode: &str) -> Command {

#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))]
pub(crate) fn spawn_workspace_open_command(target: &Path, mode: &str) -> Result<(), String> {
workspace_open_command(target, mode)
.spawn()
// 启动器进程不等，但必须收尸：Child 直接丢掉的话，子进程退出后没人
// wait()，每次在 Finder/资源管理器里打开或显示文件都会在本进程下留一个
// <defunct>（实测报告里长会话的僵尸累积就是这条路径）。
let mut command = workspace_open_command(target, mode);
crate::runtime::process::spawn_and_reap(&mut command)
.map(|_| ())
.map_err(|e| format!("Failed to open path with xdg-open: {e}"))
}
Expand Down
9 changes: 5 additions & 4 deletions crates/agent-gui/src-tauri/src/commands/workspace/git.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use wait_timeout::ChildExt;

use crate::commands::system::validate_project_folder_name;
use crate::runtime::process::{
configure_child_process_group, kill_child_process_tree_best_effort,
configure_child_process_group, kill_child_process_tree_best_effort, spawn_and_reap,
terminate_process_tree_by_pid,
};

Expand Down Expand Up @@ -1357,9 +1357,10 @@ fn spawn_system_file_manager(program: &str, args: &[String]) -> Result<(), Strin
.args(args)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.map_err(|error| format!("打开系统资源管理器失败：{error}"))?;
.stderr(Stdio::null());
// 启动器不等，但必须收尸：`Child` 直接丢掉的话，子进程退出后没人 wait()，
// 每次"在文件管理器中显示"都会在本进程下留一个 <defunct>。
spawn_and_reap(&mut command).map_err(|error| format!("打开系统资源管理器失败：{error}"))?;
Ok(())
}

Expand Down
8 changes: 7 additions & 1 deletion crates/agent-gui/src-tauri/src/runtime/managed_process.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,13 @@ const DEFAULT_WAIT_MS: u64 = 30_000;
const MAX_WAIT_MS: u64 = 300_000;
const WAIT_POLL_MS: u64 = 50;
/// Rate limit for pid-probing restored entries (no Child handle to poll).
const RESTORED_PROBE_INTERVAL_MS: u128 = 2000;
///
/// 每次探测都是一次 `fork/exec`（`ps -p <pid> -o etime=`），而这条 tick 是
/// 常驻的：只要 journal 里还有上一轮遗留的 isolated 进程，就会一直每 2s 起一个
/// 子进程。遗留进程的存活粒度不需要秒级——15s 足以在面板上及时反映它退出，
/// 而空闲时少起 7 倍的短命进程（实测报告里主进程无操作也持续占用 CPU 的一条
/// 来源）。
const RESTORED_PROBE_INTERVAL_MS: u128 = 15_000;
/// `ps -o etime` has second granularity; a restored pid whose probed start
/// time drifts beyond this from the journaled one is a reused pid, not ours.
const START_TIME_TOLERANCE_MS: i64 = 60_000;
Expand Down
104 changes: 104 additions & 0 deletions crates/agent-gui/src-tauri/src/runtime/process.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,10 +217,89 @@ pub(crate) fn probe_process_start_time(_pid: u32) -> ProcessProbe {
ProcessProbe::Unknown
}

/// `ps -o etime=` 只有秒级精度，启动时间换算允许 ±2s 误差。超出即认为这个 pid 已
/// 被别的进程复用，不能再对它发信号。
#[cfg(unix)]
const PID_START_TIME_TOLERANCE_MS: i64 = 2_000;

/// 按 pid 终止进程树，但先确认这个 pid 仍是当初启动的那个进程。
///
/// 收尸线程会在子进程退出后立刻 `wait()` 掉它，此后 pid 可以被内核复用——
/// 对一个"已经不是我们子进程"的 pid 发 TERM/KILL 会误伤无辜。所以调用方在启动时
/// 记下 `started_at_ms`，这里先比对启动时间：进程已消失（被判 Dead）或启动时间
/// 对不上（pid 复用）时直接返回。
pub(crate) fn terminate_process_tree_by_pid_if_same(
pid: u32,
started_at_ms: Option<i64>,
grace: Duration,
) {
let Some(expected_started_at_ms) = started_at_ms else {
// 启动时没探测到启动时间（`ps` 失败等）：退回旧的 pid-only 语义，
// 与 managed_process 的既有取舍一致。
terminate_process_tree_by_pid(pid, grace);
return;
};
match probe_process_start_time(pid) {
ProcessProbe::Dead => return,
ProcessProbe::Alive {
started_at_ms: actual,
} if (actual - expected_started_at_ms).abs() > PID_START_TIME_TOLERANCE_MS => return,
// Unknown（探测失败）沿用旧行为：保守发信号，宁可多杀一次进程组。
ProcessProbe::Alive { .. } | ProcessProbe::Unknown => {}
}
terminate_process_tree_by_pid(pid, grace);
}

/// Spawn a fire-and-forget child and reap it on a detached thread.
///
/// `std::process::Child` 不实现 drop-reap：`spawn()` 之后把 Child 丢掉，子进程退出
/// 时没有人 `wait()`，它就会在本进程下挂成 `<defunct>`，直到本进程退出为止。
/// 系统启动器（`open` / `explorer.exe` / `xdg-open`）必须"不等它、但要收尸"——
/// 否则每次在 Finder/资源管理器中显示文件都漏一个僵尸，长会话里 PID 表持续增长。
///
/// 返回子进程 pid，供调用方记录与测试观察。
pub(crate) fn spawn_and_reap(command: &mut Command) -> io::Result<u32> {
let child = command.spawn()?;
let pid = child.id();
spawn_child_reaper(child);
Ok(pid)
}

/// 在分离线程里 `wait()` 掉一个不再需要句柄的子进程。
///
/// 线程创建失败（极端资源耗尽）时这个子进程会退化成未收尸——比在这里 panic
/// 或阻塞调用方都更可接受，且这是可观测的：它会在进程表里显示为 `<defunct>`。
pub(crate) fn spawn_child_reaper(mut child: Child) {
let spawned = std::thread::Builder::new()
.name("child-reaper".to_string())
.spawn(move || {
let _ = child.wait();
});
if let Err(error) = spawned {
eprintln!("spawn child reaper failed (child may stay defunct): {error}");
}
}

#[cfg(all(test, unix))]
mod tests {
use super::*;

/// `ps -o stat=` 的首字符（`Z` 即僵尸）；进程已消失时返回 None。
#[cfg(test)]
fn process_state_flag(pid: u32) -> Option<String> {
let output = Command::new("ps")
.arg("-p")
.arg(pid.to_string())
.arg("-o")
.arg("stat=")
.stdin(Stdio::null())
.stderr(Stdio::null())
.output()
.ok()?;
let text = String::from_utf8_lossy(&output.stdout).trim().to_string();
if text.is_empty() { None } else { Some(text) }
}

#[test]
fn parse_ps_etime_handles_all_shapes() {
assert_eq!(parse_ps_etime_ms("05"), Some(5_000));
Expand Down Expand Up @@ -260,4 +339,29 @@ mod tests {
}
assert!(gone, "killed process still probes alive");
}

#[test]
fn detached_child_is_reaped_instead_of_left_defunct() {
// 回归锁：`spawn()` 之后丢掉 Child 的写法会让子进程退出后一直挂在父进程下
// 当 `<defunct>`（系统启动器每次调用漏一个）；收尸线程必须把它从进程表里清掉。
// 旧实现下这个循环会一直读到 "Z"，最终 panic。
let mut command = Command::new("true");
command
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null());
let pid = spawn_and_reap(&mut command).expect("true should spawn");

let mut zombies = Vec::new();
for _ in 0..150 {
match process_state_flag(pid) {
// 已收尸：进程表里不再有这一项。
None => return,
Some(state) if !state.starts_with('Z') => return,
Some(state) => zombies.push(state),
}
std::thread::sleep(Duration::from_millis(20));
}
panic!("detached child stayed defunct for 3s: {zombies:?}");
}
}
40 changes: 31 additions & 9 deletions crates/agent-gui/src-tauri/src/services/browser/launcher.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};

use crate::runtime::process::{configure_child_process_group, kill_child_process_tree_best_effort};
use crate::runtime::process::{
configure_child_process_group, kill_child_process_tree_best_effort, process_start_time_ms,
spawn_child_reaper, terminate_process_tree_by_pid_if_same,
};

/// 浏览器自动化专用 profile，与用户日常浏览器 profile 隔离（防登录态/凭据暴露）。
/// 见 docs/design/browser-automation.md。
Expand Down Expand Up @@ -76,21 +79,30 @@ fn browser_candidates() -> Vec<PathBuf> {
}

pub(crate) struct LaunchedBrowser {
pub child: Child,
/// 进程组 leader 的 pid。Child 句柄不在本结构里——它归收尸线程所有，见
/// `launch_browser` 与 `runtime::process::spawn_child_reaper`。
pid: u32,
/// 启动时刻的近似值（由 `ps -o etime` 反推），用于拒绝 pid 复用后的误杀；
/// 探测失败时为 None，此时 Drop 退回 pid-only 语义。
started_at_ms: Option<i64>,
pub executable: PathBuf,
pub debug_port: u16,
}

impl LaunchedBrowser {
/// 浏览器主进程 pid（供 BrowserManager 旁路记录，shutdown 兜底 kill 用）。
pub(crate) fn child_pid(&self) -> u32 {
self.child.id()
self.pid
}
}

impl Drop for LaunchedBrowser {
fn drop(&mut self) {
kill_child_process_tree_best_effort(&mut self.child);
terminate_process_tree_by_pid_if_same(
self.pid,
self.started_at_ms,
Duration::from_millis(300),
);
}
}

Expand Down Expand Up @@ -123,11 +135,21 @@ pub(crate) fn launch_browser(executable: &PathBuf) -> Result<LaunchedBrowser, St
.map_err(|e| format!("启动浏览器失败（{}）：{e}", executable.display()))?;

match wait_for_devtools_port(&port_file, &mut child, Duration::from_secs(15)) {
Ok(debug_port) => Ok(LaunchedBrowser {
child,
executable: executable.clone(),
debug_port,
}),
Ok(debug_port) => {
let pid = child.id();
// 浏览器进程的存活期与整个会话相当，期间没有任何 tick 会去 try_wait
// 它。用户手关窗口（或它自己崩掉）之后，不收尸就会在父进程下留一个
// `<defunct>` 直到会话结束——实测报告里"每个 Browser 会话漏一个僵尸"
// 就是这条路径。句柄交给收尸线程后，kill 改走 pid + 启动时间比对。
let started_at_ms = process_start_time_ms(pid);
spawn_child_reaper(child);
Ok(LaunchedBrowser {
pid,
started_at_ms,
executable: executable.clone(),
debug_port,
})
}
Err(error) => {
kill_child_process_tree_best_effort(&mut child);
Err(error)
Expand Down
11 changes: 11 additions & 0 deletions crates/agent-gui/src-tauri/src/services/gateway/controller.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,12 @@ impl GatewayController {
tauri::async_runtime::spawn(async move {
loop {
tokio::time::sleep(GATEWAY_CHAT_LEASE_SWEEP_INTERVAL).await;
// 未连上网关时这些 tick 什么也做不成（inbox 只有远端会写、
// 租约只服务远端会话），但每条都要抢锁扫一遍。这条循环在未配置
// 网关的机器上也常驻，所以先按 online 收敛。
if !controller.status().online {
continue;
}
if let Err(error) = controller.expire_remote_chat_leases().await {
eprintln!("expire gateway remote chat leases failed: {error}");
}
Expand All @@ -179,6 +185,11 @@ impl GatewayController {
tauri::async_runtime::spawn(async move {
loop {
tokio::time::sleep(GATEWAY_RUNTIME_STATUS_REPUBLISH_INTERVAL).await;
// 离线时发送本身就是 no-op，但快照构造 + protobuf 编码照旧执行。
// 这条循环同样在未配置网关时常驻，先按 online 收敛掉。
if !controller.status().online {
continue;
}
let Some((worker_id, state, visible, active_run_count)) =
controller.runtime_status_republish_snapshot()
else {
Expand Down
Loading