driver-manager: crash-aware reap + error_channel fd cleanup (N18 Q2+Bug4)
Splits `reap_pid` into the `reap_pid_inner(crashed: bool)` private helper plus two public entry points: - `reap_pid(pid)` — crash path. Calls `crash_tracker().record_failure` after the maps are cleared, which advances the auto-blacklist counter. - `reap_pid_clean(pid)` — clean exit path. Calls `crash_tracker().record_success` so the device's backoff is reset; a future crash starts the count from 1, not from where the previous clean-exit backoff ended. Both paths also call `error_channel::global().remove(&key)` to close the sidecar UnixStream fd (Bug4: each crash previously leaked one fd; ~1024 crashes exhausted the manager's fd table). This is the kernel of N18-Q2+Bug4; the call-site change in `main.rs` that interprets the waitpid status was committed in the prior Q1+Q2+S3 commit.
This commit is contained in:
@@ -1259,16 +1259,25 @@ impl DriverConfig {
|
||||
Ok(configs)
|
||||
}
|
||||
|
||||
/// Remove a dead-pid entry from both `spawned` and `pid_to_device`
|
||||
/// maps. Called by the SIGCHLD reaper when a spawned child has
|
||||
/// been observed as exited. Idempotent — a no-op if the pid is not
|
||||
/// present.
|
||||
///
|
||||
/// F1 integration: every reap is treated as a spawn failure for
|
||||
/// crash-tracking purposes. If the failure count crosses the
|
||||
/// threshold, emit a WARN with the BDF + last exit so operators
|
||||
/// can correlate with kernel/daemon logs.
|
||||
/// Reap a child known to have exited via crash (non-zero exit code,
|
||||
/// or a signal other than SIGTERM/SIGINT). Removes the device from
|
||||
/// the spawned/pid_to_device maps, closes the error_channel fd, and
|
||||
/// advances the CrashTracker. After `threshold` consecutive crashes
|
||||
/// the device is auto-blacklisted.
|
||||
pub fn reap_pid(&self, pid: u32) {
|
||||
self.reap_pid_inner(pid, /*crashed=*/ true);
|
||||
}
|
||||
|
||||
/// Reap a child known to have exited cleanly (exit code 0, or
|
||||
/// SIGTERM/SIGINT from our own `remove()`). Removes the device from
|
||||
/// the maps and closes the error_channel fd, but does NOT advance
|
||||
/// the crash tracker — this prevents the 5-consecutive-exits
|
||||
/// autoblacklist from triggering on intentional driver reloads.
|
||||
pub fn reap_pid_clean(&self, pid: u32) {
|
||||
self.reap_pid_inner(pid, /*crashed=*/ false);
|
||||
}
|
||||
|
||||
fn reap_pid_inner(&self, pid: u32, crashed: bool) {
|
||||
let device_key = if let Ok(mut p2d) = self.pid_to_device.lock() {
|
||||
if let Some(key) = p2d.remove(&pid) {
|
||||
if let Ok(mut spawned) = self.spawned.lock() {
|
||||
@@ -1282,15 +1291,26 @@ impl DriverConfig {
|
||||
None
|
||||
};
|
||||
if let Some(key) = device_key {
|
||||
if let Some(count) = crash_tracker().record_failure(&key) {
|
||||
log::warn!(
|
||||
"crash-threshold: driver={} device={} consecutive_failures={} \
|
||||
auto-blacklisted (override via /etc/driver-manager.d/disable-<bdf>.toml \
|
||||
or REDBEAR_DRIVER_CRASH_THRESHOLD override)",
|
||||
self.name,
|
||||
key,
|
||||
count
|
||||
);
|
||||
// N18 Bug4: close the error_channel fd on every reap path so
|
||||
// a crashed or cleanly-exited daemon doesn't leak a UnixStream
|
||||
// until process exit.
|
||||
crate::error_channel::global().remove(&key);
|
||||
if crashed {
|
||||
if let Some(count) = crash_tracker().record_failure(&key) {
|
||||
log::warn!(
|
||||
"crash-threshold: driver={} device={} consecutive_failures={} \
|
||||
auto-blacklisted (override via /etc/driver-manager.d/disable-<bdf>.toml \
|
||||
or REDBEAR_DRIVER_CRASH_THRESHOLD override)",
|
||||
self.name,
|
||||
key,
|
||||
count
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Clean exit: clear any backoff the device was in so a
|
||||
// later crash starts the count from 1, not from where the
|
||||
// previous clean-exit backoff ended.
|
||||
crash_tracker().record_success(&key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user