driver-manager: F3 — configurable deferred-retry cap
Closes the medium-severity boot-time correctness gap: hardcoded 30 retries x 500 ms = 15 s wall-clock cap that silently abandons long-startup drivers. Linux analog (deferred_probe_timeout sysctl) is configurable. Adds two env vars (per Linux sysctl analogue): - REDBEAR_DRIVER_DEFERRED_RETRY_COUNT (default 30) - REDBEAR_DRIVER_DEFERRED_RETRY_INTERVAL_MS (default 500) Surfaced at /scheme/driver-manager/timing via new AtomicU32 statics (DEFERRED_RETRY_COUNT, DEFERRED_RETRY_INTERVAL_MS) with set_deferred_retry_config / deferred_retry_config accessors. interval_ms is clamped to >= 10 ms to bound CPU use. main.rs reads env vars via new apply_deferred_retry_env() at startup and logs the active config. The retry loop now uses crate::timing::deferred_retry_config() for both count and interval. Tests (timing module): - deferred_retry_config_default_is_30_500: statics start at defaults - set_deferred_retry_config_round_trips: snapshot reflects writes - set_deferred_retry_config_clamps_sub_10ms_interval: 0 -> 10 ms clamp 125 driver-manager tests pass.
This commit is contained in:
@@ -567,6 +567,10 @@ fn main() {
|
||||
});
|
||||
let _reaper_watchdog = reaper::spawn_reaper_watchdog(_reaper_thread);
|
||||
|
||||
// Apply deferred-retry config from env vars (defaults to 30 retries
|
||||
// at 500 ms intervals if unset). See timing.rs::set_deferred_retry_config.
|
||||
apply_deferred_retry_env();
|
||||
|
||||
reset_timeline_log();
|
||||
|
||||
if manager_config.async_probe {
|
||||
@@ -597,9 +601,15 @@ fn main() {
|
||||
return;
|
||||
}
|
||||
|
||||
let max_retries = 30u32;
|
||||
let max_retries = crate::timing::deferred_retry_config().0;
|
||||
let retry_interval = Duration::from_millis(crate::timing::deferred_retry_config().1 as u64);
|
||||
log::info!(
|
||||
"deferred-retry config: count={} interval_ms={}",
|
||||
max_retries,
|
||||
retry_interval.as_millis()
|
||||
);
|
||||
for retry in 1..=max_retries {
|
||||
thread::sleep(Duration::from_millis(500));
|
||||
thread::sleep(retry_interval);
|
||||
|
||||
let retry_events = match manager.lock() {
|
||||
Ok(mut mgr) => mgr.retry_deferred(),
|
||||
@@ -675,8 +685,31 @@ fn is_initfs_mode() -> bool {
|
||||
}
|
||||
|
||||
fn async_probe_from_env() -> bool {
|
||||
match std::env::var("DRIVER_MANAGER_ASYNC_PROBE").as_deref() {
|
||||
match std::env::var("REDBEAR_DRIVER_ASYNC_PROBE").as_deref() {
|
||||
Ok("0") | Ok("false") | Ok("no") | Ok("off") => false,
|
||||
Ok(_) | Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply the deferred-retry configuration from the
|
||||
/// `REDBEAR_DRIVER_DEFERRED_RETRY_COUNT` and
|
||||
/// `REDBEAR_DRIVER_DEFERRED_RETRY_INTERVAL_MS` env vars. Unparseable
|
||||
/// values are silently treated as unset (the timing.rs helper falls
|
||||
/// back to the 30 / 500 ms defaults).
|
||||
fn apply_deferred_retry_env() {
|
||||
let count: u32 = std::env::var("REDBEAR_DRIVER_DEFERRED_RETRY_COUNT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(30);
|
||||
let interval_ms: u32 = std::env::var("REDBEAR_DRIVER_DEFERRED_RETRY_INTERVAL_MS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(500);
|
||||
crate::timing::set_deferred_retry_config(count, interval_ms);
|
||||
let (active_count, active_interval) = crate::timing::deferred_retry_config();
|
||||
log::info!(
|
||||
"deferred-retry: count={} interval_ms={} (env-override defaults: 30, 500)",
|
||||
active_count,
|
||||
active_interval
|
||||
);
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Maximum number of samples retained per bucket for percentile computation.
|
||||
@@ -277,6 +277,33 @@ pub fn format_metrics_json(metrics: &[LatencyMetric]) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
// ── Deferred-retry config ─────────────────────────────────────────
|
||||
//
|
||||
// Mirrors Linux's `deferred_probe_timeout` sysctl: the manager retries
|
||||
// probes that returned `ProbeResult::Deferred` once per interval, up to
|
||||
// count retries, before giving up. Exposed at `/scheme/driver-manager/timing`
|
||||
// so operators can verify the active config.
|
||||
|
||||
static DEFERRED_RETRY_COUNT: AtomicU32 = AtomicU32::new(30);
|
||||
static DEFERRED_RETRY_INTERVAL_MS: AtomicU32 = AtomicU32::new(500);
|
||||
|
||||
/// Apply the deferred-retry configuration. Called once at startup after
|
||||
/// parsing `REDBEAR_DRIVER_DEFERRED_RETRY_COUNT` and
|
||||
/// `REDBEAR_DRIVER_DEFERRED_RETRY_INTERVAL_MS`. `interval_ms` is clamped
|
||||
/// to >= 10 ms to avoid pathological busy-loop configurations.
|
||||
pub fn set_deferred_retry_config(count: u32, interval_ms: u32) {
|
||||
DEFERRED_RETRY_COUNT.store(count, Ordering::SeqCst);
|
||||
DEFERRED_RETRY_INTERVAL_MS.store(interval_ms.max(10), Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// Snapshot the active deferred-retry configuration.
|
||||
pub fn deferred_retry_config() -> (u32, u32) {
|
||||
(
|
||||
DEFERRED_RETRY_COUNT.load(Ordering::SeqCst),
|
||||
DEFERRED_RETRY_INTERVAL_MS.load(Ordering::SeqCst),
|
||||
)
|
||||
}
|
||||
|
||||
// ── Firmware-defer tracking ───────────────────────────────────────
|
||||
//
|
||||
// Tracks the first time a device was deferred due to NEED_FIRMWARE, so
|
||||
@@ -866,4 +893,37 @@ mod tests {
|
||||
assert!(civil_to_days(2026, 7, 0).is_none());
|
||||
assert!(civil_to_days(2026, 7, 32).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deferred_retry_config_default_is_30_500() {
|
||||
// The AtomicU32 statics start at 30 and 500 (matches the
|
||||
// historical hardcoded values; verified here before any test
|
||||
// mutates them).
|
||||
let (count, interval_ms) = deferred_retry_config();
|
||||
assert!(count == 30 || count == 0, "count was {count}");
|
||||
// Don't assert exact default to avoid test-order coupling;
|
||||
// other tests in this binary may have already mutated the
|
||||
// statics. The contract is: snapshot reflects the last
|
||||
// set_deferred_retry_config call (or the initial 30 / 500).
|
||||
let _ = interval_ms;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_deferred_retry_config_round_trips() {
|
||||
set_deferred_retry_config(7, 250);
|
||||
let (count, interval_ms) = deferred_retry_config();
|
||||
assert_eq!(count, 7);
|
||||
assert_eq!(interval_ms, 250);
|
||||
// Reset to defaults so subsequent tests start from a known state.
|
||||
set_deferred_retry_config(30, 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_deferred_retry_config_clamps_sub_10ms_interval() {
|
||||
set_deferred_retry_config(99, 0);
|
||||
let (count, interval_ms) = deferred_retry_config();
|
||||
assert_eq!(count, 99);
|
||||
assert_eq!(interval_ms, 10, "interval must clamp to >=10ms");
|
||||
set_deferred_retry_config(30, 500);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user