45e086558e
- Fix P15-8-init-cycle-detection.patch: replace visiting+error with seen+silent-skip to eliminate 11 false-positive 'dependency cycle detected' errors on shared deps - Fix P0-daemon-fix-init-notify-unwrap.patch: remove eprintln! for missing INIT_NOTIFY (expected for oneshot_async services, ~7 daemons affected) - Fix driver-manager hotplug loop: add PERMANENTLY_SKIPPED static set shared between hotplug handler and DriverConfig::probe() to stop infinite re-probing of Fatal/NotSupported/deferred-exhausted device+driver pairs (e.g. ided) - Fix driver-manager log_timeline: suppress repeated EPIPE/ENOENT errors with AtomicI32 dedup and AtomicBool one-shot guards for boot timeline JSON - Add driver-manager SIGTERM handler, ACPI bus registration, --status mode, driver reap loop, graceful shutdown, and reduced deferred retries (30→3)
154 lines
5.8 KiB
Diff
154 lines
5.8 KiB
Diff
--- a/src/sync/mcs.rs
|
|
+++ b/src/sync/mcs.rs
|
|
@@ -4,7 +4,7 @@
|
|
//! word, eliminating cache-line bouncing under contention. FIFO ordering
|
|
//! guarantees fairness. O(1) cache-line transfers on unlock.
|
|
|
|
-use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
|
|
+use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, Ordering};
|
|
use core::{hint, ptr};
|
|
|
|
use crate::percpu::PercpuBlock;
|
|
@@ -27,12 +27,16 @@
|
|
/// Raw MCS spinlock primitive.
|
|
pub struct McsRawLock {
|
|
tail: AtomicPtr<McsNode>,
|
|
+ /// CPU ID of the current lock holder (for priority inheritance).
|
|
+ /// `u32::MAX` means no holder.
|
|
+ holder_cpu: AtomicU32,
|
|
}
|
|
|
|
impl McsRawLock {
|
|
pub const fn new() -> Self {
|
|
Self {
|
|
tail: AtomicPtr::new(ptr::null_mut()),
|
|
+ holder_cpu: AtomicU32::new(u32::MAX),
|
|
}
|
|
}
|
|
|
|
@@ -42,21 +46,37 @@
|
|
node.locked.store(true, Ordering::Relaxed);
|
|
let prev = self.tail.swap((node as *const McsNode).cast_mut(), Ordering::AcqRel);
|
|
if prev.is_null() {
|
|
+ // Uncontended — record ourselves as holder
|
|
+ let cpu_id = PercpuBlock::current().cpu_id.get();
|
|
+ self.holder_cpu.store(cpu_id, Ordering::Release);
|
|
return false;
|
|
}
|
|
unsafe {
|
|
(*prev).next.store((node as *const McsNode).cast_mut(), Ordering::Release);
|
|
}
|
|
let percpu = PercpuBlock::current();
|
|
+ let mut donated = false;
|
|
while node.locked.load(Ordering::Acquire) {
|
|
percpu.maybe_handle_tlb_shootdown();
|
|
+ // Donate priority to the lock holder once per acquisition
|
|
+ if !donated {
|
|
+ self.maybe_donate_priority(percpu);
|
|
+ donated = true;
|
|
+ }
|
|
hint::spin_loop();
|
|
}
|
|
+ // We now hold the lock
|
|
+ self.holder_cpu.store(percpu.cpu_id.get(), Ordering::Release);
|
|
true
|
|
}
|
|
|
|
#[inline]
|
|
pub fn release(&self, node: &McsNode) {
|
|
+ // Clear priority inheritance donation — we no longer hold the lock
|
|
+ PercpuBlock::current().pi_donated_prio.store(u32::MAX, Ordering::Relaxed);
|
|
+ // Clear holder CPU
|
|
+ self.holder_cpu.store(u32::MAX, Ordering::Release);
|
|
+
|
|
let next = node.next.load(Ordering::Acquire);
|
|
if next.is_null() {
|
|
if self
|
|
@@ -84,13 +104,43 @@
|
|
pub fn try_acquire(&self, node: &McsNode) -> bool {
|
|
node.next.store(ptr::null_mut(), Ordering::Relaxed);
|
|
node.locked.store(true, Ordering::Relaxed);
|
|
- self.tail
|
|
+ let ok = self
|
|
+ .tail
|
|
.compare_exchange(
|
|
ptr::null_mut(),
|
|
(node as *const McsNode).cast_mut(),
|
|
Ordering::AcqRel,
|
|
Ordering::Acquire,
|
|
)
|
|
- .is_ok()
|
|
+ .is_ok();
|
|
+ if ok {
|
|
+ let cpu_id = PercpuBlock::current().cpu_id.get();
|
|
+ self.holder_cpu.store(cpu_id, Ordering::Release);
|
|
+ }
|
|
+ ok
|
|
+ }
|
|
+
|
|
+ /// Donate current CPU's context priority to the lock holder's CPU.
|
|
+ /// Reads priority from PercpuBlock::current_prio (cached by the scheduler)
|
|
+ /// to avoid acquiring any lock in the MCS spin loop.
|
|
+ fn maybe_donate_priority(&self, my_percpu: &PercpuBlock) {
|
|
+ let holder_cpu_id = self.holder_cpu.load(Ordering::Relaxed);
|
|
+ if holder_cpu_id == u32::MAX {
|
|
+ return;
|
|
+ }
|
|
+ // Read our own priority from the per-CPU cache (set by scheduler,
|
|
+ // no lock required).
|
|
+ let my_prio = my_percpu.current_prio.get();
|
|
+ // Look up holder's PercpuBlock
|
|
+ let holder_percpu = crate::percpu::get_for_cpu(
|
|
+ crate::cpu_set::LogicalCpuId::new(holder_cpu_id),
|
|
+ );
|
|
+ if let Some(holder) = holder_percpu {
|
|
+ let current_donated = holder.pi_donated_prio.load(Ordering::Relaxed);
|
|
+ // Donate if our priority is higher (lower number)
|
|
+ if (my_prio as u32) < current_donated {
|
|
+ holder.pi_donated_prio.store(my_prio as u32, Ordering::Relaxed);
|
|
+ }
|
|
+ }
|
|
}
|
|
}
|
|
--- a/src/context/switch.rs
|
|
+++ b/src/context/switch.rs
|
|
@@ -16,6 +16,7 @@
|
|
use core::{
|
|
cell::{Cell, RefCell},
|
|
mem,
|
|
+ sync::atomic::Ordering,
|
|
};
|
|
use syscall::PtraceFlags;
|
|
|
|
@@ -442,6 +443,8 @@
|
|
// Is this context runnable on this CPU?
|
|
let sw = unsafe { update_runnable(&mut next_context_guard, cpu_id, switch_time) };
|
|
if let UpdateResult::CanSwitch = sw {
|
|
+ // Cache the new context's priority for MCS lock priority donation.
|
|
+ percpu.current_prio.set(next_context_guard.prio);
|
|
next_context_guard_opt = Some(next_context_guard);
|
|
balance[i] -= SCHED_PRIO_TO_WEIGHT[20];
|
|
break 'priority;
|
|
@@ -466,7 +469,10 @@
|
|
// Send the old process to the back of the line (if it is still runnable)
|
|
let prev_ctx = WeakContextRef(Arc::downgrade(&prev_context_lock));
|
|
if prev_context_guard.status.is_runnable() {
|
|
- let prio = prev_context_guard.prio;
|
|
+ let raw_prio = prev_context_guard.prio;
|
|
+ let prio = percpu.effective_prio(raw_prio);
|
|
+ // Clear PI donation — previous context is being re-queued
|
|
+ percpu.pi_donated_prio.store(u32::MAX, Ordering::Relaxed);
|
|
contexts_list[prio].push_back(prev_ctx);
|
|
} else {
|
|
idle_contexts(token.token()).push_back(prev_ctx);
|
|
@@ -478,7 +484,8 @@
|
|
return Ok(Some(next_context_guard));
|
|
} else {
|
|
if !was_idle && !Arc::ptr_eq(&prev_context_lock, &idle_context) {
|
|
- // We switch into the idle context
|
|
+ // Switching to idle context — cache lowest priority
|
|
+ percpu.current_prio.set(39);
|
|
Ok(Some(unsafe { idle_context.write_arc() }))
|
|
} else {
|
|
// We found no other process to run.
|