--- a/src/sync/mcs.rs +++ b/src/sync/mcs.rs @@ -4,7 +4,7 @@ //! word, eliminating cache-line bouncing under contention. FIFO ordering //! guarantees fairness. O(1) cache-line transfers on unlock. -use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, Ordering}; use core::{hint, ptr}; use crate::percpu::PercpuBlock; @@ -27,12 +27,16 @@ /// Raw MCS spinlock primitive. pub struct McsRawLock { tail: AtomicPtr, + /// CPU ID of the current lock holder (for priority inheritance). + /// `u32::MAX` means no holder. + holder_cpu: AtomicU32, } impl McsRawLock { pub const fn new() -> Self { Self { tail: AtomicPtr::new(ptr::null_mut()), + holder_cpu: AtomicU32::new(u32::MAX), } } @@ -42,21 +46,37 @@ node.locked.store(true, Ordering::Relaxed); let prev = self.tail.swap((node as *const McsNode).cast_mut(), Ordering::AcqRel); if prev.is_null() { + // Uncontended — record ourselves as holder + let cpu_id = PercpuBlock::current().cpu_id.get(); + self.holder_cpu.store(cpu_id, Ordering::Release); return false; } unsafe { (*prev).next.store((node as *const McsNode).cast_mut(), Ordering::Release); } let percpu = PercpuBlock::current(); + let mut donated = false; while node.locked.load(Ordering::Acquire) { percpu.maybe_handle_tlb_shootdown(); + // Donate priority to the lock holder once per acquisition + if !donated { + self.maybe_donate_priority(percpu); + donated = true; + } hint::spin_loop(); } + // We now hold the lock + self.holder_cpu.store(percpu.cpu_id.get(), Ordering::Release); true } #[inline] pub fn release(&self, node: &McsNode) { + // Clear priority inheritance donation — we no longer hold the lock + PercpuBlock::current().pi_donated_prio.store(u32::MAX, Ordering::Relaxed); + // Clear holder CPU + self.holder_cpu.store(u32::MAX, Ordering::Release); + let next = node.next.load(Ordering::Acquire); if next.is_null() { if self @@ -84,13 +104,43 @@ pub fn try_acquire(&self, node: &McsNode) -> bool { node.next.store(ptr::null_mut(), Ordering::Relaxed); node.locked.store(true, Ordering::Relaxed); - self.tail + let ok = self + .tail .compare_exchange( ptr::null_mut(), (node as *const McsNode).cast_mut(), Ordering::AcqRel, Ordering::Acquire, ) - .is_ok() + .is_ok(); + if ok { + let cpu_id = PercpuBlock::current().cpu_id.get(); + self.holder_cpu.store(cpu_id, Ordering::Release); + } + ok + } + + /// Donate current CPU's context priority to the lock holder's CPU. + /// Reads priority from PercpuBlock::current_prio (cached by the scheduler) + /// to avoid acquiring any lock in the MCS spin loop. + fn maybe_donate_priority(&self, my_percpu: &PercpuBlock) { + let holder_cpu_id = self.holder_cpu.load(Ordering::Relaxed); + if holder_cpu_id == u32::MAX { + return; + } + // Read our own priority from the per-CPU cache (set by scheduler, + // no lock required). + let my_prio = my_percpu.current_prio.get(); + // Look up holder's PercpuBlock + let holder_percpu = crate::percpu::get_for_cpu( + crate::cpu_set::LogicalCpuId::new(holder_cpu_id), + ); + if let Some(holder) = holder_percpu { + let current_donated = holder.pi_donated_prio.load(Ordering::Relaxed); + // Donate if our priority is higher (lower number) + if (my_prio as u32) < current_donated { + holder.pi_donated_prio.store(my_prio as u32, Ordering::Relaxed); + } + } } } --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -16,6 +16,7 @@ use core::{ cell::{Cell, RefCell}, mem, + sync::atomic::Ordering, }; use syscall::PtraceFlags; @@ -442,6 +443,8 @@ // Is this context runnable on this CPU? let sw = unsafe { update_runnable(&mut next_context_guard, cpu_id, switch_time) }; if let UpdateResult::CanSwitch = sw { + // Cache the new context's priority for MCS lock priority donation. + percpu.current_prio.set(next_context_guard.prio); next_context_guard_opt = Some(next_context_guard); balance[i] -= SCHED_PRIO_TO_WEIGHT[20]; break 'priority; @@ -466,7 +469,10 @@ // Send the old process to the back of the line (if it is still runnable) let prev_ctx = WeakContextRef(Arc::downgrade(&prev_context_lock)); if prev_context_guard.status.is_runnable() { - let prio = prev_context_guard.prio; + let raw_prio = prev_context_guard.prio; + let prio = percpu.effective_prio(raw_prio); + // Clear PI donation — previous context is being re-queued + percpu.pi_donated_prio.store(u32::MAX, Ordering::Relaxed); contexts_list[prio].push_back(prev_ctx); } else { idle_contexts(token.token()).push_back(prev_ctx); @@ -478,7 +484,8 @@ return Ok(Some(next_context_guard)); } else { if !was_idle && !Arc::ptr_eq(&prev_context_lock, &idle_context) { - // We switch into the idle context + // Switching to idle context — cache lowest priority + percpu.current_prio.set(39); Ok(Some(unsafe { idle_context.write_arc() })) } else { // We found no other process to run.