vasilito
34360e1e4f
P0-P2: Barrier SMP, sigmask/pthread_kill races, robust mutexes, RT scheduling, POSIX sched API P3: PerCpuSched struct, per-CPU wiring, work stealing, load balancing, initial placement P4: 64-shard futex table, REQUEUE, PI futexes (LOCK_PI/UNLOCK_PI/TRYLOCK_PI), robust futexes, vruntime tracking, min-vruntime SCHED_OTHER selection P5: setpriority/getpriority, pthread_setaffinity_np, pthread_setname_np, pthread_setschedparam (Redox) P6: Cache-affine scheduling (last_cpu + vruntime bonus), NUMA topology kernel hints + numad userspace daemon Stability fixes: make_consistent stores 0 (dead TID fix), cond.rs error propagation, SPIN_COUNT adaptive spinning, Sys::open &str fix, PI futex CAS race, proc.rs lock ordering, barrier destroy Patches: 33 kernel + 58 relibc patches, all tracked in recipes Docs: KERNEL-SCHEDULER-MULTITHREAD-IMPROVEMENT-PLAN.md updated, SCHEDULER-REVIEW-FINAL.md created Architecture: NUMA topology parsing stays userspace (numad daemon), kernel stores lightweight NumaTopology hints
126 lines
3.7 KiB
Diff
126 lines
3.7 KiB
Diff
diff --git a/src/sync/barrier.rs b/src/sync/barrier.rs
|
|
index 6204a23..b5847b5 100644
|
|
--- a/src/sync/barrier.rs
|
|
+++ b/src/sync/barrier.rs
|
|
@@ -1,18 +1,34 @@
|
|
-use core::num::NonZeroU32;
|
|
+use core::{
|
|
+ num::NonZeroU32,
|
|
+ sync::atomic::{AtomicU32, Ordering},
|
|
+};
|
|
|
|
pub struct Barrier {
|
|
original_count: NonZeroU32,
|
|
// 4
|
|
lock: crate::sync::Mutex<Inner>,
|
|
// 16
|
|
- cvar: crate::header::pthread::RlctCond,
|
|
+ cvar: FutexState,
|
|
// 24
|
|
}
|
|
#[derive(Debug)]
|
|
struct Inner {
|
|
- count: u32,
|
|
- // TODO: Overflows might be problematic... 64-bit?
|
|
- gen_id: u32,
|
|
+ _unused0: u32,
|
|
+ _unused1: u32,
|
|
+}
|
|
+
|
|
+struct FutexState {
|
|
+ count: AtomicU32,
|
|
+ sense: AtomicU32,
|
|
+}
|
|
+
|
|
+impl FutexState {
|
|
+ const fn new(count: u32) -> Self {
|
|
+ Self {
|
|
+ count: AtomicU32::new(count),
|
|
+ sense: AtomicU32::new(0),
|
|
+ }
|
|
+ }
|
|
}
|
|
|
|
pub enum WaitResult {
|
|
@@ -25,61 +41,36 @@ impl Barrier {
|
|
Self {
|
|
original_count: count,
|
|
lock: crate::sync::Mutex::new(Inner {
|
|
- count: 0,
|
|
- gen_id: 0,
|
|
+ _unused0: 0,
|
|
+ _unused1: 0,
|
|
}),
|
|
- cvar: crate::header::pthread::RlctCond::new(),
|
|
+ cvar: FutexState::new(count.get()),
|
|
}
|
|
}
|
|
pub fn wait(&self) -> WaitResult {
|
|
- let mut guard = self.lock.lock();
|
|
- let gen_id = guard.gen_id;
|
|
-
|
|
- guard.count += 1;
|
|
-
|
|
- if guard.count == self.original_count.get() {
|
|
- guard.gen_id = guard.gen_id.wrapping_add(1);
|
|
- guard.count = 0;
|
|
- if let Ok(()) = self.cvar.broadcast() {}; // TODO handle error
|
|
+ let _ = &self.lock;
|
|
+ let sense = self.cvar.sense.load(Ordering::Acquire);
|
|
|
|
- drop(guard);
|
|
+ if self.cvar.count.fetch_sub(1, Ordering::AcqRel) == 1 {
|
|
+ self.cvar
|
|
+ .count
|
|
+ .store(self.original_count.get(), Ordering::Relaxed);
|
|
+ self.cvar
|
|
+ .sense
|
|
+ .store(sense.wrapping_add(1), Ordering::Release);
|
|
+ crate::sync::futex_wake(&self.cvar.sense, i32::MAX);
|
|
|
|
WaitResult::NotifiedAll
|
|
} else {
|
|
- while guard.gen_id == gen_id {
|
|
- guard = self.cvar.wait_inner_typedmutex(guard);
|
|
- }
|
|
-
|
|
- WaitResult::Waited
|
|
- }
|
|
- /*
|
|
- let mut guard = self.lock.lock();
|
|
- let Inner { count, gen_id } = *guard;
|
|
-
|
|
- let last = self.original_count.get() - 1;
|
|
-
|
|
- if count == last {
|
|
- eprintln!("last {:?}", *guard);
|
|
- guard.gen_id = guard.gen_id.wrapping_add(1);
|
|
- guard.count = 0;
|
|
-
|
|
- drop(guard);
|
|
-
|
|
- self.cvar.broadcast();
|
|
-
|
|
- WaitResult::NotifiedAll
|
|
- } else {
|
|
- guard.count += 1;
|
|
-
|
|
- while guard.count != last && guard.gen_id == gen_id {
|
|
- eprintln!("before {:?}", *guard);
|
|
- guard = self.cvar.wait_inner_typedmutex(guard);
|
|
- eprintln!("after {:?}", *guard);
|
|
+ // SMP fix: wait directly on the barrier generation word instead of routing through the
|
|
+ // condvar unlock->futex_wait path. If the last thread flips `sense` after we load it
|
|
+ // but before our futex wait starts, the futex observes a stale value and returns
|
|
+ // immediately instead of sleeping forever after a missed broadcast wakeup.
|
|
+ while self.cvar.sense.load(Ordering::Acquire) == sense {
|
|
+ let _ = crate::sync::futex_wait(&self.cvar.sense, sense, None);
|
|
}
|
|
|
|
WaitResult::Waited
|
|
}
|
|
- */
|
|
}
|
|
}
|
|
-static LOCK: crate::sync::Mutex<()> = crate::sync::Mutex::new(());
|