vasilito
34360e1e4f
P0-P2: Barrier SMP, sigmask/pthread_kill races, robust mutexes, RT scheduling, POSIX sched API P3: PerCpuSched struct, per-CPU wiring, work stealing, load balancing, initial placement P4: 64-shard futex table, REQUEUE, PI futexes (LOCK_PI/UNLOCK_PI/TRYLOCK_PI), robust futexes, vruntime tracking, min-vruntime SCHED_OTHER selection P5: setpriority/getpriority, pthread_setaffinity_np, pthread_setname_np, pthread_setschedparam (Redox) P6: Cache-affine scheduling (last_cpu + vruntime bonus), NUMA topology kernel hints + numad userspace daemon Stability fixes: make_consistent stores 0 (dead TID fix), cond.rs error propagation, SPIN_COUNT adaptive spinning, Sys::open &str fix, PI futex CAS race, proc.rs lock ordering, barrier destroy Patches: 33 kernel + 58 relibc patches, all tracked in recipes Docs: KERNEL-SCHEDULER-MULTITHREAD-IMPROVEMENT-PLAN.md updated, SCHEDULER-REVIEW-FINAL.md created Architecture: NUMA topology parsing stays userspace (numad daemon), kernel stores lightweight NumaTopology hints
90 lines
2.9 KiB
Diff
90 lines
2.9 KiB
Diff
diff --git a/src/percpu.rs b/src/percpu.rs
|
|
index f4ad5e6..1844d62 100644
|
|
--- a/src/percpu.rs
|
|
+++ b/src/percpu.rs
|
|
@@ -1,4 +1,5 @@
|
|
use alloc::{
|
|
+ collections::VecDeque,
|
|
sync::{Arc, Weak},
|
|
vec::Vec,
|
|
};
|
|
@@ -12,7 +13,10 @@ use syscall::PtraceFlags;
|
|
|
|
use crate::{
|
|
arch::device::ArchPercpuMisc,
|
|
- context::{empty_cr3, memory::AddrSpaceWrapper, switch::ContextSwitchPercpu},
|
|
+ context::{
|
|
+ empty_cr3, memory::AddrSpaceWrapper, switch::ContextSwitchPercpu, WeakContextRef,
|
|
+ RUN_QUEUE_COUNT,
|
|
+ },
|
|
cpu_set::{LogicalCpuId, MAX_CPU_COUNT},
|
|
cpu_stats::{CpuStats, CpuStatsData},
|
|
ptrace::Session,
|
|
@@ -20,6 +24,42 @@ use crate::{
|
|
syscall::debug::SyscallDebugInfo,
|
|
};
|
|
|
|
+#[allow(dead_code)]
|
|
+pub struct PerCpuSched {
|
|
+ pub run_queues: [VecDeque<WeakContextRef>; RUN_QUEUE_COUNT],
|
|
+ pub run_queues_lock: AtomicBool,
|
|
+ pub balance: Cell<[usize; RUN_QUEUE_COUNT]>,
|
|
+ pub last_queue: Cell<usize>,
|
|
+}
|
|
+
|
|
+impl PerCpuSched {
|
|
+ pub const fn new() -> Self {
|
|
+ const EMPTY: VecDeque<WeakContextRef> = VecDeque::new();
|
|
+ Self {
|
|
+ run_queues: [EMPTY; RUN_QUEUE_COUNT],
|
|
+ run_queues_lock: AtomicBool::new(false),
|
|
+ balance: Cell::new([0; RUN_QUEUE_COUNT]),
|
|
+ last_queue: Cell::new(0),
|
|
+ }
|
|
+ }
|
|
+
|
|
+ pub fn take_lock(&self) {
|
|
+ while self
|
|
+ .run_queues_lock
|
|
+ .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
|
|
+ .is_err()
|
|
+ {
|
|
+ while self.run_queues_lock.load(Ordering::Relaxed) {
|
|
+ core::hint::spin_loop();
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+
|
|
+ pub fn release_lock(&self) {
|
|
+ self.run_queues_lock.store(false, Ordering::Release);
|
|
+ }
|
|
+}
|
|
+
|
|
/// The percpu block, that stored all percpu variables.
|
|
pub struct PercpuBlock {
|
|
/// A unique immutable number that identifies the current CPU - used for scheduling
|
|
@@ -31,7 +71,12 @@ pub struct PercpuBlock {
|
|
pub current_addrsp: RefCell<Option<Arc<AddrSpaceWrapper>>>,
|
|
pub new_addrsp_tmp: Cell<Option<Arc<AddrSpaceWrapper>>>,
|
|
pub wants_tlb_shootdown: AtomicBool,
|
|
- pub balance: Cell<[usize; 40]>,
|
|
+
|
|
+ pub sched: PerCpuSched,
|
|
+
|
|
+ // Legacy DWRR state used by context/switch.rs until the per-CPU scheduler migration is
|
|
+ // finished.
|
|
+ pub balance: Cell<[usize; RUN_QUEUE_COUNT]>,
|
|
pub last_queue: Cell<usize>,
|
|
|
|
// TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
|
|
@@ -187,7 +232,8 @@ impl PercpuBlock {
|
|
current_addrsp: RefCell::new(None),
|
|
new_addrsp_tmp: Cell::new(None),
|
|
wants_tlb_shootdown: AtomicBool::new(false),
|
|
- balance: Cell::new([0; 40]),
|
|
+ sched: PerCpuSched::new(),
|
|
+ balance: Cell::new([0; RUN_QUEUE_COUNT]),
|
|
last_queue: Cell::new(39),
|
|
ptrace_flags: Cell::new(PtraceFlags::empty()),
|
|
ptrace_session: RefCell::new(None),
|