diff --git a/src/context/context.rs b/src/context/context.rs index fbb20c4c7c..8bd44177e0 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -279,7 +279,6 @@ impl Context { pub fn unblock_no_ipi(&mut self) -> bool { if self.status.is_soft_blocked() { self.status = Status::Runnable; - self.wake = None; self.status_reason = ""; true diff --git a/src/context/mod.rs b/src/context/mod.rs index f5a8a3a266..7eb3e9199b 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -9,13 +9,17 @@ use alloc::{ use core::{cmp::Reverse, num::NonZeroUsize, ops::Deref}; use crate::{ - context::memory::AddrSpaceWrapper, + context::{ + memory::AddrSpaceWrapper, + switch::{BASE_SLICE_TICKS, NANOS_PER_TICK, SCALE, SCHED_PRIO_TO_WEIGHT, TICK_INTERVAL}, + }, cpu_set::LogicalCpuSet, + ipi::{ipi, IpiKind, IpiTarget}, memory::{RmmA, RmmArch, TableKind}, percpu::PercpuBlock, sync::{ ArcRwLockWriteGuard, CleanLockToken, LockToken, Mutex, MutexGuard, RwLock, RwLockReadGuard, - RwLockWriteGuard, L0, L1, L2, L4, + RwLockWriteGuard, L0, L1, L2, L3, L4, }, syscall::error::Result, }; @@ -135,6 +139,32 @@ pub fn run_contexts_try(token: LockToken<'_, L0>) -> Option, token: &mut LockToken<'_, L3>) -> bool { + let was_blocked = { + let mut guard = context_lock.write(token.token()); + if !guard.unblock_no_ipi() { + return false; + } + + true + }; + + let percpu: &PercpuBlock = PercpuBlock::current(); + + let weak = WeakContextRef(Arc::downgrade(context_lock)); + let cpu_id = context_lock.write(token.token()).cpu_id; + + percpu.switch_internals.wakeup_list.borrow_mut().push(weak); + + if let Some(cpu_id) = cpu_id + && cpu_id != crate::cpu_id() + { + ipi(IpiKind::Wakeup, IpiTarget::Other); + } + + true +} + pub fn init(token: &mut CleanLockToken) { let owner = None; // kmain not owned by any fd let mut context = Context::new(owner).expect("failed to create kmain context"); diff --git a/src/context/switch.rs b/src/context/switch.rs index 2996a86871..915921a29f 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -13,7 +13,7 @@ use crate::{ percpu::PercpuBlock, sync::{ArcRwLockWriteGuard, CleanLockToken, L4}, }; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use core::{ cell::{Cell, RefCell}, cmp::Reverse, @@ -32,16 +32,16 @@ enum UpdateResult { } // A simple geometric series where value[i] ~= value[i + 1] * 1.25 -const SCHED_PRIO_TO_WEIGHT: [usize; 40] = [ +pub const SCHED_PRIO_TO_WEIGHT: [usize; 40] = [ 88761, 71755, 56483, 46273, 36291, 29154, 23254, 18705, 14949, 11916, 9548, 7620, 6100, 4904, 3906, 3121, 2501, 1991, 1586, 1277, 1024, 820, 655, 526, 423, 335, 272, 215, 172, 137, 110, 87, 70, 56, 45, 36, 29, 23, 18, 15, ]; -const SCALE: u128 = 1 << 40; -const TICK_INTERVAL: u64 = 3; // Approx 6.75 ms -const BASE_SLICE_TICKS: u64 = TICK_INTERVAL * 3; // Approx 20.25 ms -const NANOS_PER_TICK: u128 = 2_250_000; // 2.25 ms +pub const SCALE: u128 = 1 << 40; +pub const TICK_INTERVAL: u64 = 3; // Approx 6.75 ms +pub const BASE_SLICE_TICKS: u64 = TICK_INTERVAL * 3; // Approx 20.25 ms +pub const NANOS_PER_TICK: u128 = 2_250_000; // 2.25 ms /// Determines if a given context is eligible to be scheduled on a given CPU (in /// principle, the current CPU). @@ -190,6 +190,8 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { let mut wakeups = wakeup_contexts(token); let mut push_idle: SmallVec<[WeakContextRef; 16]> = SmallVec::new(); + // These timers coukd have expired + let mut timers: SmallVec<[(u128, WeakContextRef); 16]> = SmallVec::new(); if let Some(mut run_contexts) = run_contexts_try(token.token()) { // Pop Timers while let Some((wake, _)) = run_contexts.timers.first() { @@ -197,12 +199,31 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { break; } - if let Some((_, context_ref)) = run_contexts.timers.pop_first() { - wakeups.push(context_ref); + if let Some(entry) = run_contexts.timers.pop_first() { + timers.push(entry); } } } + for (wake, context_ref) in timers { + let Some(context_lock) = context_ref.upgrade() else { + continue; + }; + + let guard = context_lock.read(token.token()); + if guard.status.is_soft_blocked() && guard.wake == Some(wake) { + wakeups.push(context_ref); + } + } + + // Drain from percpu + let mut percpu_wake = percpu.switch_internals.wakeup_list.replace(Vec::new()); + if percpu_wake.len() > 0 { + for context_ref in percpu_wake.iter() { + wakeups.push(context_ref.clone()); + } + } + if wakeups.len() > 0 { let mut run_contexts = run_contexts(token.token()); for context_ref in wakeups { @@ -697,6 +718,9 @@ pub struct ContextSwitchPercpu { /// The idle process. idle_ctxt: RefCell>>, pub(crate) being_sigkilled: Cell, + + // wakeups + pub(crate) wakeup_list: RefCell>, } impl ContextSwitchPercpu { @@ -708,6 +732,7 @@ impl ContextSwitchPercpu { current_ctxt: RefCell::new(None), idle_ctxt: RefCell::new(None), being_sigkilled: Cell::new(false), + wakeup_list: RefCell::new(Vec::new()), #[cfg(feature = "profiling")] current_dbg_id: core::sync::atomic::AtomicU32::new(!0), diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index e2bb2541ac..c9c8fea3f9 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -4,7 +4,7 @@ use crate::{ context::{HardBlockedReason, LockedFdTbl, SignalState}, file::InternalFlags, memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan, UnmapVec}, - Context, ContextLock, Status, + unblock_context, Context, ContextLock, Status, }, memory::{Page, VirtualAddress, PAGE_SIZE}, ptrace, @@ -1248,8 +1248,7 @@ impl ContextHandle { Ok(size_of::()) } ContextVerb::Interrupt => { - let mut guard = context.write(token.token()); - guard.unblock(); + unblock_context(&context, &mut token.token().downgrade()); Ok(size_of::()) } ContextVerb::ForceKill => { @@ -1278,13 +1277,16 @@ impl ContextHandle { } crate::syscall::exit_this_context(None, token); } else { - let mut ctxt = context.write(token.token()); - //trace!("FORCEKILL NONSELF={} {}, SELF={}", ctxt.debug_id, ctxt.pid, context::current().read().debug_id); - if ctxt.status.is_dead() { - return Ok(size_of::()); + { + let mut ctxt = context.write(token.token()); + //trace!("FORCEKILL NONSELF={} {}, SELF={}", ctxt.debug_id, ctxt.pid, context::current().read().debug_id); + if ctxt.status.is_dead() { + return Ok(size_of::()); + } + ctxt.status = context::Status::Runnable; + ctxt.being_sigkilled = true; } - ctxt.status = context::Status::Runnable; - ctxt.being_sigkilled = true; + unblock_context(&context, &mut token.token().downgrade()); Ok(size_of::()) } } diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 2141f83d2c..0eb0afd8f0 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -22,7 +22,7 @@ use crate::{ handle_notify_files, AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant, GrantFileRef, MmapMode, PageSpan, UnmapVec, DANGLING, }, - BorrowedHtBuf, ContextLock, PreemptGuard, PreemptGuardL1, Status, + unblock_context, BorrowedHtBuf, ContextLock, PreemptGuard, PreemptGuardL1, Status, }, event, memory::{Frame, Page, VirtualAddress, PAGE_SIZE}, @@ -956,7 +956,7 @@ impl UserInner { match context.upgrade() { Some(context) => { *o = State::Responded(response); - context.write(lock_token.token()).unblock(); + unblock_context(&context, &mut lock_token.token().downgrade()); } _ => { states.remove(tag as usize); diff --git a/src/sync/wait_condition.rs b/src/sync/wait_condition.rs index e521c64189..989b06fdda 100644 --- a/src/sync/wait_condition.rs +++ b/src/sync/wait_condition.rs @@ -6,7 +6,7 @@ use alloc::{ }; use crate::{ - context::{self, ContextLock, PreemptGuardL2}, + context::{self, unblock_context, ContextLock, PreemptGuardL2}, sync::{CleanLockToken, LockToken, Mutex, L1, L2, L3}, }; @@ -33,7 +33,7 @@ impl WaitCondition { let len = contexts.len(); while let Some(context_weak) = contexts.pop() { if let Some(context_ref) = context_weak.upgrade() { - context_ref.write(token.token()).unblock(); + unblock_context(&context_ref, &mut token.token()); } } len @@ -46,7 +46,7 @@ impl WaitCondition { let len = contexts.len(); for context_weak in contexts.iter() { if let Some(context_ref) = context_weak.upgrade() { - context_ref.write(token.token()).unblock(); + unblock_context(&context_ref, &mut token.token()); } } len diff --git a/src/syscall/futex.rs b/src/syscall/futex.rs index fe8a4bf0bd..8fb09d508c 100644 --- a/src/syscall/futex.rs +++ b/src/syscall/futex.rs @@ -15,7 +15,7 @@ use crate::{ context::{ self, memory::{AddrSpace, AddrSpaceWrapper}, - ContextLock, + unblock_context, ContextLock, }, memory::{Page, PhysicalAddress, VirtualAddress}, sync::{CleanLockToken, Mutex, L1}, @@ -214,7 +214,7 @@ pub fn futex( continue; } if let Some(ctx) = futex.context_lock.upgrade() { - ctx.write(token.token()).unblock(); + unblock_context(&ctx, &mut token.token().downgrade()); } futexes.swap_remove(i); woken += 1;