diff --git a/src/context/context.rs b/src/context/context.rs index 0be56c7508..9e663274d9 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -13,6 +13,7 @@ use crate::{ context::{ self, arch, file::{FileDescriptor, LockedFileDescription}, + run_contexts_mut, }, cpu_set::{LogicalCpuId, LogicalCpuSet}, cpu_stats, @@ -137,6 +138,10 @@ pub struct Context { pub userspace: bool, pub being_sigkilled: bool, pub fmap_ret: Option, + /// Priority + pub prio: usize, + /// Enqueued + pub enqueued: bool, // TODO: id can reappear after wraparound? pub owner_proc_id: Option, @@ -194,6 +199,8 @@ impl Context { files: Arc::new(RwLock::new(FdTbl::new())), userspace: false, fmap_ret: None, + prio: 20, + enqueued: false, being_sigkilled: false, owner_proc_id, diff --git a/src/context/mod.rs b/src/context/mod.rs index 62a0a575b5..dd389bdf4b 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -2,7 +2,11 @@ //! //! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and [osdev](https://wiki.osdev.org/Context_Switching) -use alloc::{collections::BTreeSet, sync::Arc}; +use alloc::{ + collections::{BTreeSet, VecDeque}, + string::String, + sync::Arc, +}; use core::num::NonZeroUsize; use crate::{ @@ -71,6 +75,22 @@ pub use self::arch::empty_cr3; // the context file descriptors. static CONTEXTS: RwLock> = RwLock::new(BTreeSet::new()); +// Actual context store for the scheduler +static RUN_CONTEXTS: RwLock = RwLock::new(RunContextData::new()); + +pub struct RunContextData { + set: [VecDeque; 40], +} + +impl RunContextData { + pub const fn new() -> Self { + const EMPTY_VEC: VecDeque = VecDeque::new(); + Self { + set: [EMPTY_VEC; 40], + } + } +} + /// Get the global schemes list, const pub fn contexts(token: LockToken<'_, L0>) -> RwLockReadGuard<'_, L1, BTreeSet> { CONTEXTS.read(token) @@ -81,6 +101,14 @@ pub fn contexts_mut(token: LockToken<'_, L0>) -> RwLockWriteGuard<'_, L1, BTreeS CONTEXTS.write(token) } +pub fn run_contexts(token: LockToken<'_, L0>) -> RwLockReadGuard<'_, L1, RunContextData> { + RUN_CONTEXTS.read(token) +} + +pub fn run_contexts_mut(token: LockToken<'_, L0>) -> RwLockWriteGuard<'_, L1, RunContextData> { + RUN_CONTEXTS.write(token) +} + pub fn init(token: &mut CleanLockToken) { let owner = None; // kmain not owned by any fd let mut context = Context::new(owner).expect("failed to create kmain context"); @@ -95,6 +123,7 @@ pub fn init(token: &mut CleanLockToken) { context.status = Status::Runnable; context.running = true; context.cpu_id = Some(crate::cpu_id()); + context.enqueued = false; let context_lock = Arc::new(ContextLock::new(context)); @@ -110,6 +139,52 @@ pub fn init(token: &mut CleanLockToken) { } } +pub fn wakeup_context(context_lock: &Arc>) { + let mut global_token = unsafe { CleanLockToken::new() }; + let mut local_token = unsafe { CleanLockToken::new() }; + + let mut run_queues = run_contexts_mut(global_token.token()); + let mut context = context_lock.write(local_token.token()); + + context.wake = None; + + /* + if context.status.is_soft_blocked() { + context.status = Status::Runnable; + context.status_reason = ""; + + if !context.enqueued { + let prio = context.prio; + run_queues.set[prio].push_back(ContextRef(Arc::clone(context_lock))); + context.enqueued = true; + } + } + */ + + context.unblock(); + + if context.status.is_runnable() && !context.running && !context.enqueued { + let prio = context.prio; + run_queues.set[prio].push_back(ContextRef(Arc::clone(context_lock))); + context.enqueued = true; + } +} + +pub fn set_priority( + context_lock: &Arc>, + new_prio: usize, + local_token: &mut CleanLockToken, +) -> Result<(), String> { + if new_prio >= 40 { + return Err("Priority out of bounds".into()); + } + + let mut guard = context_lock.write(local_token.token()); + guard.prio = new_prio; + + Ok(()) +} + pub fn current() -> Arc { PercpuBlock::current() .switch_internals @@ -174,6 +249,10 @@ pub fn spawn( contexts_mut(token.token()).insert(context_ref); + let run_ref = ContextRef(Arc::clone(&context_lock)); + run_contexts_mut(token.token()).set[20].push_back(run_ref); + context_lock.write(token.token()).enqueued = true; + Ok(context_lock) } diff --git a/src/context/switch.rs b/src/context/switch.rs index c3815bf200..6a614520c8 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -8,13 +8,16 @@ use core::{ sync::atomic::Ordering, }; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use syscall::PtraceFlags; use crate::{ - context::{arch, contexts, ArcContextLockWriteGuard, Context, ContextLock}, + context::{ + self, arch, contexts, run_contexts_mut, ArcContextLockWriteGuard, Context, ContextLock, + Status, + }, cpu_set::LogicalCpuId, - cpu_stats, + cpu_stats, log, percpu::PercpuBlock, sync::CleanLockToken, }; @@ -26,6 +29,13 @@ enum UpdateResult { Skip, } +// A simple geometric series where value[i] ~= value[i - 1] * 1.25 +const sched_prio_to_weight: [usize; 40] = [ + 88761, 71755, 56483, 46273, 36291, 29154, 23254, 18705, 14949, 11916, 9548, 7620, 6100, 4904, + 3906, 3121, 2501, 1991, 1586, 1277, 1024, 820, 655, 526, 423, 335, 272, 215, 172, 137, 110, 87, + 70, 56, 45, 36, 29, 23, 18, 15, +]; + /// Determines if a given context is eligible to be scheduled on a given CPU (in /// principle, the current CPU). /// @@ -54,15 +64,6 @@ unsafe fn update_runnable( return UpdateResult::Skip; } - // If context is soft-blocked and has a wake-up time, check if it should wake up. - if context.status.is_soft_blocked() - && let Some(wake) = context.wake - && switch_time >= wake - { - context.wake = None; - context.unblock_no_ipi(); - } - // If the context is runnable, indicate it can be switched to. if context.status.is_runnable() { UpdateResult::CanSwitch @@ -124,11 +125,8 @@ pub enum SwitchResult { AllContextsIdle, } -/// Selects and switches to the next context using a round-robin scheduler. -/// -/// This function performs the context switch, checking each context in a loop for eligibility -/// until it finds a context ready to run. If no other context is runnable, it returns to the -/// idle context. +/// This function performs the context switch, using select_next_context to +/// actually select the next context to switch to. /// /// # Warning /// This is not memory-unsafe to call. But do NOT call this while holding locks! @@ -157,73 +155,42 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { percpu.maybe_handle_tlb_shootdown(); } - let cpu_id = crate::cpu_id(); - - let mut switch_context_opt = None; + // Alarm (previously in update_runnable) + // TODO: Optimise this somehow + let mut wakeups = Vec::new(); { - let contexts = contexts(token.token()); - - // Lock the previous context. - let prev_context_lock = crate::context::current(); - // We are careful not to lock this context twice - let prev_context_guard = unsafe { prev_context_lock.write_arc() }; - - if !prev_context_guard.is_preemptable() { - // Unset global lock - arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst); - - // Pretend to have finished switching, so CPU is not idled - return SwitchResult::Switched; - } - - let idle_context = percpu.switch_internals.idle_context(); - - // Stateful flag used to skip the idle process the first time it shows up. - // After that, this flag is set to `false` so the idle process can be - // picked up. - let mut skip_idle = true; - - // Attempt to locate the next context to switch to. - for next_context_lock in contexts - // Include all contexts with IDs greater than the current... - .range(( - Bound::Excluded(ContextRef(Arc::clone(&prev_context_lock))), - Bound::Unbounded, - )) - // ... and all contexts with IDs less than the current... - .chain(contexts.range(( - Bound::Unbounded, - Bound::Excluded(ContextRef(Arc::clone(&prev_context_lock))), - ))) - .filter_map(ContextRef::upgrade) - // ... and the idle context... - .chain(Some(Arc::clone(&idle_context))) - // ... but not the current context (note the `Bound::Excluded`), - // which is already locked. - { - if Arc::ptr_eq(&next_context_lock, &idle_context) && skip_idle { - // Skip idle process the first time it shows up, but allow it - // to be picked up again the next time. - skip_idle = false; + let current_context = context::current(); + let contexts_guard = contexts(token.token()); + for context_ref in contexts_guard.iter().filter_map(|r| r.upgrade()) { + if Arc::ptr_eq(&context_ref, ¤t_context) { continue; } - - { - // Lock next context - // We are careful not to lock this context twice - let mut next_context_guard = unsafe { next_context_lock.write_arc() }; - - // Check if the context is runnable and can be switched to. - if let UpdateResult::CanSwitch = - unsafe { update_runnable(&mut next_context_guard, cpu_id, switch_time) } - { - // Store locks for previous and next context and break out from loop - // for the switch - switch_context_opt = Some((prev_context_guard, next_context_guard)); - break; + let mut local_token = unsafe { CleanLockToken::new() }; + let guard = context_ref.read(local_token.token()); + if guard.status.is_soft_blocked() { + if let Some(wake) = guard.wake { + if switch_time >= wake { + wakeups.push(Arc::clone(&context_ref)); + continue; + } } } + + if guard.status.is_runnable() && !guard.enqueued && !guard.running { + wakeups.push(Arc::clone(&context_ref)); + } } + } + + for context_lock in wakeups { + context::wakeup_context(&context_lock); + } + + let cpu_id = crate::cpu_id(); + + let switch_context_opt = match select_next_context(token, percpu, cpu_id, switch_time) { + Ok(opt) => opt, + Err(early_ret) => return early_ret, }; // Update per-cpu times @@ -297,8 +264,8 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { prev_context.syscall_debug_info = percpu .syscall_debug_info .replace(next_context.syscall_debug_info); - prev_context.syscall_debug_info.on_switch_from(token); - next_context.syscall_debug_info.on_switch_to(token); + prev_context.syscall_debug_info.on_switch_from(); + next_context.syscall_debug_info.on_switch_to(); } percpu @@ -328,6 +295,133 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { } } +/// This is the scheduler function which currently utilises Deficit Weighted Round Robin Scheduler +fn select_next_context( + token: &mut CleanLockToken, + percpu: &PercpuBlock, + cpu_id: LogicalCpuId, + switch_time: u128, +) -> Result, SwitchResult> { + let mut contexts_data = run_contexts_mut(token.token()); + let mut contexts_list = &mut contexts_data.set; + let mut balance = percpu.balance.get(); + let mut i = percpu.last_queue.get() % 40; + + // Lock the previous context. + let prev_context_lock = crate::context::current(); + // We are careful not to lock this context twice + let mut prev_context_guard = unsafe { prev_context_lock.write_arc() }; + + // If we cannot even preempt the prev context, no need to go any further + if !prev_context_guard.is_preemptable() { + // Unset global lock + arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst); + + // Pretend to have finished switching, so CPU is not idled + return Err(SwitchResult::Switched); + } + + let idle_context = percpu.switch_internals.idle_context(); + + let mut empty_queues = 0; + let mut total_iters = 0; + let mut next_context_guard_opt = None; + + 'priority: loop { + i = (i + 1) % 40; + total_iters += 1; + + // The least prioritised queue takes <5000 iters to build up + // balance = sched_prio_to_weight[20], if we have already spent + // that many iters and not found any context, it is better to just + // skip for now + if total_iters >= 5000 { + break 'priority; + } + let contexts = contexts_list + .get_mut(i) + .expect("i should be between [0, 39]!"); + + if contexts.is_empty() { + balance[i] = 0; // We do not allow a queue to build up its balance when nobody is using it + empty_queues += 1; + if empty_queues >= 40 { + // If all queues are empty, just break out + break 'priority; + } + continue; + } else { + empty_queues = 0; + } + + if balance[i] < sched_prio_to_weight[20] { + // This queue does not have enough balance to run, + // increment the balance! + balance[i] += sched_prio_to_weight[i]; + continue; + } + + let len = contexts.len(); + for _ in 0..len { + let next_context_lock = match contexts.pop_front() { + Some(lock) => match lock.upgrade() { + Some(new_lock) => new_lock, + None => continue, // Ghost Process, just continue + }, + None => break, // Empty Queue + }; + + let mut next_context_guard = unsafe { next_context_lock.write_arc() }; + next_context_guard.enqueued = false; + + if !next_context_guard.status.is_runnable() { + continue; // Lazy removal of blocked contexts + } + + // Is this context runnable on this CPU? + if let UpdateResult::CanSwitch = + unsafe { update_runnable(&mut next_context_guard, cpu_id, switch_time) } + { + next_context_guard_opt = Some(next_context_guard); + balance[i] -= sched_prio_to_weight[20]; + break 'priority; + } else { + contexts.push_back(ContextRef(Arc::clone(&next_context_lock))); + next_context_guard.enqueued = true; + } + } + } + percpu.balance.set(balance); + percpu.last_queue.set(i); + + if let Some(next_context_guard) = next_context_guard_opt { + // We found a new process! + // Send the old process to the back of the line (if it is still runnable) + if prev_context_guard.status.is_runnable() + && !Arc::ptr_eq(&prev_context_lock, &idle_context) + { + let prio = prev_context_guard.prio; + contexts_list[prio].push_back(ContextRef(Arc::clone(&prev_context_lock))); + prev_context_guard.enqueued = true; + } + + return Ok(Some((prev_context_guard, next_context_guard))); + } else { + // We found no other process to run. + if prev_context_guard.status.is_runnable() + && !Arc::ptr_eq(&prev_context_lock, &idle_context) + { + arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst); + return Err(SwitchResult::Switched); + } else if Arc::ptr_eq(&prev_context_lock, &idle_context) { + return Ok(None); + } else { + let idle_guard = unsafe { idle_context.write_arc() }; + return Ok(Some((prev_context_guard, idle_guard))); + } + } +} + /// Holds per-CPU state necessary for context switching. /// /// This struct contains information such as the idle context, current context, and PIT tick counts, diff --git a/src/percpu.rs b/src/percpu.rs index fa6adf64ba..dbf944497a 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -12,10 +12,11 @@ use syscall::PtraceFlags; use crate::{ arch::device::ArchPercpuMisc, - context::{empty_cr3, memory::AddrSpaceWrapper, switch::ContextSwitchPercpu}, + context::{empty_cr3, memory::AddrSpaceWrapper, switch::ContextSwitchPercpu, Context}, cpu_set::{LogicalCpuId, MAX_CPU_COUNT}, cpu_stats::{CpuStats, CpuStatsData}, ptrace::Session, + sync::{RwLock, L4}, syscall::debug::SyscallDebugInfo, }; @@ -30,6 +31,8 @@ pub struct PercpuBlock { pub current_addrsp: RefCell>>, pub new_addrsp_tmp: Cell>>, pub wants_tlb_shootdown: AtomicBool, + pub balance: Cell<[usize; 40]>, + pub last_queue: Cell, // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it // first to avoid cache invalidation. @@ -184,6 +187,8 @@ impl PercpuBlock { current_addrsp: RefCell::new(None), new_addrsp_tmp: Cell::new(None), wants_tlb_shootdown: AtomicBool::new(false), + balance: Cell::new([0; 40]), + last_queue: Cell::new(39), ptrace_flags: Cell::new(PtraceFlags::empty()), ptrace_session: RefCell::new(None), inside_syscall: Cell::new(false), diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 082e052e1e..91afd3dc2b 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -1187,8 +1187,7 @@ impl ContextHandle { Ok(size_of::()) } ContextVerb::Interrupt => { - let mut guard = context.write(token.token()); - guard.unblock(); + context::wakeup_context(&context); Ok(size_of::()) } ContextVerb::ForceKill => { diff --git a/src/scheme/user.rs b/src/scheme/user.rs index b80b456bcf..3b60a9ae19 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -951,7 +951,8 @@ impl UserInner { match context.upgrade() { Some(context) => { *o = State::Responded(response); - context.write(token.token()).unblock(); + // context.write(token.token()).unblock(); + context::wakeup_context(&context); } _ => { states.remove(tag as usize); diff --git a/src/sync/wait_condition.rs b/src/sync/wait_condition.rs index bda7a8cabc..b882be9454 100644 --- a/src/sync/wait_condition.rs +++ b/src/sync/wait_condition.rs @@ -33,7 +33,8 @@ impl WaitCondition { let len = contexts.len(); while let Some(context_weak) = contexts.pop() { if let Some(context_ref) = context_weak.upgrade() { - context_ref.write(token.token()).unblock(); + // context_ref.write(token.token()).unblock(); + context::wakeup_context(&context_ref); } } len @@ -46,7 +47,8 @@ impl WaitCondition { let len = contexts.len(); for context_weak in contexts.iter() { if let Some(context_ref) = context_weak.upgrade() { - context_ref.write(token.token()).unblock(); + // context_ref.write(token.token()).unblock(); + context::wakeup_context(&context_ref); } } len diff --git a/src/syscall/futex.rs b/src/syscall/futex.rs index 2a0312dd86..40498178c4 100644 --- a/src/syscall/futex.rs +++ b/src/syscall/futex.rs @@ -200,7 +200,8 @@ pub fn futex( i += 1; continue; } - futex.context_lock.write(token.token()).unblock(); + // futex.context_lock.write(token.token()).unblock(); + context::wakeup_context(&futex.context_lock); futexes.swap_remove(i); woken += 1; } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 137e4e8d99..ba81962c0b 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -239,6 +239,9 @@ pub fn syscall( SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d)).map(|()| 0), SYS_MREMAP => mremap(b, c, d, e, f, token), + SYS_SETPRIORITY => unsafe { process::setpriority(b, c, d) }, + SYS_GETPRIORITY => process::getpriority(b, c), + _ => Err(Error::new(ENOSYS)), } } diff --git a/src/syscall/process.rs b/src/syscall/process.rs index fa907da17d..fc8bcf0efd 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -7,9 +7,10 @@ use syscall::data::GlobalSchemes; use crate::{ context::{ context::SyscallFrame, + contexts, file::{FileDescription, FileDescriptor, InternalFlags}, memory::{AddrSpace, Grant, PageSpan}, - ContextRef, + ContextLock, ContextRef, }, event, sync::{CleanLockToken, RwLock}, @@ -274,3 +275,73 @@ fn insert_fd(scheme: SchemeId, number: usize, cloexec: bool, token: &mut CleanLo .expect("failed to insert fd to current context") .get() } + +pub unsafe fn setpriority(which: usize, who: usize, prio: usize) -> Result { + if prio >= 40 { + return Err(Error::new(EINVAL)); + } + + if which != 0 { + return Err(Error::new(EINVAL)); // TODO: Support for PRIO_PGRP and PRIO_USER + } + + let mut token = unsafe { CleanLockToken::new() }; + let mut local_token = unsafe { CleanLockToken::new() }; + + let context_lock = { + if who == 0 { + context::current() + } else { + let contexts_guard = contexts(token.token()); + + let found = contexts_guard.iter().filter_map(|r| r.upgrade()).find(|c| { + let guard = c.read(local_token.token()); + guard.pid == who + }); + + match found { + Some(c) => c, + None => return Err(Error::new(ESRCH)), + } + } + }; + + { + let current_lock = context::current(); + let current_euid = context::current().read(local_token.token()).euid; + let target_euid = context_lock.read(local_token.token()).euid; + + if current_euid != 0 && current_euid != target_euid { + return Err(Error::new(EPERM)); + } + } + + match context::set_priority(&context_lock, prio, &mut local_token) { + Ok(_) => Ok(0), + Err(_) => Err(Error::new(ESRCH)), + } +} + +pub fn getpriority(which: usize, who: usize) -> Result { + let mut token = unsafe { CleanLockToken::new() }; + let mut local_token = unsafe { CleanLockToken::new() }; + + let context_lock = { + if who == 0 { + context::current() + } else { + let contexts_guard = contexts(token.token()); + + let found = contexts_guard.iter().filter_map(|r| r.upgrade()).find(|c| { + let guard = c.read(local_token.token()); + guard.pid == who + }); + + match found { + Some(c) => c, + None => return Err(Error::new(ESRCH)), + } + } + }; + Ok(context_lock.read(local_token.token()).prio) +}