diff --git a/rmm/src/lib.rs b/rmm/src/lib.rs index 3088ec989e..3be8f35e6e 100644 --- a/rmm/src/lib.rs +++ b/rmm/src/lib.rs @@ -1,6 +1,8 @@ #![no_std] #![allow(clippy::new_without_default)] +use core::ops::Add; + pub use crate::{allocator::*, arch::*, page::*}; mod allocator; diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index 0ff841214b..fd1e88b948 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -1,29 +1,34 @@ //! # ACPI //! Code to parse the ACPI tables -use alloc::{boxed::Box, string::String, vec::Vec}; - use core::ptr::NonNull; +use alloc::{boxed::Box, string::String, vec::Vec}; + use hashbrown::HashMap; use spin::{Once, RwLock}; -use crate::memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch}; +use crate::{ + acpi::rxsdt::RxsdtIter, + memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch}, +}; use self::{hpet::Hpet, madt::Madt, rsdp::Rsdp, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sdt, xsdt::Xsdt}; #[cfg(target_arch = "aarch64")] mod gtdt; -pub mod fadt; -pub mod facs; pub mod hpet; pub mod madt; mod rsdp; mod rsdt; mod rxsdt; pub mod sdt; +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] +pub mod slit; #[cfg(target_arch = "aarch64")] mod spcr; +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] +pub mod srat; mod xsdt; unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::memory::PageMapper) { @@ -76,24 +81,20 @@ pub enum RxsdtEnum { Xsdt(Xsdt), } impl Rxsdt for RxsdtEnum { - fn iter(&self) -> Box> { + fn iter(&self) -> RxsdtIter { match self { - Self::Rsdt(rsdt) => ::iter(rsdt), - Self::Xsdt(xsdt) => ::iter(xsdt), + Self::Rsdt(rsdt) => rsdt.iter(), + Self::Xsdt(xsdt) => xsdt.iter(), } } } pub static RXSDT_ENUM: Once = Once::new(); -/// Parse the ACPI tables to gather CPU, interrupt, and timer information -pub unsafe fn init(already_supplied_rsdp: Option>) { +/// Initialses the global `RXSDT_ENUM` if RSDT or XSDT was found and maps the SDT pages. +/// It does not perform any allocations +pub unsafe fn init_before_mem(already_supplied_rsdp: Option>) { unsafe { - { - let mut sdt_ptrs = SDT_POINTERS.write(); - *sdt_ptrs = Some(HashMap::new()); - } - // Search for RSDP let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp); @@ -138,6 +139,22 @@ pub unsafe fn init(already_supplied_rsdp: Option>) { for sdt in rxsdt.iter() { get_sdt(sdt, &mut KernelMapper::lock_rw()); } + } else { + error!("NO RSDP FOUND"); + return; + } + } +} + +/// Parse the ACPI tables to gather CPU, interrupt, and timer information. The code performs allocations, so +/// it must be called only after the allocator is set up. +pub unsafe fn init_after_mem(already_supplied_rsdp: Option>) { + if let Some(rxsdt) = RXSDT_ENUM.get() { + unsafe { + { + let mut sdt_ptrs = SDT_POINTERS.write(); + *sdt_ptrs = Some(HashMap::new()); + } for sdt_address in rxsdt.iter() { let sdt = &*(RmmA::phys_to_virt(sdt_address).data() as *const Sdt); @@ -160,44 +177,6 @@ pub unsafe fn init(already_supplied_rsdp: Option>) { Hpet::init(); #[cfg(target_arch = "aarch64")] gtdt::Gtdt::init(); - // Phase II: parse the FADT to extract the PM1a_CNT - // and PM1a_STS port addresses used by the S3 entry - // path. Hardware-agnostic — works on any platform - // with a working FADT. - if let Some(fadt_sdts) = find_sdt("FACP").first() { - fadt::init(fadt_sdts); - } else { - warn!("ACPI: no FADT (FACP) found, S3 entry path disabled"); - } - // Phase II.X.W: parse the FACS to extract the - // xfirmware_waking_vector. This is the address the - // platform firmware jumps to on S3 wake. The kernel's - // S3 resume trampoline in arch/x86_shared/s3_resume.rs - // is written to this address by acpid via the - // SetS3WakingVector AcPiVerb. - // - // The FACS is found via the FADT's x_firmware_ctrl - // field (64-bit) or firmware_ctrl field (32-bit). - // The FADT parser caches the FACS address. We map the - // FACS here because it is not listed in the RSDT/XSDT - // and therefore was not mapped during table discovery. - const FACS_MIN_SIZE: usize = 64; - let mut facs_addr = fadt::x_firmware_ctrl(); - if facs_addr == 0 { - facs_addr = fadt::firmware_ctrl() as u64; - } - if facs_addr != 0 { - let facs_phys = PhysicalAddress::new(facs_addr as usize); - map_linearly(facs_phys, FACS_MIN_SIZE, &mut KernelMapper::lock_rw()); - let facs_sdt = unsafe { - &*(RmmA::phys_to_virt(facs_phys).data() as *const Sdt) - }; - facs::init(facs_sdt); - } else { - warn!("ACPI: no FACS found (neither x_firmware_ctrl nor firmware_ctrl), S3 resume path disabled"); - } - } else { - error!("NO RSDP FOUND"); } } } diff --git a/src/acpi/rsdt.rs b/src/acpi/rsdt.rs index e5e7d54428..fc95e8d141 100644 --- a/src/acpi/rsdt.rs +++ b/src/acpi/rsdt.rs @@ -2,6 +2,8 @@ use alloc::boxed::Box; use core::convert::TryFrom; use rmm::PhysicalAddress; +use crate::acpi::{rxsdt::RxsdtIter, RxsdtEnum}; + use super::{rxsdt::Rxsdt, sdt::Sdt}; #[derive(Debug)] @@ -24,29 +26,11 @@ impl Rsdt { } impl Rxsdt for Rsdt { - fn iter(&self) -> Box> { - Box::new(RsdtIter { sdt: self.0, i: 0 }) - } -} - -pub struct RsdtIter { - sdt: &'static Sdt, - i: usize, -} - -impl Iterator for RsdtIter { - type Item = PhysicalAddress; - fn next(&mut self) -> Option { - if self.i < self.sdt.data_len() / size_of::() { - let item = unsafe { - (self.sdt.data_address() as *const u32) - .add(self.i) - .read_unaligned() - }; - self.i += 1; - Some(PhysicalAddress::new(item as usize)) - } else { - None + fn iter(&self) -> RxsdtIter { + RxsdtIter { + sdt: self.0, + i: 0, + rxsdt_enum: RxsdtEnum::Rsdt(Rsdt(self.0)), } } } diff --git a/src/acpi/rxsdt.rs b/src/acpi/rxsdt.rs index a4bc6c967f..f3cc49b7ea 100644 --- a/src/acpi/rxsdt.rs +++ b/src/acpi/rxsdt.rs @@ -1,6 +1,36 @@ use alloc::boxed::Box; use rmm::PhysicalAddress; +use crate::acpi::{sdt::Sdt, RxsdtEnum}; + pub trait Rxsdt { - fn iter(&self) -> Box>; + fn iter(&self) -> RxsdtIter; +} + +pub struct RxsdtIter { + pub sdt: &'static Sdt, + pub i: usize, + pub rxsdt_enum: RxsdtEnum, +} + +impl Iterator for RxsdtIter { + type Item = PhysicalAddress; + fn next(&mut self) -> Option { + if self.i < self.sdt.data_len() / size_of::() { + let item = unsafe { + match self.rxsdt_enum { + RxsdtEnum::Rsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned( + (self.sdt.data_address() as *const u32).add(self.i), + ) as usize), + RxsdtEnum::Xsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned( + (self.sdt.data_address() as *const u64).add(self.i), + ) as usize), + } + }; + self.i += 1; + Some(item) + } else { + None + } + } } diff --git a/src/acpi/srat/aarch64.rs b/src/acpi/srat/aarch64.rs new file mode 100644 index 0000000000..090a9ff41e --- /dev/null +++ b/src/acpi/srat/aarch64.rs @@ -0,0 +1,89 @@ +use core::{ops::Add, slice, u32}; + +use rmm::{Arch, BumpAllocator, FrameAllocator, FrameCount, PhysicalAddress}; + +use crate::{ + acpi::srat::{to_usize, Srat, SratEntry}, + cpu_set::MAX_CPU_COUNT, + memory::{round_up_pages, PAGE_SIZE}, + numa::{self, assign_memory_id, assign_node_id, NumaMemory}, +}; + +pub fn init_srat(dom_node_map: &mut [u32], cpus: &mut [u32], mem: &mut [NumaMemory], srat: &Srat) { + let mut cpu_count = 0; + let mut memory_count = 0; + + srat.into_iter().for_each(|e| match e { + SratEntry::GiccAffinity(gicc_affinity) => { + if gicc_affinity.flags & 1 != 0 { + cpu_count += 1 + } + } + SratEntry::MemoryAffinity(memory_affinity) => { + if memory_affinity.flags & 1 != 0 && memory_affinity.flags & (1 << 1) == 0 { + memory_count += 1 + } + } + _ => (), + }); + + assert!( + memory_count <= numa::MAX_DOMAINS, + "Found {} memory blocks while only a maximum of {} are supported", + memory_count, + numa::MAX_DOMAINS + ); + + assert!( + cpu_count <= MAX_CPU_COUNT, + "Found more number of CPUs than supported" + ); + + for affinity in srat { + match affinity { + SratEntry::MemoryAffinity(memory_affinity) => { + let start = to_usize( + memory_affinity.base_address_low, + memory_affinity.base_address_high, + ); + let length = to_usize(memory_affinity.length_low, memory_affinity.length_high); + if length == 0 { + continue; + } + if memory_affinity.flags & 1 == 0 { + // memory disabled + continue; + } + if memory_affinity.flags & (1 << 1) != 0 { + // memory hot-pluggable + continue; + } + if dom_node_map[memory_affinity.proximity_domain as usize] == u32::MAX { + let node = assign_node_id(true); + dom_node_map[memory_affinity.proximity_domain as usize] = node as u32; + } + let mem_id = assign_memory_id() as u32; + mem[mem_id as usize] = NumaMemory { + start, + length, + node_id: dom_node_map[memory_affinity.proximity_domain as usize], + _pad: [0u8; 4], + }; + } + SratEntry::GiccAffinity(gicc_affinity) => { + if gicc_affinity.flags & 1 == 0 { + // disabled + continue; + } + let id = gicc_affinity.processor_uid; + let dom = gicc_affinity.proximity_domain; + if dom_node_map[dom as usize] == u32::MAX { + let node = assign_node_id(true); + dom_node_map[dom as usize] = node as u32; + } + cpus[id as usize] = dom_node_map[dom as usize]; + } + _ => continue, + } + } +} diff --git a/src/acpi/xsdt.rs b/src/acpi/xsdt.rs index bb59065e52..4f0f1e16ec 100644 --- a/src/acpi/xsdt.rs +++ b/src/acpi/xsdt.rs @@ -2,6 +2,8 @@ use alloc::boxed::Box; use core::convert::TryFrom; use rmm::PhysicalAddress; +use crate::acpi::{rxsdt::RxsdtIter, RxsdtEnum}; + use super::{rxsdt::Rxsdt, sdt::Sdt}; #[derive(Debug)] @@ -24,27 +26,11 @@ impl Xsdt { } impl Rxsdt for Xsdt { - fn iter(&self) -> Box> { - Box::new(XsdtIter { sdt: self.0, i: 0 }) - } -} - -pub struct XsdtIter { - sdt: &'static Sdt, - i: usize, -} - -impl Iterator for XsdtIter { - type Item = PhysicalAddress; - fn next(&mut self) -> Option { - if self.i < self.sdt.data_len() / size_of::() { - let item = unsafe { - core::ptr::read_unaligned((self.sdt.data_address() as *const u64).add(self.i)) - }; - self.i += 1; - Some(PhysicalAddress::new(item as usize)) - } else { - None + fn iter(&self) -> RxsdtIter { + RxsdtIter { + sdt: self.0, + i: 0, + rxsdt_enum: RxsdtEnum::Xsdt(Xsdt(self.0)), } } } diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index 7a7c0ae815..295c273501 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -12,6 +12,8 @@ use crate::{ startup::KernelArgs, }; +use crate::numa; + /// Test of zero values in BSS. static BSS_TEST_ZERO: SyncUnsafeCell = SyncUnsafeCell::new(0); /// Test of non-zero values in data. @@ -102,13 +104,22 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { // Initialize RMM #[cfg(target_arch = "x86")] - crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000)); + let mut bump_allocator = + crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000)); #[cfg(target_arch = "x86_64")] - crate::startup::memory::init(&args, Some(0x100000), None); + let mut bump_allocator = crate::startup::memory::init(&args, Some(0x100000), None); // Initialize paging paging::init(); + if cfg!(feature = "acpi") { + crate::acpi::init_before_mem(args.acpi_rsdp()); + } + + numa::init(&mut bump_allocator); + + crate::memory::init_mm(bump_allocator); + #[cfg(target_arch = "x86_64")] crate::arch::alternative::early_init(true); @@ -130,7 +141,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { // Read ACPI tables, starts APs if cfg!(feature = "acpi") { - crate::acpi::init(args.acpi_rsdp()); + crate::acpi::init_after_mem(args.acpi_rsdp()); device::init_after_acpi(); } crate::profiling::init(); diff --git a/src/context/context.rs b/src/context/context.rs index 0211abe87a..6ca468d8f4 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -314,6 +314,7 @@ impl Context { pub fn unblock_no_ipi(&mut self) -> bool { if self.status.is_soft_blocked() { self.status = Status::Runnable; + self.wake = None; self.status_reason = ""; true diff --git a/src/context/mod.rs b/src/context/mod.rs index 7e125f0e2a..1518dc3367 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -3,10 +3,10 @@ //! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and [osdev](https://wiki.osdev.org/Context_Switching) use alloc::{ - collections::{BTreeSet, VecDeque}, + collections::{BTreeMap, BTreeSet, VecDeque}, sync::{Arc, Weak}, }; -use core::{num::NonZeroUsize, ops::Deref}; +use core::{cmp::Reverse, num::NonZeroUsize, ops::Deref}; use crate::{ context::memory::AddrSpaceWrapper, @@ -81,16 +81,30 @@ static RUN_CONTEXTS: Mutex = Mutex::new(RunContextData::new( static IDLE_CONTEXTS: Mutex> = Mutex::new(VecDeque::new()); pub struct RunContextData { - set: [VecDeque; 40], + // queue: VecDeque, + queue: BTreeMap<(u64, Reverse, u32), (u64, u64, WeakContextRef)>, // ((vd, rem_slice, ctxt_id), (vtime, weight, context)) + timers: BTreeSet<(u128, WeakContextRef)>, // (wake, context) + count: usize, + v: u64, + total_weight: u64, + min_vtime: u64, } impl RunContextData { pub const fn new() -> Self { - const EMPTY_VEC: VecDeque = VecDeque::new(); Self { - set: [EMPTY_VEC; 40], + queue: BTreeMap::new(), + timers: BTreeSet::new(), + count: 0, + v: 0, + total_weight: 0, + min_vtime: 0, } } + pub fn update_count(&mut self) -> usize { + self.count = self.queue.len(); + self.count + } } /// Get the global schemes list, const @@ -117,6 +131,10 @@ pub fn run_contexts(token: LockToken<'_, L0>) -> MutexGuard<'_, L1, RunContextDa RUN_CONTEXTS.lock(token) } +pub fn run_contexts_try(token: LockToken<'_, L0>) -> Option> { + RUN_CONTEXTS.try_lock(token) +} + pub fn init(token: &mut CleanLockToken) { let owner = None; // kmain not owned by any fd let mut context = Context::new(owner).expect("failed to create kmain context"); @@ -126,12 +144,18 @@ pub fn init(token: &mut CleanLockToken) { context.name.clear(); context.name.push_str("[kmain]"); + #[cfg(feature = "profiling")] + { + crate::profiling::DBG_ID_MAP + .write(token.token()) + .insert(context.debug_id, context.name); + } + self::arch::EMPTY_CR3.call_once(|| RmmA::table(TableKind::User)); context.status = Status::Runnable; context.running = true; context.cpu_id = Some(crate::cpu_id()); - context.start_time = crate::time::monotonic(token); let context_lock = Arc::new(ContextLock::new(context)); @@ -224,9 +248,6 @@ pub fn spawn( let stack = Kstack::new()?; let mut context = Context::new(owner_proc_id)?; - context.start_time = crate::time::monotonic(token); - context.pgrp = context.pid; - context.session = context.pid; let _ = context.set_addr_space(Some(AddrSpaceWrapper::new()?), token.downgrade()); context @@ -235,6 +256,7 @@ pub fn spawn( context.kstack = Some(stack); context.userspace = userspace_allowed; + context.queue_key = Some((context.vd, Reverse(context.rem_slice), context.debug_id)); let context_lock = Arc::new(ContextLock::new(context)); let context_ref = ContextRef(Arc::clone(&context_lock)); @@ -326,3 +348,11 @@ impl Drop for PreemptGuardL2<'_> { self.context.write(self.token.token()).preempt_locks -= 1; } } + +pub fn get_contexts_stats(token: &mut CleanLockToken) -> (usize, usize, usize) { + let alive = contexts(token.downgrade()).len(); + let running = run_contexts(token.token()).count; + let blocked = idle_contexts(token.downgrade()).len(); + + (alive, running, blocked) +} diff --git a/src/context/switch.rs b/src/context/switch.rs index 39fa424237..5de01b01ef 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -4,20 +4,25 @@ use crate::{ context::{ - self, arch, idle_contexts, idle_contexts_try, run_contexts, ArcContextLockWriteGuard, - Context, ContextLock, WeakContextRef, + self, arch, idle_contexts, idle_contexts_try, memory::AddrSpaceSwitchReadGuard, + run_contexts, run_contexts_try, ArcContextLockWriteGuard, Context, ContextLock, + WeakContextRef, }, cpu_set::LogicalCpuId, cpu_stats::{self, CpuState}, percpu::PercpuBlock, sync::{ArcRwLockWriteGuard, CleanLockToken, L4}, }; -use alloc::{sync::Arc, vec::Vec}; +use alloc::sync::Arc; use core::{ cell::{Cell, RefCell}, - hint, mem, + cmp::Reverse, + hint, matches, mem, + option::Option::{None, Some}, sync::atomic::Ordering, + u64, }; +use smallvec::SmallVec; use syscall::PtraceFlags; enum UpdateResult { @@ -26,13 +31,18 @@ enum UpdateResult { Blocked, } -// A simple geometric series where value[i] ~= value[i - 1] * 1.25 +// A simple geometric series where value[i] ~= value[i + 1] * 1.25 const SCHED_PRIO_TO_WEIGHT: [usize; 40] = [ 88761, 71755, 56483, 46273, 36291, 29154, 23254, 18705, 14949, 11916, 9548, 7620, 6100, 4904, 3906, 3121, 2501, 1991, 1586, 1277, 1024, 820, 655, 526, 423, 335, 272, 215, 172, 137, 110, 87, 70, 56, 45, 36, 29, 23, 18, 15, ]; +const SCALE: u128 = 1 << 40; +const TICK_INTERVAL: u64 = 3; // Approx 6.75 ms +const BASE_SLICE_TICKS: u64 = TICK_INTERVAL * 3; // Approx 20.25 ms +const NANOS_PER_TICK: u128 = 2_250_000; // 2.25 ms + /// Determines if a given context is eligible to be scheduled on a given CPU (in /// principle, the current CPU). /// @@ -96,7 +106,9 @@ pub fn tick(token: &mut CleanLockToken) { ticks_cell.set(new_ticks); // Trigger a context switch after every 3 ticks (approx. 6.75 ms). - if new_ticks >= 3 { + if new_ticks >= TICK_INTERVAL as usize + && arch::CONTEXT_SWITCH_LOCK.load(Ordering::Relaxed) == false + { switch(token); crate::context::signal::signal_handler(token); } @@ -175,15 +187,80 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { } // Alarm (previously in update_runnable) - let wakeups = wakeup_contexts(token, switch_time); + let mut wakeups = wakeup_contexts(token); + let mut push_idle: SmallVec<[WeakContextRef; 16]> = SmallVec::new(); + + if let Some(mut run_contexts) = run_contexts_try(token.token()) { + // Pop Timers + while let Some((wake, _)) = run_contexts.timers.first() { + if *wake > switch_time { + break; + } + + if let Some((_, context_ref)) = run_contexts.timers.pop_first() { + wakeups.push(context_ref); + } + } + } if wakeups.len() > 0 { let mut run_contexts = run_contexts(token.token()); - for (prio, context_lock) in wakeups { - run_contexts.set[prio].push_back(context_lock); + for context_ref in wakeups { + let Some(context_lock) = context_ref.upgrade() else { + continue; + }; + + let Some(mut guard) = (unsafe { context_lock.try_write_arc() }) else { + push_idle.push(context_ref); + continue; + }; + + let new_vtime = guard.vtime.max(run_contexts.v); + guard.vtime = new_vtime; + + let weight = SCHED_PRIO_TO_WEIGHT[guard.prio] as u64; + let scaled_slice = (BASE_SLICE_TICKS as u128 * SCALE) / weight as u128; + + if !guard.is_active { + guard.is_active = true; + run_contexts.total_weight += weight; + } + + guard.vd = new_vtime + scaled_slice as u64; + guard.rem_slice = BASE_SLICE_TICKS * SCALE as u64; + let key = (guard.vd, Reverse(guard.rem_slice), guard.debug_id); + guard.queue_key = Some(key); + drop(guard); + + run_contexts + .queue + .insert(key, (new_vtime, weight, context_ref)); } } + { + let mut idle_list = idle_contexts(token.downgrade()); + for context_ref in push_idle { + idle_list.push_back(context_ref); + } + } + + /* // uncomment to debug contexts count + let cpu_count = crate::cpu_count() as usize; + let len_idle = idle_contexts(token.downgrade()).len(); + let all_contexts = context::contexts(token.downgrade()) + .len() + .saturating_sub(cpu_count); // ignore kmain + print!( + "\r TIME {}.{} IDLE {} WAKEUPS {} ALL {} ", + switch_time / 1000_000_000, + (switch_time / 100_000_000) % 10, + len_idle, + wakeups_len, + all_contexts + ); + */ + let cpu_id = crate::cpu_id(); // Update per-cpu times @@ -192,21 +269,19 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { let was_idle = percpu.stats.add_time(percpu_ms) == CpuState::Idle as u8; percpu.switch_internals.switch_time.set(switch_time); - let switch_context_opt = match select_next_context( + let switch_context_opt = select_next_context( token, percpu, cpu_id, switch_time, + percpu_nanos, was_idle, &mut prev_context_guard, - ) { - Ok(opt) => opt, - Err(early_ret) => return early_ret, - }; + ); // Switch process states, TSS stack pointer, and store new context ID match switch_context_opt { - Some(mut next_context_guard) => { + Some((mut next_context_guard, addr_space_guard)) => { // Update context states and prepare for the switch. let prev_context = &mut *prev_context_guard; let next_context = &mut *next_context_guard; @@ -221,13 +296,7 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { // Update times if !was_idle { - let delta = switch_time.saturating_sub(prev_context.switch_time); - prev_context.cpu_time += delta; - if prev_context.userspace { - prev_context.utime += delta; - } else { - prev_context.stime += delta; - } + prev_context.cpu_time += switch_time.saturating_sub(prev_context.switch_time); } next_context.switch_time = switch_time; if next_context.userspace { @@ -272,6 +341,14 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { prev_context.inside_syscall = percpu.inside_syscall.replace(next_context.inside_syscall); + #[cfg(feature = "profiling")] + { + percpu + .switch_internals + .current_dbg_id + .store(next_context.debug_id, Ordering::Relaxed); + } + #[cfg(feature = "syscall_debug")] { prev_context.syscall_debug_info = percpu @@ -286,7 +363,11 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { .being_sigkilled .set(next_context.being_sigkilled); + // Anything implement Drop must be manually dropped now + drop(prev_context_lock); + unsafe { + percpu.new_addrsp_guard.set(addr_space_guard); arch::switch_to(prev_context, next_context); } @@ -308,9 +389,11 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { } } -fn wakeup_contexts(token: &mut CleanLockToken, switch_time: u128) -> Vec<(usize, WeakContextRef)> { - // TODO: Optimise this somehow. Perhaps using a separate timer queue? - let mut wakeups = Vec::new(); +fn wakeup_contexts( + token: &mut CleanLockToken, +) -> SmallVec<[WeakContextRef; 16]> { + // TODO: Optimise this somehow + let mut wakeups = SmallVec::new(); let current_context = context::current(); let Some(idle_contexts) = idle_contexts_try(token.downgrade()) else { // other cpus may spawning or killing contexts so let's skip wakeups to avoid contention @@ -333,21 +416,14 @@ fn wakeup_contexts(token: &mut CleanLockToken, switch_time: u128) -> Vec<(usize, idle_contexts.push_back(context_ref); continue; }; - if guard.status.is_soft_blocked() { - if let Some(wake) = guard.wake { - if switch_time >= wake { - let prio = guard.prio; - drop(guard); - wakeups.push((prio, context_ref)); - continue; - } - } + if guard.status.is_dead() { + // TODO: who hold this dead context? + continue; } if guard.status.is_runnable() && !guard.running { - let prio = guard.prio; drop(guard); - wakeups.push((prio, context_ref)); + wakeups.push(context_ref); continue; } @@ -357,137 +433,250 @@ fn wakeup_contexts(token: &mut CleanLockToken, switch_time: u128) -> Vec<(usize, wakeups } -/// This is the scheduler function which currently utilises Deficit Weighted Round Robin Scheduler +/// This is the scheduler function which currently utilises EEVDF Scheduler fn select_next_context( token: &mut CleanLockToken, percpu: &PercpuBlock, cpu_id: LogicalCpuId, switch_time: u128, + elapsed_time: u64, was_idle: bool, prev_context_guard: &mut ArcRwLockWriteGuard, -) -> Result, SwitchResult> { +) -> Option<(ArcContextLockWriteGuard, Option)> { let contexts_data = run_contexts(token.token()); let (mut contexts_data, mut token) = contexts_data.into_split(); - let contexts_list = &mut contexts_data.set; let idle_context = percpu.switch_internals.idle_context(); - let mut balance = percpu.balance.get(); - let mut i = percpu.last_queue.get() % 40; // Lock the previous context. let prev_context_lock = crate::context::current(); + let is_idle = Arc::ptr_eq(&prev_context_lock, &idle_context); + let prev_runnable = !is_idle && prev_context_guard.status.is_runnable(); + let is_timer = prev_context_guard.wake.is_some(); - let mut empty_queues = 0; - let mut total_iters = 0; - let mut next_context_guard_opt = None; + let elapsed_ticks = elapsed_time as u128 * SCALE / NANOS_PER_TICK; - let total_contexts: usize = contexts_list.iter().map(|q| q.len()).sum(); - let mut skipped_contexts = 0; + if prev_runnable { + let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64; + prev_context_guard.rem_slice = prev_context_guard + .rem_slice + .saturating_sub((elapsed_ticks) as u64); + let scaled_task = elapsed_ticks / weight as u128; + prev_context_guard.vtime += scaled_task as u64; - 'priority: loop { - i = (i + 1) % 40; - total_iters += 1; - - // The least prioritised queue takes <5000 iters to build up - // balance = sched_prio_to_weight[20], if we have already spent - // that many iters and not found any context, it is better to just - // skip for now - if total_iters >= 5000 { - break 'priority; + if prev_context_guard.vtime < contexts_data.v { + prev_context_guard.vtime = contexts_data.v; } - if skipped_contexts > total_contexts && total_contexts > 0 { - break 'priority; + let is_yield = (elapsed_time as u128) < (TICK_INTERVAL as u128 * NANOS_PER_TICK) / 2; + + if is_yield { + let unconsumed = prev_context_guard.rem_slice as u128; + let penalty = unconsumed / weight as u128; + prev_context_guard.vtime += penalty as u64; + prev_context_guard.rem_slice = 0; } - let contexts = contexts_list - .get_mut(i) - .expect("i should be between [0, 39]!"); + if prev_context_guard.rem_slice == 0 { + prev_context_guard.rem_slice = BASE_SLICE_TICKS * SCALE as u64; + let scaled_slice = (BASE_SLICE_TICKS as u128 * SCALE) / weight as u128; + prev_context_guard.vd = prev_context_guard.vtime + scaled_slice as u64; + } + } else if !is_idle { + if prev_context_guard.is_active { + prev_context_guard.is_active = false; + let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64; + contexts_data.total_weight = contexts_data.total_weight.saturating_sub(weight); + } + prev_context_guard.rem_slice = 0; - if contexts.is_empty() { - empty_queues += 1; - if empty_queues >= 40 { - // If all queues are empty, just break out - break 'priority; - } - continue; + if let Some(wake) = prev_context_guard.wake { + contexts_data + .timers + .insert((wake, WeakContextRef(Arc::downgrade(&prev_context_lock)))); + } + } + + let mut eligible_best = None; + let mut prev_is_eligible = false; + + let mut ineligible_best = None; + let mut ineligible_min_vtime = u64::MAX; + let mut ineligible_vd = u64::MAX; + + if prev_runnable { + if prev_context_guard.vtime <= contexts_data.v { + prev_is_eligible = true; } else { - empty_queues = 0; + ineligible_min_vtime = prev_context_guard.vtime; + ineligible_vd = prev_context_guard.vd; } + } - if balance[i] < SCHED_PRIO_TO_WEIGHT[20] { - // This queue does not have enough balance to run, - // increment the balance! - balance[i] += SCHED_PRIO_TO_WEIGHT[i]; + // New BTreeMap based walk + let mut weight_change: u64 = 0; + let mut contexts_to_remove: SmallVec<[(u64, Reverse, u32); 16]> = SmallVec::new(); + for ((vd, rem_slice, ctxt_id), (vtime, context_weight, context_ref)) in + contexts_data.queue.iter() + { + if *vtime > ineligible_min_vtime && *vtime > contexts_data.v { continue; } - let len = contexts.len(); - for _ in 0..len { - let (next_context_ref, next_context_lock) = match contexts.pop_front() { - Some(lock) => match lock.upgrade() { - Some(new_lock) => (lock, new_lock), - None => { - skipped_contexts += 1; - continue; // Ghost Process, just continue - } - }, - None => break, // Empty Queue - }; + let Some(context_lock) = context_ref.upgrade() else { + weight_change += *context_weight as u64; + contexts_to_remove.push((*vd, *rem_slice, *ctxt_id)); + continue; + }; - if Arc::ptr_eq(&next_context_lock, &prev_context_lock) { - contexts.push_back(next_context_ref); - continue; - } - if Arc::ptr_eq(&next_context_lock, &idle_context) { - contexts.push_back(next_context_ref); - continue; - } - let mut next_context_guard = unsafe { next_context_lock.write_arc() }; + if Arc::ptr_eq(&context_lock, &idle_context) + || Arc::ptr_eq(&context_lock, &prev_context_lock) + { + //weight_change += *context_weight as u64; + //contexts_to_remove.push((*vd, *rem_slice, *ctxt_id)); + continue; + } - // Is this context runnable on this CPU? - let sw = unsafe { update_runnable(&mut next_context_guard, cpu_id, switch_time) }; - if let UpdateResult::CanSwitch = sw { - next_context_guard_opt = Some(next_context_guard); - balance[i] -= SCHED_PRIO_TO_WEIGHT[20]; - break 'priority; + let Some(mut guard) = (unsafe { context_lock.try_write_arc() }) else { + continue; + }; + + let sw = unsafe { update_runnable(&mut guard, cpu_id, switch_time) }; + + if matches!(sw, UpdateResult::Blocked) { + if guard.is_active { + guard.is_active = false; + weight_change += context_weight; + } + guard.rem_slice = 0; + guard.queue_key = None; + + contexts_to_remove.push((*vd, *rem_slice, *ctxt_id)); + drop(guard); + // Reenqueue should be handled by unblock + idle_contexts(token.token()).push_back(context_ref.clone()); + continue; + } + + if !matches!(sw, UpdateResult::CanSwitch) { + continue; + } + + let mut best_addr_space = None; + if let Some(addr_space) = &guard.addr_space { + let mut t = unsafe { CleanLockToken::new() }; + if let Some(addr) = addr_space.inner.try_read(t.token()) { + best_addr_space = Some(AddrSpaceSwitchReadGuard::new(addr)); } else { - if matches!(sw, UpdateResult::Blocked) { - idle_contexts(token.token()).push_back(next_context_ref); - } else { - contexts.push_back(next_context_ref); - }; - skipped_contexts += 1; + continue; + } + } - if skipped_contexts >= total_contexts { - break 'priority; + if *vtime <= contexts_data.v { + // Eligible + eligible_best = Some((guard, best_addr_space)); + break; + } else { + // Ineligible + if *vtime < ineligible_min_vtime { + ineligible_min_vtime = *vtime; + ineligible_vd = *vd; + if let Some((old_guard, old_addr_space)) = ineligible_best { + drop(old_guard); + drop(old_addr_space); } + ineligible_best = Some((guard, best_addr_space)); } } } - percpu.balance.set(balance); - percpu.last_queue.set(i); - if !Arc::ptr_eq(&prev_context_lock, &idle_context) { - // Send the old process to the back of the line (if it is still runnable) - let prev_ctx = WeakContextRef(Arc::downgrade(&prev_context_lock)); - if prev_context_guard.status.is_runnable() { - let prio = prev_context_guard.prio; - contexts_list[prio].push_back(prev_ctx); - } else { - idle_contexts(token.token()).push_back(prev_ctx); + contexts_data.total_weight = contexts_data.total_weight.saturating_sub(weight_change); + + for old_key in contexts_to_remove { + contexts_data.queue.remove(&old_key); + } + + // No eligible context was found + if !(prev_is_eligible || eligible_best.is_some()) && ineligible_min_vtime != u64::MAX { + contexts_data.v = ineligible_min_vtime; // Advance V + + let prev_is_earliest = prev_runnable && prev_context_guard.vtime <= ineligible_min_vtime; + + if prev_is_earliest { + eligible_best = None; + } else if ineligible_best.is_some() { + let prev_has_slice = prev_runnable && prev_context_guard.rem_slice > 0; + + if prev_has_slice && prev_context_guard.vd <= ineligible_vd { + eligible_best = None; + } else { + eligible_best = ineligible_best.take(); + } + } + } else if prev_is_eligible && eligible_best.is_some() { + if let Some((ref guard, _)) = eligible_best { + if prev_context_guard.vd < guard.vd + || (prev_context_guard.vd == guard.vd + && prev_context_guard.rem_slice > guard.rem_slice) + { + eligible_best = None; + } } } - if let Some(next_context_guard) = next_context_guard_opt { - // We found a new process! - return Ok(Some(next_context_guard)); - } else { - if !was_idle && !Arc::ptr_eq(&prev_context_lock, &idle_context) { - // We switch into the idle context - Ok(Some(unsafe { idle_context.write_arc() })) + let mut final_winner = None; + + if let Some((mut chosen_guard, addr_space)) = eligible_best { + if let Some(key) = chosen_guard.queue_key.take() { + contexts_data.queue.remove(&key); + } + final_winner = Some((chosen_guard, addr_space)); + } + + if final_winner.is_some() || prev_runnable { + if contexts_data.total_weight > 0 { + let v_advance = elapsed_ticks as u128 / contexts_data.total_weight as u128; + contexts_data.v += v_advance as u64; + } + + if let Some((chosen_guard, addr_space)) = final_winner { + if prev_runnable { + let (vd, rem_slice, ctxt_id, vtime) = ( + prev_context_guard.vd, + prev_context_guard.rem_slice, + prev_context_guard.debug_id, + prev_context_guard.vtime, + ); + prev_context_guard.queue_key = Some((vd, Reverse(rem_slice), ctxt_id)); + + let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64; + contexts_data.queue.insert( + (vd, Reverse(rem_slice), ctxt_id), + ( + vtime, + weight, + WeakContextRef(Arc::downgrade(&prev_context_lock)), + ), + ); + } else if !is_idle && !is_timer { + idle_contexts(token.token()) + .push_back(WeakContextRef(Arc::downgrade(&prev_context_lock))); + } + + return Some((chosen_guard, addr_space)); } else { - // We found no other process to run. - Ok(None) + return None; + } + } else { + if !is_idle && !is_timer { + idle_contexts(token.token()) + .push_back(WeakContextRef(Arc::downgrade(&prev_context_lock))); + } + + let prev_is_dead = !is_idle && !prev_context_guard.status.is_runnable(); + if (!was_idle || prev_is_dead) && !is_idle { + return Some(unsafe { (idle_context.write_arc(), None) }); + } else { + return None; } } } @@ -503,6 +692,10 @@ pub struct ContextSwitchPercpu { current_ctxt: RefCell>>, + // TODO: just access current_ctxt directly? + #[cfg(feature = "profiling")] + pub(crate) current_dbg_id: core::sync::atomic::AtomicU32, + /// The idle process. idle_ctxt: RefCell>>, pub(crate) being_sigkilled: Cell, @@ -517,6 +710,9 @@ impl ContextSwitchPercpu { current_ctxt: RefCell::new(None), idle_ctxt: RefCell::new(None), being_sigkilled: Cell::new(false), + + #[cfg(feature = "profiling")] + current_dbg_id: core::sync::atomic::AtomicU32::new(!0), } } diff --git a/src/main.rs b/src/main.rs index 32f491d0e8..8380e02f94 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,10 +3,8 @@ //! The Redox OS Kernel is a microkernel that supports `x86_64` systems and //! provides Unix-like syscalls for primarily Rust applications -#![feature(asm_cfg)] // Stabilized in 1.93 -#![feature(if_let_guard)] #![feature(int_roundings)] -#![feature(iter_next_chunk)] +#![cfg_attr(dtb, feature(iter_next_chunk))] #![feature(sync_unsafe_cell)] #![feature(btree_cursors)] #![cfg_attr(not(test), no_std)] @@ -70,6 +68,9 @@ mod log; /// Memory management mod memory; +/// NUMA support +mod numa; + /// Panic mod panic; diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 3afe964005..c6f4e99216 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -4,11 +4,13 @@ use core::{ cell::SyncUnsafeCell, num::NonZeroUsize, - sync::atomic::{AtomicUsize, Ordering}, + ops::AddAssign, + slice, + sync::atomic::{AtomicU8, AtomicUsize, Ordering}, }; pub use kernel_mapper::KernelMapper; -use spin::Mutex; +use spin::{once::Once, Mutex}; pub use crate::arch::CurrentRmmArch as RmmA; use crate::{ @@ -17,11 +19,12 @@ use crate::{ memory::{AccessMode, PfError}, }, kernel_executable_offsets::{__usercopy_end, __usercopy_start}, + numa, sync::CleanLockToken, syscall::error::{Error, ENOMEM}, }; pub use rmm::{Arch as RmmArch, PageFlags, PhysicalAddress, TableKind, VirtualAddress}; -use rmm::{BumpAllocator, FrameAllocator, FrameCount, FrameUsage}; +use rmm::{BumpAllocator, FrameAllocator, FrameCount, FrameUsage, MemoryArea}; mod kernel_mapper; pub mod page; @@ -54,7 +57,15 @@ pub fn free_frames() -> usize { /// Get the number of frames used pub fn used_frames() -> usize { // TODO: Include bump allocator static pages? - FREELIST.lock().used_frames + FREE_LISTS + .get() + .unwrap() + .iter() + .map(|e| { + let l = e.lock(); + l.used_frames + }) + .sum() } pub fn total_frames() -> usize { // TODO: Include bump allocator static pages? @@ -63,19 +74,39 @@ pub fn total_frames() -> usize { /// Allocate a range of frames pub fn allocate_p2frame(order: u32) -> Option { - allocate_p2frame_complex(order, (), None, order).map(|(f, _)| f) + let initial_index = get_round_robin_index(); + let mut index = initial_index; + + loop { + if let Some(frame) = allocate_p2frame_complex(order, (), None, order, index).map(|(f, _)| f) + { + return Some(frame); + } + index = get_round_robin_index(); + if index == initial_index { + return None; + } + } } pub fn allocate_frame() -> Option { allocate_p2frame(0) } + +fn get_round_robin_index() -> usize { + static CURRENT_INDEX: AtomicUsize = AtomicUsize::new(0); + let len = FREE_LISTS.get().unwrap().len(); + CURRENT_INDEX.fetch_add(1, Ordering::Relaxed) % len +} + // TODO: Flags, strategy pub fn allocate_p2frame_complex( _req_order: u32, _flags: (), _strategy: Option<()>, min_order: u32, + index: usize, ) -> Option<(Frame, usize)> { - let mut freelist = FREELIST.lock(); + let mut freelist = FREE_LISTS.get().unwrap()[index].lock(); let (frame_order, frame) = freelist .for_orders @@ -112,7 +143,9 @@ pub fn allocate_p2frame_complex( freelist.for_orders[frame_order as usize] = next_free.frame(); // TODO: Is this LIFO cache optimal? - //info!("MIN{min_order}FRAMEORD{frame_order}"); + // if min_order > 0 { + // info!("MIN {min_order} FRAMEORD {frame_order}"); + // } for order in (min_order..frame_order).rev() { //info!("SPLIT ORDER {order}"); let order_page_count = 1 << order; @@ -146,8 +179,19 @@ pub fn allocate_p2frame_complex( Some((frame, PAGE_SIZE << min_order)) } +fn get_index_for_deallocation(addr: usize) -> Option { + for (i, list) in FREE_LISTS.get().unwrap().iter().enumerate() { + let l = list.lock(); + if addr >= l.lower_limit && addr < l.upper_limit { + return Some(i); + } + } + None +} + pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) { - let mut freelist = FREELIST.lock(); + let index = get_index_for_deallocation(orig_frame.physaddr.get()).expect("Expected an index"); + let mut freelist = FREE_LISTS.get().unwrap()[index].lock(); let initial_info = get_page_info(orig_frame) .unwrap_or_else(|| panic!("missing PageInfo for {orig_frame:?} being freed")); @@ -231,7 +275,9 @@ pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) { old_head_info.set_prev(P2Frame::new(Some(new_head), largest_order)); } - //info!("FREED {frame:?}+2^{order}"); + // if order > 0 { + // info!("FREED {current:?}+2^{order}"); + // } freelist.used_frames -= 1 << order; } @@ -476,13 +522,12 @@ struct AllocatorData { } #[derive(Debug)] struct FreeList { + upper_limit: usize, + lower_limit: usize, for_orders: [Option; ORDER_COUNT as usize], used_frames: usize, } -static FREELIST: Mutex = Mutex::new(FreeList { - for_orders: [None; ORDER_COUNT as usize], - used_frames: 0, -}); +static FREE_LISTS: Once<&'static [Mutex]> = Once::new(); pub struct Section { base: Frame, @@ -499,6 +544,8 @@ const _: () = { #[cold] fn init_sections(mut allocator: BumpAllocator) { + let number_of_memory_regions = numa::number_of_memory_regions(); + let (free_areas, offset_into_first_free_area) = allocator.free_areas(); let free_areas_iter = || { @@ -524,6 +571,18 @@ fn init_sections(mut allocator: BumpAllocator) { .next_multiple_of(MAX_SECTION_SIZE); let aligned_start = area.base.data() / MAX_SECTION_SIZE * MAX_SECTION_SIZE; + if number_of_memory_regions > 0 { + if let Some(next_memory_region) = + numa::nearest_next_memory_region(area.base.data(), false) + && next_memory_region.start < area.base.add(area.size).data() + && next_memory_region.start + next_memory_region.length + > area.base.add(area.size).data() + && !next_memory_region.start.is_multiple_of(MAX_SECTION_SIZE) + { + return (aligned_end - aligned_start) / MAX_SECTION_SIZE + 1; + } + } + (aligned_end - aligned_start) / MAX_SECTION_SIZE }) .sum(); @@ -543,76 +602,141 @@ fn init_sections(mut allocator: BumpAllocator) { let mut iter = free_areas_iter().peekable(); - let mut i = 0; + let mut sections_fill = |region: Option, + i: &mut usize, // out parameter + force: bool| + -> Option { + let mut iter = free_areas_iter().peekable(); - while let Some(mut memory_map_area) = iter.next() { - // TODO: NonZeroUsize - - // TODO: x86_32 fails without this check - if memory_map_area.size == 0 { - continue; - } - - assert_ne!( - memory_map_area.size, 0, - "RMM should enforce areas are not zeroed" - ); - - // TODO: Should RMM do this? - - while let Some(next_area) = iter.peek() - && next_area.base == memory_map_area.base.add(memory_map_area.size) - { - memory_map_area.size += next_area.size; - let _ = iter.next(); - } - - assert_eq!( - memory_map_area.base.data() % PAGE_SIZE, - 0, - "RMM should enforce area alignment" - ); - assert_eq!( - memory_map_area.size % PAGE_SIZE, - 0, - "RMM should enforce area length alignment" - ); - - let mut pages_left = memory_map_area.size.div_floor(PAGE_SIZE); - let mut base = Frame::containing(memory_map_area.base); - - while pages_left > 0 { - let page_info_max_count = core::cmp::min(pages_left, MAX_SECTION_PAGE_COUNT); - let pages_to_next_section = - (MAX_SECTION_SIZE - (base.base().data() % MAX_SECTION_SIZE)) / PAGE_SIZE; - let page_info_count = core::cmp::min(page_info_max_count, pages_to_next_section); - - let page_info_array_size_pages = - (page_info_count * size_of::()).div_ceil(PAGE_SIZE); - let page_info_array = unsafe { - let base = allocator - .allocate(FrameCount::new(page_info_array_size_pages)) - .expect("failed to allocate page info array"); - core::slice::from_raw_parts_mut( - RmmA::phys_to_virt(base).data() as *mut PageInfo, - page_info_count, - ) - }; - for p in &*page_info_array { - assert_eq!(p.next.load(Ordering::Relaxed), 0); - assert_eq!(p.refcount.load(Ordering::Relaxed), 0); + while let Some(mut memory_map_area) = iter.next() { + if !{ + if let Some(region) = region + && !force + { + memory_map_area.base >= region.base + && memory_map_area.base.data() < region.base.data() + region.size + } else { + true + } + } { + continue; } - sections[i] = Section { - base, - frames: page_info_array, - }; - i += 1; + // TODO: NonZeroUsize - pages_left -= page_info_count; - base = base.next_by(page_info_count); + // TODO: x86_32 fails without this check + if memory_map_area.size == 0 { + continue; + } + + assert_ne!( + memory_map_area.size, 0, + "RMM should enforce areas are not of length 0" + ); + + // TODO: Should RMM do this? + + while let Some(next_area) = iter.peek() + && next_area.base == memory_map_area.base.add(memory_map_area.size) + && if let Some(region) = region { + next_area.base >= region.base + && next_area.base.data() < region.base.data() + region.size + && !force + } else { + true + } + { + memory_map_area.size += next_area.size; + let _ = iter.next(); + } + + assert_eq!( + memory_map_area.base.data() % PAGE_SIZE, + 0, + "RMM should enforce area alignment" + ); + assert_eq!( + memory_map_area.size % PAGE_SIZE, + 0, + "RMM should enforce area length alignment" + ); + + let mut pages_left = memory_map_area.size.div_floor(PAGE_SIZE); + let mut base = Frame::containing(memory_map_area.base); + let mut return_value = None; + + while pages_left > 0 { + let page_info_max_count = core::cmp::min(pages_left, MAX_SECTION_PAGE_COUNT); + let pages_to_next_section = + (MAX_SECTION_SIZE - (base.base().data() % MAX_SECTION_SIZE)) / PAGE_SIZE; + let mut page_info_count = + core::cmp::min(page_info_max_count, pages_to_next_section); + + if !force + && let Some(region) = region + && base.base() >= region.base + && base.base().data() < region.base.data() + region.size + && (page_info_count * PAGE_SIZE + > (region.base.data() + region.size - base.base().data())) + { + page_info_count = + (region.base.data() + region.size - base.base().data()) / PAGE_SIZE; + return_value = Some(MemoryArea { + base: PhysicalAddress::new(region.base.data() + region.size), + size: (base.physaddr.get() + pages_left * PAGE_SIZE) + - (region.base.data() + region.size), + }); + } + + let page_info_array_size_pages = + (page_info_count * size_of::()).div_ceil(PAGE_SIZE); + let page_info_array = unsafe { + let base = allocator + .allocate(FrameCount::new(page_info_array_size_pages)) + .expect("failed to allocate page info array"); + core::slice::from_raw_parts_mut( + RmmA::phys_to_virt(base).data() as *mut PageInfo, + page_info_count, + ) + }; + for p in &*page_info_array { + assert_eq!(p.next.load(Ordering::Relaxed), 0); + assert_eq!(p.refcount.load(Ordering::Relaxed), 0); + } + + sections[*i] = Section { + base, + frames: page_info_array, + }; + i.add_assign(1); + + pages_left -= page_info_count; + base = base.next_by(page_info_count); + if let Some(return_value) = return_value { + return Some(return_value); + } + } } + return None; + }; + let mut i = 0; + + if number_of_memory_regions > 0 + && let Some(regions) = numa::memory_regions() + { + for region in regions { + let mut sections_fill_result = Some(region); + let mut force = false; + + while let Some(mut region) = sections_fill_result { + sections_fill_result = sections_fill(Some(region), &mut i, force); + force = true; + } + } + } else { + sections_fill(None, &mut i, false); } + let sections = &mut sections[..i]; sections.sort_unstable_by_key(|s| s.base); @@ -634,11 +758,12 @@ fn init_sections(mut allocator: BumpAllocator) { } } - let mut first_pages: [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] = - [None; ORDER_COUNT as usize]; - let mut last_pages = first_pages; - - let mut append_page = |page: Frame, info: &'static PageInfo, order| { + let mut append_page = |page: Frame, + info: &'static PageInfo, + order, + first_pages: &mut [Option<(Frame, &'static PageInfo)>; 11], + last_pages: &mut [Option<(Frame, &PageInfo)>; 11], + allocator: &mut BumpAllocator| { let this_page = (page, info); if page.base() < allocator.abs_offset() { @@ -674,76 +799,153 @@ fn init_sections(mut allocator: BumpAllocator) { }; } - for section in &*sections { - let mut base = section.base; - let mut frames = section.frames; + let mut free_list_fill = |region: Option, + allocator: &mut BumpAllocator| + -> [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] { + let mut first_pages: [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] = + [None; ORDER_COUNT as usize]; + let mut last_pages = first_pages; - for order in 0..=MAX_ORDER { - let pages_for_current_order = 1 << order; - - debug_assert_eq!(frames.len() % pages_for_current_order, 0); - debug_assert!(base.is_aligned_to_order(order)); - - if !frames.is_empty() && order != MAX_ORDER && !base.is_aligned_to_order(order + 1) { - frames[0].next.store(order as usize, Ordering::Relaxed); - // The first section page is not aligned to the next order size. - - //info!("ORDER {order}: FIRST {base:?}"); - append_page(base, &frames[0], order); - - base = base.next_by(pages_for_current_order); - frames = &frames[pages_for_current_order..]; - } else { - //info!("ORDER {order}: FIRST SKIP"); - } - - if !frames.is_empty() - && order != MAX_ORDER - && !base.next_by(frames.len()).is_aligned_to_order(order + 1) + for section in &*sections { + if let Some(region) = region + && (section.base.physaddr.get() < region.base.data() + || section.base.physaddr.get() >= region.base.data() + region.size) { - // The last section page is not aligned to the next order size. - - let off = frames.len() - pages_for_current_order; - let final_page = base.next_by(off); - - frames[off].next.store(order as usize, Ordering::Relaxed); - - //info!("ORDER {order}: LAST {final_page:?}"); - append_page(final_page, &frames[off], order); - - frames = &frames[..off]; - } else { - //info!("ORDER {order}: LAST SKIP"); + continue; } - if frames.is_empty() { - break; - } + let mut base = section.base; + let mut frames = section.frames; + + for order in 0..=MAX_ORDER { + let pages_for_current_order = 1 << order; - if order == MAX_ORDER { debug_assert_eq!(frames.len() % pages_for_current_order, 0); - debug_assert!(base.is_aligned_to_order(MAX_ORDER)); + debug_assert!(base.is_aligned_to_order(order)); - for (off, info) in frames.iter().enumerate().step_by(pages_for_current_order) { - info.next.store(MAX_ORDER as usize, Ordering::Relaxed); - append_page(base.next_by(off), info, MAX_ORDER); + if !frames.is_empty() && order != MAX_ORDER && !base.is_aligned_to_order(order + 1) + { + frames[0].next.store(order as usize, Ordering::Relaxed); + // The first section page is not aligned to the next order size. + + //info!("ORDER {order}: FIRST {base:?}"); + append_page( + base, + &frames[0], + order, + &mut first_pages, + &mut last_pages, + allocator, + ); + + base = base.next_by(pages_for_current_order); + frames = &frames[pages_for_current_order..]; + } else { + //info!("ORDER {order}: FIRST SKIP"); + } + + if !frames.is_empty() + && order != MAX_ORDER + && !base.next_by(frames.len()).is_aligned_to_order(order + 1) + { + // The last section page is not aligned to the next order size. + + let off = frames.len() - pages_for_current_order; + let final_page = base.next_by(off); + + frames[off].next.store(order as usize, Ordering::Relaxed); + + //info!("ORDER {order}: LAST {final_page:?}"); + append_page( + final_page, + &frames[off], + order, + &mut first_pages, + &mut last_pages, + allocator, + ); + + frames = &frames[..off]; + } else { + //info!("ORDER {order}: LAST SKIP"); + } + + if frames.is_empty() { + break; + } + + if order == MAX_ORDER { + debug_assert_eq!(frames.len() % pages_for_current_order, 0); + debug_assert!(base.is_aligned_to_order(MAX_ORDER)); + + for (off, info) in frames.iter().enumerate().step_by(pages_for_current_order) { + info.next.store(MAX_ORDER as usize, Ordering::Relaxed); + append_page( + base.next_by(off), + info, + MAX_ORDER, + &mut first_pages, + &mut last_pages, + allocator, + ); + } } } + + //info!("SECTION from {:?}, {} pages, array at {:p}", section.base, section.frames.len(), section.frames); + } + for (order, tuple_opt) in last_pages.iter().enumerate() { + let Some((frame, info)) = tuple_opt else { + continue; + }; + debug_assert!(frame.is_aligned_to_order(order as u32)); + let free = info.as_free().unwrap(); + debug_assert_eq!(free.prev().order(), order as u32); + free.set_next(P2Frame::new(None, order as u32)); } - //info!("SECTION from {:?}, {} pages, array at {:p}", section.base, section.frames.len(), section.frames); - } - for (order, tuple_opt) in last_pages.iter().enumerate() { - let Some((frame, info)) = tuple_opt else { - continue; + first_pages + }; + + let free_lists; + if let Some(regions) = numa::memory_regions() { + let free_list_page = allocator + .allocate(FrameCount::new( + (numa::number_of_memory_regions() * size_of::>()) + .div_ceil(PAGE_SIZE), + )) + .expect("Failed to allocate free list page"); + let va = unsafe { RmmA::phys_to_virt(free_list_page).data() as *mut Mutex }; + free_lists = unsafe { slice::from_raw_parts_mut(va, numa::number_of_memory_regions()) }; + for (i, region) in regions.enumerate() { + let free_list = FreeList { + for_orders: free_list_fill(Some(region), &mut allocator) + .map(|pair| pair.map(|(frame, _)| frame)), + used_frames: 0, + upper_limit: region.base.add(region.size).data(), + lower_limit: region.base.data(), + }; + free_lists[i] = Mutex::new(free_list); + } + } else { + let free_list_page = allocator + .allocate(FrameCount::new( + size_of::>().div_ceil(PAGE_SIZE), + )) + .expect("Failed to allocate free list page"); + let va = unsafe { RmmA::phys_to_virt(free_list_page).data() as *mut Mutex }; + free_lists = unsafe { slice::from_raw_parts_mut(va, 1) }; + let free_list = FreeList { + for_orders: free_list_fill(None, &mut allocator) + .map(|pair| pair.map(|(frame, _)| frame)), + used_frames: 0, + upper_limit: usize::MAX, + lower_limit: 0, }; - debug_assert!(frame.is_aligned_to_order(order as u32)); - let free = info.as_free().unwrap(); - debug_assert_eq!(free.prev().order(), order as u32); - free.set_next(P2Frame::new(None, order as u32)); + free_lists[0] = Mutex::new(free_list); } - FREELIST.lock().for_orders = first_pages.map(|pair| pair.map(|(frame, _)| frame)); + FREE_LISTS.call_once(|| free_lists); //debug_freelist(); debug!("Initial freelist consistent"); @@ -1027,10 +1229,9 @@ pub fn page_fault_handler( let mut token = unsafe { CleanLockToken::new() }; match context::memory::try_correcting_page_tables(faulting_page, mode, &mut token) { Ok(()) => return Ok(()), - Err(PfError::Oom) => { - return Err(Segv); - } - Err(PfError::Segv | PfError::RecursionLimitExceeded | PfError::NonfatalInternalError) => (), + Err(PfError::Oom) => todo!("oom"), + Err(PfError::Segv | PfError::RecursionLimitExceeded) => (), + Err(PfError::NonfatalInternalError) => todo!(), } } diff --git a/src/numa.rs b/src/numa.rs new file mode 100644 index 0000000000..4844ed8ff7 --- /dev/null +++ b/src/numa.rs @@ -0,0 +1,224 @@ +use core::ops::Add; + +use crate::{ + acpi, + cpu_set::LogicalCpuId, + sync::{CleanLockToken, Mutex, L0}, +}; +use alloc::{sync::Arc, vec::Vec}; +use hashbrown::HashMap; +use rmm::{Arch, BumpAllocator, MemoryArea, PhysicalAddress}; +use spin::once::Once; +use syscall::{Error, Result, ENODATA, EOPNOTSUPP}; + +pub const MAX_DOMAINS: usize = 128; + +static DOMAIN_NODE_MAP: Once<&'static [u32]> = Once::new(); +static NUMA_CPUS: Once<&'static [u32]> = Once::new(); +static NUMA_MEMORY: Once<&'static [NumaMemory]> = Once::new(); +static DISTANCES: Once<&'static [u8]> = Once::new(); + +#[derive(Debug, Clone)] +pub struct NumaMemory { + pub start: usize, + pub length: usize, + pub node_id: u32, + pub _pad: [u8; 4], +} + +#[derive(Debug)] +pub struct NumaCpu { + pub id: u32, +} + +pub fn init(allocator: &mut BumpAllocator) { + #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] + { + acpi::srat::init(allocator, &DOMAIN_NODE_MAP, &NUMA_CPUS, &NUMA_MEMORY); + acpi::slit::init(allocator, &DISTANCES); + } +} + +pub fn assign_node_id(modify: bool) -> u8 { + static mut NODE_ID: u8 = 0; + if unsafe { NODE_ID } >= 128 { + panic!("Maximum number of domains supported is 128"); + } + unsafe { + NODE_ID += 1; + let return_value = NODE_ID - 1; + if !modify { + NODE_ID -= 1; + } + return_value + } +} + +pub fn assign_memory_id() -> u8 { + static mut MEMORY_ID: u8 = 0; + if unsafe { MEMORY_ID } >= 128 { + panic!("Maximum number of memory regions supported is 128"); + } + let old = unsafe { MEMORY_ID }; + unsafe { MEMORY_ID = MEMORY_ID.add(1) }; + old +} + +pub fn domain_to_node_id(domain_id: u32) -> Option { + Some(*DOMAIN_NODE_MAP.get()?.get(domain_id as usize)?) +} + +pub fn cpu_belongs_to_which_node(cpu_id: usize) -> Option { + Some(*NUMA_CPUS.get()?.get(cpu_id)?) +} + +/// A helper function that prints information about NUMA - available nodes, cpus and memory blocks in them +/// their starts and lengths +pub fn dump_info() { + if let Some(map) = DOMAIN_NODE_MAP.get() + && let Some(cpus) = NUMA_CPUS.get() + && let Some(memories) = NUMA_MEMORY.get() + { + println!("Number of NUMA nodes: {}", assign_node_id(false)); + for i in 0..cpus.len() { + if cpus[i] == u32::MAX { + continue; + } + println!("CPU {} : Node {}", i, cpus[i]) + } + for i in 0..memories.len() { + if memories[i].length == 0 { + continue; + } + println!( + "Memory Block starting at address {:#x} of size {:#x} bytes : Node {}", + memories[i].start, memories[i].length, memories[i].node_id + ); + } + } else { + println!( + "The system has either no support for NUMA or there was an error during initialisation" + ); + } +} + +pub struct NumaMemoryIter { + i: usize, + mem: &'static [NumaMemory], +} + +impl Iterator for NumaMemoryIter { + type Item = MemoryArea; + + fn next(&mut self) -> Option { + let mem = self.mem.get(self.i)?; + if mem.length == 0 { + return None; + } + self.i += 1; + Some(MemoryArea { + base: PhysicalAddress::new(mem.start), + size: mem.length, + }) + } +} + +impl NumaMemoryIter { + /// Skips an arbitrarily chosen `i`th element. Unlike `skip`, which skips the first `n` elements, + /// `iskip` can ignore non-consecutive elements + pub fn iskip(&self, addr: usize) -> Option { + let i = self.mem.binary_search_by_key(&addr, |e| e.start).ok()?; + Some(i) + } +} + +pub fn number_of_memory_regions() -> usize { + if let Some(mem) = NUMA_MEMORY.get() { + mem.iter() + .map(|e| if e.length != 0 { 1 } else { 0 }) + .sum::() + } else { + 0 // TODO: or should 1 be returned? + } +} + +pub fn memory_regions() -> Option { + if let Some(mem) = NUMA_MEMORY.get() { + Some(NumaMemoryIter { i: 0, mem }) + } else { + None + } +} + +pub fn nearest_next_memory_region(addr: usize, overlap: bool) -> Option<&'static NumaMemory> { + NUMA_MEMORY + .get()? + .iter() + .filter_map(|e| { + if if overlap { + e.start >= addr + } else { + e.start > addr + } { + Some(e) + } else { + None + } + }) + .min_by_key(|e| e.start) +} + +pub fn nearest_preceding_memory_region(addr: usize, overlap: bool) -> Option<&'static NumaMemory> { + NUMA_MEMORY + .get()? + .iter() + .filter_map(|e| { + if if overlap { + e.start <= addr + } else { + e.start < addr + } { + Some(e) + } else { + None + } + }) + .max_by_key(|e| e.start) +} + +pub fn get_numa_info(token: &mut CleanLockToken) -> Result> { + let cpu_info = NUMA_CPUS + .get() + .ok_or(Error::new(EOPNOTSUPP))? + .iter() + .map(|e| e.to_ne_bytes()) + .flatten() + .collect::>(); + let mem_info = NUMA_MEMORY + .get() + .ok_or(Error::new(EOPNOTSUPP))? + .iter() + .map(|e| { + [ + e.start.to_ne_bytes(), + e.length.to_ne_bytes(), + (e.node_id as usize).to_ne_bytes(), + ] + }) + .flatten() + .flatten() + .collect::>(); + let mut numa_info = Vec::new(); + numa_info.extend(cpu_info); + numa_info.extend(mem_info); + Ok(numa_info) +} + +pub fn get_numa_distance_info(token: &mut CleanLockToken) -> Result> { + Ok(DISTANCES + .get() + .ok_or(Error::new(ENODATA))? + .iter() + .map(|e| *e) + .collect()) +} diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index ceab9aaea7..0923c47ef4 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -6,8 +6,6 @@ //! The kernel validates paths and file descriptors before they are passed to schemes, //! also stripping the scheme identifier of paths if necessary. -// TODO: Move handling of the global namespace to userspace. - use alloc::{ sync::{Arc, Weak}, vec::Vec, diff --git a/src/scheme/sys/mod.rs b/src/scheme/sys/mod.rs index 51f0e51add..80762ba26f 100644 --- a/src/scheme/sys/mod.rs +++ b/src/scheme/sys/mod.rs @@ -112,6 +112,8 @@ const FILES: &[(&str, Kind)] = &[ ("irq", Rd(irq::resource)), ("log", Rd(log::resource)), ("mem", Rd(mem::resource)), + ("numa", Rd(crate::numa::get_numa_info)), + ("numa_dist", Rd(crate::numa::get_numa_distance_info)), ("syscall", Rd(syscall::resource)), ("uname", Rd(uname::resource)), ("env", Rd(|_| Ok(Vec::from(crate::startup::init_env())))), diff --git a/src/startup/memory.rs b/src/startup/memory.rs index 26922dde0a..2f78c98f75 100644 --- a/src/startup/memory.rs +++ b/src/startup/memory.rs @@ -1,6 +1,7 @@ use crate::{ arch::CurrentRmmArch, memory::PAGE_SIZE, + numa, startup::{memory::BootloaderMemoryKind::Null, KernelArgs}, }; use core::{ diff --git a/src/startup/mod.rs b/src/startup/mod.rs index 1992c7fbbf..7b3f532fa4 100644 --- a/src/startup/mod.rs +++ b/src/startup/mod.rs @@ -1,18 +1,20 @@ use core::{ - hint, slice, + hint, + ptr::NonNull, + slice, sync::atomic::{AtomicBool, Ordering}, }; -use core::ptr::NonNull; use crate::{ arch::interrupt, - context, - context::switch::SwitchResult, + context::{self, switch::SwitchResult}, memory::{PhysicalAddress, RmmA, RmmArch}, profiling, scheme, sync::CleanLockToken, }; +use crate::numa; + pub mod memory; #[repr(C, packed(8))] @@ -105,7 +107,7 @@ impl KernelArgs { ) }; if data.starts_with(b"RSD PTR ") { - Some(NonNull::new(data.as_ptr() as *mut u8).unwrap()) + Some(NonNull::from_ref(data).cast()) } else { None } @@ -186,6 +188,7 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { } } + numa::dump_info(); run_userspace(&mut token) } @@ -230,15 +233,8 @@ fn run_userspace(token: &mut CleanLockToken) -> ! { interrupt::enable_and_nop(); } SwitchResult::AllContextsIdle => { - // Enable interrupts, then enter the deepest MWAIT - // C-state (C6/C7/C8/C9/C10/S0iX). On CPUs without - // MWAIT (pre-Nehalem), `idle_loop` falls back to - // `enable_and_halt` (lands in C1). The MWAIT path - // enables Arrow Lake-H to actually reach S0i2/S0i3 - // substates and dramatically reduce idle power on the - // LG Gram 2025; without it the kernel only lands in - // C1 and the CPU stays relatively warm. - interrupt::idle_loop(); + // Enable interrupts, then halt CPU (to save power) until the next interrupt is actually fired. + interrupt::enable_and_halt(); } } }