From c83b38eb705822c83730d518f7ab06be0f89ec0a Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Mon, 6 Jul 2026 19:30:05 +0200 Subject: [PATCH 1/7] Remove obsolete TODO, run rustfmt. --- src/acpi/rsdt.rs | 8 ++++++-- src/acpi/rxsdt.rs | 14 +++++++++----- src/acpi/xsdt.rs | 8 ++++++-- src/scheme/mod.rs | 2 -- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/acpi/rsdt.rs b/src/acpi/rsdt.rs index 2fd63f192e..fc95e8d141 100644 --- a/src/acpi/rsdt.rs +++ b/src/acpi/rsdt.rs @@ -2,7 +2,7 @@ use alloc::boxed::Box; use core::convert::TryFrom; use rmm::PhysicalAddress; -use crate::acpi::{RxsdtEnum, rxsdt::RxsdtIter}; +use crate::acpi::{rxsdt::RxsdtIter, RxsdtEnum}; use super::{rxsdt::Rxsdt, sdt::Sdt}; @@ -27,6 +27,10 @@ impl Rsdt { impl Rxsdt for Rsdt { fn iter(&self) -> RxsdtIter { - RxsdtIter { sdt: self.0, i: 0 , rxsdt_enum: RxsdtEnum::Rsdt(Rsdt(self.0))} + RxsdtIter { + sdt: self.0, + i: 0, + rxsdt_enum: RxsdtEnum::Rsdt(Rsdt(self.0)), + } } } diff --git a/src/acpi/rxsdt.rs b/src/acpi/rxsdt.rs index 457800f00e..f3cc49b7ea 100644 --- a/src/acpi/rxsdt.rs +++ b/src/acpi/rxsdt.rs @@ -1,7 +1,7 @@ use alloc::boxed::Box; use rmm::PhysicalAddress; -use crate::acpi::{RxsdtEnum, sdt::Sdt}; +use crate::acpi::{sdt::Sdt, RxsdtEnum}; pub trait Rxsdt { fn iter(&self) -> RxsdtIter; @@ -18,10 +18,14 @@ impl Iterator for RxsdtIter { fn next(&mut self) -> Option { if self.i < self.sdt.data_len() / size_of::() { let item = unsafe { - match self.rxsdt_enum{ - RxsdtEnum::Rsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned((self.sdt.data_address() as *const u32).add(self.i)) as usize), - RxsdtEnum::Xsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned((self.sdt.data_address() as *const u64).add(self.i)) as usize), - } + match self.rxsdt_enum { + RxsdtEnum::Rsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned( + (self.sdt.data_address() as *const u32).add(self.i), + ) as usize), + RxsdtEnum::Xsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned( + (self.sdt.data_address() as *const u64).add(self.i), + ) as usize), + } }; self.i += 1; Some(item) diff --git a/src/acpi/xsdt.rs b/src/acpi/xsdt.rs index c95addc1d3..4f0f1e16ec 100644 --- a/src/acpi/xsdt.rs +++ b/src/acpi/xsdt.rs @@ -2,7 +2,7 @@ use alloc::boxed::Box; use core::convert::TryFrom; use rmm::PhysicalAddress; -use crate::acpi::{RxsdtEnum, rxsdt::RxsdtIter}; +use crate::acpi::{rxsdt::RxsdtIter, RxsdtEnum}; use super::{rxsdt::Rxsdt, sdt::Sdt}; @@ -27,6 +27,10 @@ impl Xsdt { impl Rxsdt for Xsdt { fn iter(&self) -> RxsdtIter { - RxsdtIter { sdt: self.0, i: 0, rxsdt_enum: RxsdtEnum::Xsdt(Xsdt(self.0)) } + RxsdtIter { + sdt: self.0, + i: 0, + rxsdt_enum: RxsdtEnum::Xsdt(Xsdt(self.0)), + } } } diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index 5e4c78b02c..13b2b45d8d 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -6,8 +6,6 @@ //! The kernel validates paths and file descriptors before they are passed to schemes, //! also stripping the scheme identifier of paths if necessary. -// TODO: Move handling of the global namespace to userspace. - use alloc::{ sync::{Arc, Weak}, vec::Vec, From 38642377f3acc1870b11fb6c62b31707202e91a1 Mon Sep 17 00:00:00 2001 From: Akshit Gaur Date: Tue, 7 Jul 2026 12:43:41 +0000 Subject: [PATCH 2/7] Separate timers --- src/context/context.rs | 1 + src/context/mod.rs | 7 ++++++- src/context/switch.rs | 46 +++++++++++++++++++++++++----------------- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/context/context.rs b/src/context/context.rs index 8bd44177e0..fbb20c4c7c 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -279,6 +279,7 @@ impl Context { pub fn unblock_no_ipi(&mut self) -> bool { if self.status.is_soft_blocked() { self.status = Status::Runnable; + self.wake = None; self.status_reason = ""; true diff --git a/src/context/mod.rs b/src/context/mod.rs index 2ec4ded54a..1518dc3367 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -83,6 +83,7 @@ static IDLE_CONTEXTS: Mutex> = Mutex::new(VecDeque: pub struct RunContextData { // queue: VecDeque, queue: BTreeMap<(u64, Reverse, u32), (u64, u64, WeakContextRef)>, // ((vd, rem_slice, ctxt_id), (vtime, weight, context)) + timers: BTreeSet<(u128, WeakContextRef)>, // (wake, context) count: usize, v: u64, total_weight: u64, @@ -91,9 +92,9 @@ pub struct RunContextData { impl RunContextData { pub const fn new() -> Self { - const EMPTY_VEC: VecDeque = VecDeque::new(); Self { queue: BTreeMap::new(), + timers: BTreeSet::new(), count: 0, v: 0, total_weight: 0, @@ -130,6 +131,10 @@ pub fn run_contexts(token: LockToken<'_, L0>) -> MutexGuard<'_, L1, RunContextDa RUN_CONTEXTS.lock(token) } +pub fn run_contexts_try(token: LockToken<'_, L0>) -> Option> { + RUN_CONTEXTS.try_lock(token) +} + pub fn init(token: &mut CleanLockToken) { let owner = None; // kmain not owned by any fd let mut context = Context::new(owner).expect("failed to create kmain context"); diff --git a/src/context/switch.rs b/src/context/switch.rs index b94a02b713..5de01b01ef 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -5,7 +5,8 @@ use crate::{ context::{ self, arch, idle_contexts, idle_contexts_try, memory::AddrSpaceSwitchReadGuard, - run_contexts, ArcContextLockWriteGuard, Context, ContextLock, WeakContextRef, + run_contexts, run_contexts_try, ArcContextLockWriteGuard, Context, ContextLock, + WeakContextRef, }, cpu_set::LogicalCpuId, cpu_stats::{self, CpuState}, @@ -186,13 +187,24 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { } // Alarm (previously in update_runnable) - let wakeups = wakeup_contexts(token, switch_time); - let wakeups_len = wakeups.len(); + let mut wakeups = wakeup_contexts(token); let mut push_idle: SmallVec<[WeakContextRef; 16]> = SmallVec::new(); - if wakeups_len > 0 { - let mut run_contexts = run_contexts(token.token()); + if let Some(mut run_contexts) = run_contexts_try(token.token()) { + // Pop Timers + while let Some((wake, _)) = run_contexts.timers.first() { + if *wake > switch_time { + break; + } + if let Some((_, context_ref)) = run_contexts.timers.pop_first() { + wakeups.push(context_ref); + } + } + } + + if wakeups.len() > 0 { + let mut run_contexts = run_contexts(token.token()); for context_ref in wakeups { let Some(context_lock) = context_ref.upgrade() else { continue; @@ -379,9 +391,8 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { fn wakeup_contexts( token: &mut CleanLockToken, - switch_time: u128, ) -> SmallVec<[WeakContextRef; 16]> { - // TODO: Optimise this somehow. Perhaps using a separate timer queue? + // TODO: Optimise this somehow let mut wakeups = SmallVec::new(); let current_context = context::current(); let Some(idle_contexts) = idle_contexts_try(token.downgrade()) else { @@ -405,15 +416,7 @@ fn wakeup_contexts( idle_contexts.push_back(context_ref); continue; }; - if guard.status.is_soft_blocked() { - if let Some(wake) = guard.wake { - if switch_time >= wake { - drop(guard); - wakeups.push(context_ref); - continue; - } - } - } else if guard.status.is_dead() { + if guard.status.is_dead() { // TODO: who hold this dead context? continue; } @@ -448,6 +451,7 @@ fn select_next_context( let prev_context_lock = crate::context::current(); let is_idle = Arc::ptr_eq(&prev_context_lock, &idle_context); let prev_runnable = !is_idle && prev_context_guard.status.is_runnable(); + let is_timer = prev_context_guard.wake.is_some(); let elapsed_ticks = elapsed_time as u128 * SCALE / NANOS_PER_TICK; @@ -484,6 +488,12 @@ fn select_next_context( contexts_data.total_weight = contexts_data.total_weight.saturating_sub(weight); } prev_context_guard.rem_slice = 0; + + if let Some(wake) = prev_context_guard.wake { + contexts_data + .timers + .insert((wake, WeakContextRef(Arc::downgrade(&prev_context_lock)))); + } } let mut eligible_best = None; @@ -647,7 +657,7 @@ fn select_next_context( WeakContextRef(Arc::downgrade(&prev_context_lock)), ), ); - } else if !is_idle { + } else if !is_idle && !is_timer { idle_contexts(token.token()) .push_back(WeakContextRef(Arc::downgrade(&prev_context_lock))); } @@ -657,7 +667,7 @@ fn select_next_context( return None; } } else { - if !is_idle { + if !is_idle && !is_timer { idle_contexts(token.token()) .push_back(WeakContextRef(Arc::downgrade(&prev_context_lock))); } From 621a0f4be9928efef854d38466e36609d56c266a Mon Sep 17 00:00:00 2001 From: R Aadarsh Date: Wed, 1 Jul 2026 10:46:27 +0530 Subject: [PATCH 3/7] * Split free areas at NUMA memory boundaries * Fix incorrect name usage in ACPI ARM parsing code --- rmm/src/lib.rs | 2 + src/acpi/srat/aarch64.rs | 2 +- src/memory/mod.rs | 228 ++++++++++++++++++++++++++++----------- src/numa.rs | 83 +++++++++++++- 4 files changed, 248 insertions(+), 67 deletions(-) diff --git a/rmm/src/lib.rs b/rmm/src/lib.rs index 3088ec989e..3be8f35e6e 100644 --- a/rmm/src/lib.rs +++ b/rmm/src/lib.rs @@ -1,6 +1,8 @@ #![no_std] #![allow(clippy::new_without_default)] +use core::ops::Add; + pub use crate::{allocator::*, arch::*, page::*}; mod allocator; diff --git a/src/acpi/srat/aarch64.rs b/src/acpi/srat/aarch64.rs index fd37e13a71..090a9ff41e 100644 --- a/src/acpi/srat/aarch64.rs +++ b/src/acpi/srat/aarch64.rs @@ -35,7 +35,7 @@ pub fn init_srat(dom_node_map: &mut [u32], cpus: &mut [u32], mem: &mut [NumaMemo ); assert!( - cpu_count <= cpu_set::MAX_CPU_COUNT, + cpu_count <= MAX_CPU_COUNT, "Found more number of CPUs than supported" ); diff --git a/src/memory/mod.rs b/src/memory/mod.rs index da8f7e4708..6ce90a58eb 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -4,6 +4,7 @@ use core::{ cell::SyncUnsafeCell, num::NonZeroUsize, + ops::AddAssign, sync::atomic::{AtomicUsize, Ordering}, }; @@ -11,6 +12,8 @@ pub use kernel_mapper::KernelMapper; use spin::Mutex; pub use crate::arch::CurrentRmmArch as RmmA; +#[cfg(feature = "numa")] +use crate::numa; use crate::{ context::{ self, @@ -21,7 +24,7 @@ use crate::{ syscall::error::{Error, ENOMEM}, }; pub use rmm::{Arch as RmmArch, PageFlags, PhysicalAddress, TableKind, VirtualAddress}; -use rmm::{BumpAllocator, FrameAllocator, FrameCount, FrameUsage}; +use rmm::{BumpAllocator, FrameAllocator, FrameCount, FrameUsage, MemoryArea}; mod kernel_mapper; pub mod page; @@ -503,6 +506,17 @@ const _: () = { #[cold] fn init_sections(mut allocator: BumpAllocator) { + let number_of_memory_regions = { + #[cfg(feature = "numa")] + { + numa::number_of_memory_regions() + } + #[cfg(not(feature = "numa"))] + { + 0 + } + }; + let (free_areas, offset_into_first_free_area) = allocator.free_areas(); let free_areas_iter = || { @@ -528,6 +542,21 @@ fn init_sections(mut allocator: BumpAllocator) { .next_multiple_of(MAX_SECTION_SIZE); let aligned_start = area.base.data() / MAX_SECTION_SIZE * MAX_SECTION_SIZE; + #[cfg(feature = "numa")] + { + if number_of_memory_regions > 0 { + if let Some(next_memory_region) = + numa::nearest_next_memory_region(area.base.data(), false) + && next_memory_region.start < area.base.add(area.size).data() + && next_memory_region.start + next_memory_region.length + > area.base.add(area.size).data() + && !next_memory_region.start.is_multiple_of(MAX_SECTION_SIZE) + { + return (aligned_end - aligned_start) / MAX_SECTION_SIZE + 1; + } + } + } + (aligned_end - aligned_start) / MAX_SECTION_SIZE }) .sum(); @@ -547,76 +576,145 @@ fn init_sections(mut allocator: BumpAllocator) { let mut iter = free_areas_iter().peekable(); - let mut i = 0; + let mut sections_fill = |region: Option, + i: &mut usize, // out parameter + force: bool| + -> Option { + let mut iter = free_areas_iter().peekable(); - while let Some(mut memory_map_area) = iter.next() { - // TODO: NonZeroUsize - - // TODO: x86_32 fails without this check - if memory_map_area.size == 0 { - continue; - } - - assert_ne!( - memory_map_area.size, 0, - "RMM should enforce areas are not of length 0" - ); - - // TODO: Should RMM do this? - - while let Some(next_area) = iter.peek() - && next_area.base == memory_map_area.base.add(memory_map_area.size) - { - memory_map_area.size += next_area.size; - let _ = iter.next(); - } - - assert_eq!( - memory_map_area.base.data() % PAGE_SIZE, - 0, - "RMM should enforce area alignment" - ); - assert_eq!( - memory_map_area.size % PAGE_SIZE, - 0, - "RMM should enforce area length alignment" - ); - - let mut pages_left = memory_map_area.size.div_floor(PAGE_SIZE); - let mut base = Frame::containing(memory_map_area.base); - - while pages_left > 0 { - let page_info_max_count = core::cmp::min(pages_left, MAX_SECTION_PAGE_COUNT); - let pages_to_next_section = - (MAX_SECTION_SIZE - (base.base().data() % MAX_SECTION_SIZE)) / PAGE_SIZE; - let page_info_count = core::cmp::min(page_info_max_count, pages_to_next_section); - - let page_info_array_size_pages = - (page_info_count * size_of::()).div_ceil(PAGE_SIZE); - let page_info_array = unsafe { - let base = allocator - .allocate(FrameCount::new(page_info_array_size_pages)) - .expect("failed to allocate page info array"); - core::slice::from_raw_parts_mut( - RmmA::phys_to_virt(base).data() as *mut PageInfo, - page_info_count, - ) - }; - for p in &*page_info_array { - assert_eq!(p.next.load(Ordering::Relaxed), 0); - assert_eq!(p.refcount.load(Ordering::Relaxed), 0); + while let Some(mut memory_map_area) = iter.next() { + if !{ + if let Some(region) = region + && !force + { + memory_map_area.base >= region.base + && memory_map_area.base.data() < region.base.data() + region.size + } else { + true + } + } { + continue; } - sections[i] = Section { - base, - frames: page_info_array, - }; - i += 1; + // TODO: NonZeroUsize - pages_left -= page_info_count; - base = base.next_by(page_info_count); + // TODO: x86_32 fails without this check + if memory_map_area.size == 0 { + continue; + } + + assert_ne!( + memory_map_area.size, 0, + "RMM should enforce areas are not of length 0" + ); + + // TODO: Should RMM do this? + + while let Some(next_area) = iter.peek() + && next_area.base == memory_map_area.base.add(memory_map_area.size) + && if let Some(region) = region { + next_area.base >= region.base + && next_area.base.data() < region.base.data() + region.size + && !force + } else { + true + } + { + memory_map_area.size += next_area.size; + let _ = iter.next(); + } + + assert_eq!( + memory_map_area.base.data() % PAGE_SIZE, + 0, + "RMM should enforce area alignment" + ); + assert_eq!( + memory_map_area.size % PAGE_SIZE, + 0, + "RMM should enforce area length alignment" + ); + + let mut pages_left = memory_map_area.size.div_floor(PAGE_SIZE); + let mut base = Frame::containing(memory_map_area.base); + let mut return_value = None; + + while pages_left > 0 { + let page_info_max_count = core::cmp::min(pages_left, MAX_SECTION_PAGE_COUNT); + let pages_to_next_section = + (MAX_SECTION_SIZE - (base.base().data() % MAX_SECTION_SIZE)) / PAGE_SIZE; + let mut page_info_count = + core::cmp::min(page_info_max_count, pages_to_next_section); + + if !force + && let Some(region) = region + && base.base() >= region.base + && base.base().data() < region.base.data() + region.size + && (page_info_count * PAGE_SIZE + > (region.base.data() + region.size - base.base().data())) + { + page_info_count = + (region.base.data() + region.size - base.base().data()) / PAGE_SIZE; + return_value = Some(MemoryArea { + base: PhysicalAddress::new(region.base.data() + region.size), + size: (base.physaddr.get() + pages_left * PAGE_SIZE) + - (region.base.data() + region.size), + }); + } + + let page_info_array_size_pages = + (page_info_count * size_of::()).div_ceil(PAGE_SIZE); + let page_info_array = unsafe { + let base = allocator + .allocate(FrameCount::new(page_info_array_size_pages)) + .expect("failed to allocate page info array"); + core::slice::from_raw_parts_mut( + RmmA::phys_to_virt(base).data() as *mut PageInfo, + page_info_count, + ) + }; + for p in &*page_info_array { + assert_eq!(p.next.load(Ordering::Relaxed), 0); + assert_eq!(p.refcount.load(Ordering::Relaxed), 0); + } + + sections[*i] = Section { + base, + frames: page_info_array, + }; + i.add_assign(1); + + pages_left -= page_info_count; + base = base.next_by(page_info_count); + if let Some(return_value) = return_value { + return Some(return_value); + } + } } + return None; + }; + let mut i = 0; + + #[cfg(feature = "numa")] + if number_of_memory_regions > 0 + && let Some(regions) = numa::memory_regions() + { + for region in regions { + let mut sections_fill_result = Some(region); + let mut force = false; + + while let Some(mut region) = sections_fill_result { + sections_fill_result = sections_fill(Some(region), &mut i, force); + force = true; + } + } + } else { + sections_fill(None, &mut i, false); } + + #[cfg(not(feature = "numa"))] + sections_fill(None, &mut i, false); + let sections = &mut sections[..i]; sections.sort_unstable_by_key(|s| s.base); diff --git a/src/numa.rs b/src/numa.rs index a8429c68c6..b3265bb88d 100644 --- a/src/numa.rs +++ b/src/numa.rs @@ -7,7 +7,7 @@ use crate::{ }; use alloc::{sync::Arc, vec::Vec}; use hashbrown::HashMap; -use rmm::{Arch, BumpAllocator}; +use rmm::{Arch, BumpAllocator, MemoryArea, PhysicalAddress}; use spin::once::Once; pub const MAX_DOMAINS: usize = 128; @@ -100,3 +100,84 @@ pub fn dump_info() { ); } } + +pub struct NumaMemoryIter { + i: usize, + mem: &'static [NumaMemory], +} + +impl Iterator for NumaMemoryIter { + type Item = MemoryArea; + + fn next(&mut self) -> Option { + let mem = self.mem.get(self.i)?; + self.i += 1; + Some(MemoryArea { + base: PhysicalAddress::new(mem.start), + size: mem.length, + }) + } +} + +impl NumaMemoryIter { + /// Skips an arbitrarily chosen `i`th element. Unlike `skip`, which skips the first `n` elements, + /// `iskip` can ignore non-consecutive elements + pub fn iskip(&self, addr: usize) -> Option { + let i = self.mem.binary_search_by_key(&addr, |e| e.start).ok()?; + Some(i) + } +} + +pub fn number_of_memory_regions() -> usize { + if let Some(mem) = NUMA_MEMORY.get() { + mem.iter() + .map(|e| if e.length != 0 { 1 } else { 0 }) + .sum::() + } else { + 0 // TODO: or should 1 be returned? + } +} + +pub fn memory_regions() -> Option { + if let Some(mem) = NUMA_MEMORY.get() { + Some(NumaMemoryIter { i: 0, mem }) + } else { + None + } +} + +pub fn nearest_next_memory_region(addr: usize, overlap: bool) -> Option<&'static NumaMemory> { + NUMA_MEMORY + .get()? + .iter() + .filter_map(|e| { + if if overlap { + e.start >= addr + } else { + e.start > addr + } { + Some(e) + } else { + None + } + }) + .min_by_key(|e| e.start) +} + +pub fn nearest_preceding_memory_region(addr: usize, overlap: bool) -> Option<&'static NumaMemory> { + NUMA_MEMORY + .get()? + .iter() + .filter_map(|e| { + if if overlap { + e.start <= addr + } else { + e.start < addr + } { + Some(e) + } else { + None + } + }) + .max_by_key(|e| e.start) +} From bfbbec47a982626b7d482bc9e22387d89ae3cc5a Mon Sep 17 00:00:00 2001 From: R Aadarsh Date: Wed, 8 Jul 2026 21:23:22 +0530 Subject: [PATCH 4/7] Create seperate free lists for each NUMA memory regions --- Cargo.toml | 1 - src/acpi/mod.rs | 2 - src/arch/x86_shared/start.rs | 2 - src/main.rs | 1 - src/memory/mod.rs | 301 +++++++++++++++++++++++------------ src/numa.rs | 3 + src/startup/memory.rs | 3 +- src/startup/mod.rs | 2 - 8 files changed, 206 insertions(+), 109 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ea56939857..825b56a74a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,6 @@ default = [ "acpi", #"debugger", "multi_core", - # "numa", "serial_debug", "self_modifying", "x86_kvm_pv", diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index 34125f31b4..8e742b5da8 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -23,11 +23,9 @@ mod rsdp; mod rsdt; mod rxsdt; pub mod sdt; -#[cfg(feature = "numa")] pub mod slit; #[cfg(target_arch = "aarch64")] mod spcr; -#[cfg(feature = "numa")] pub mod srat; mod xsdt; diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index 9d8ec3bf21..3a7abbdc39 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -12,7 +12,6 @@ use crate::{ startup::KernelArgs, }; -#[cfg(feature = "numa")] use crate::numa; /// Test of zero values in BSS. @@ -117,7 +116,6 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { crate::acpi::init_before_mem(args.acpi_rsdp()); } - #[cfg(feature = "numa")] numa::init(&mut bump_allocator); crate::memory::init_mm(bump_allocator); diff --git a/src/main.rs b/src/main.rs index 23a60c006a..8380e02f94 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,7 +69,6 @@ mod log; mod memory; /// NUMA support -#[cfg(feature = "numa")] mod numa; /// Panic diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 6ce90a58eb..16c85d694b 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -5,21 +5,21 @@ use core::{ cell::SyncUnsafeCell, num::NonZeroUsize, ops::AddAssign, - sync::atomic::{AtomicUsize, Ordering}, + slice, + sync::atomic::{AtomicU8, AtomicUsize, Ordering}, }; pub use kernel_mapper::KernelMapper; -use spin::Mutex; +use spin::{once::Once, Mutex}; pub use crate::arch::CurrentRmmArch as RmmA; -#[cfg(feature = "numa")] -use crate::numa; use crate::{ context::{ self, memory::{AccessMode, PfError}, }, kernel_executable_offsets::{__usercopy_end, __usercopy_start}, + numa, sync::CleanLockToken, syscall::error::{Error, ENOMEM}, }; @@ -57,7 +57,15 @@ pub fn free_frames() -> usize { /// Get the number of frames used pub fn used_frames() -> usize { // TODO: Include bump allocator static pages? - FREELIST.lock().used_frames + FREE_LISTS + .get() + .unwrap() + .iter() + .map(|e| { + let l = e.lock(); + l.used_frames + }) + .sum() } pub fn total_frames() -> usize { // TODO: Include bump allocator static pages? @@ -66,19 +74,42 @@ pub fn total_frames() -> usize { /// Allocate a range of frames pub fn allocate_p2frame(order: u32) -> Option { - allocate_p2frame_complex(order, (), None, order).map(|(f, _)| f) + let initial_index = get_round_robin_index(); + let mut index = initial_index; + + loop { + if let Some(frame) = allocate_p2frame_complex(order, (), None, order, index).map(|(f, _)| f) + { + return Some(frame); + } + index = get_round_robin_index(); + if index == initial_index { + return None; + } + } } pub fn allocate_frame() -> Option { allocate_p2frame(0) } + +fn get_round_robin_index() -> usize { + static CURRENT_INDEX: AtomicU8 = AtomicU8::new(0); + if CURRENT_INDEX.load(Ordering::Acquire) as usize == FREE_LISTS.get().unwrap().len() { + CURRENT_INDEX.store(1, Ordering::Release); + return 0; + } + CURRENT_INDEX.fetch_add(1, Ordering::AcqRel) as usize +} + // TODO: Flags, strategy pub fn allocate_p2frame_complex( _req_order: u32, _flags: (), _strategy: Option<()>, min_order: u32, + index: usize, ) -> Option<(Frame, usize)> { - let mut freelist = FREELIST.lock(); + let mut freelist = FREE_LISTS.get().unwrap()[index].lock(); let (frame_order, frame) = freelist .for_orders @@ -151,8 +182,19 @@ pub fn allocate_p2frame_complex( Some((frame, PAGE_SIZE << min_order)) } +fn get_index_for_deallocation(addr: usize) -> Option { + for (i, list) in FREE_LISTS.get().unwrap().iter().enumerate() { + let l = list.lock(); + if addr >= l.lower_limit && addr < l.upper_limit { + return Some(i); + } + } + None +} + pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) { - let mut freelist = FREELIST.lock(); + let index = get_index_for_deallocation(orig_frame.physaddr.get()).expect("Expected an index"); + let mut freelist = FREE_LISTS.get().unwrap()[index].lock(); let initial_info = get_page_info(orig_frame) .unwrap_or_else(|| panic!("missing PageInfo for {orig_frame:?} being freed")); @@ -483,13 +525,12 @@ struct AllocatorData { } #[derive(Debug)] struct FreeList { + upper_limit: usize, + lower_limit: usize, for_orders: [Option; ORDER_COUNT as usize], used_frames: usize, } -static FREELIST: Mutex = Mutex::new(FreeList { - for_orders: [None; ORDER_COUNT as usize], - used_frames: 0, -}); +static FREE_LISTS: Once<&'static [Mutex]> = Once::new(); pub struct Section { base: Frame, @@ -506,16 +547,7 @@ const _: () = { #[cold] fn init_sections(mut allocator: BumpAllocator) { - let number_of_memory_regions = { - #[cfg(feature = "numa")] - { - numa::number_of_memory_regions() - } - #[cfg(not(feature = "numa"))] - { - 0 - } - }; + let number_of_memory_regions = numa::number_of_memory_regions(); let (free_areas, offset_into_first_free_area) = allocator.free_areas(); @@ -542,18 +574,15 @@ fn init_sections(mut allocator: BumpAllocator) { .next_multiple_of(MAX_SECTION_SIZE); let aligned_start = area.base.data() / MAX_SECTION_SIZE * MAX_SECTION_SIZE; - #[cfg(feature = "numa")] - { - if number_of_memory_regions > 0 { - if let Some(next_memory_region) = - numa::nearest_next_memory_region(area.base.data(), false) - && next_memory_region.start < area.base.add(area.size).data() - && next_memory_region.start + next_memory_region.length - > area.base.add(area.size).data() - && !next_memory_region.start.is_multiple_of(MAX_SECTION_SIZE) - { - return (aligned_end - aligned_start) / MAX_SECTION_SIZE + 1; - } + if number_of_memory_regions > 0 { + if let Some(next_memory_region) = + numa::nearest_next_memory_region(area.base.data(), false) + && next_memory_region.start < area.base.add(area.size).data() + && next_memory_region.start + next_memory_region.length + > area.base.add(area.size).data() + && !next_memory_region.start.is_multiple_of(MAX_SECTION_SIZE) + { + return (aligned_end - aligned_start) / MAX_SECTION_SIZE + 1; } } @@ -695,7 +724,6 @@ fn init_sections(mut allocator: BumpAllocator) { }; let mut i = 0; - #[cfg(feature = "numa")] if number_of_memory_regions > 0 && let Some(regions) = numa::memory_regions() { @@ -712,9 +740,6 @@ fn init_sections(mut allocator: BumpAllocator) { sections_fill(None, &mut i, false); } - #[cfg(not(feature = "numa"))] - sections_fill(None, &mut i, false); - let sections = &mut sections[..i]; sections.sort_unstable_by_key(|s| s.base); @@ -736,11 +761,12 @@ fn init_sections(mut allocator: BumpAllocator) { } } - let mut first_pages: [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] = - [None; ORDER_COUNT as usize]; - let mut last_pages = first_pages; - - let mut append_page = |page: Frame, info: &'static PageInfo, order| { + let mut append_page = |page: Frame, + info: &'static PageInfo, + order, + first_pages: &mut [Option<(Frame, &'static PageInfo)>; 11], + last_pages: &mut [Option<(Frame, &PageInfo)>; 11], + allocator: &mut BumpAllocator| { let this_page = (page, info); if page.base() < allocator.abs_offset() { @@ -776,76 +802,153 @@ fn init_sections(mut allocator: BumpAllocator) { }; } - for section in &*sections { - let mut base = section.base; - let mut frames = section.frames; + let mut free_list_fill = |region: Option, + allocator: &mut BumpAllocator| + -> [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] { + let mut first_pages: [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] = + [None; ORDER_COUNT as usize]; + let mut last_pages = first_pages; - for order in 0..=MAX_ORDER { - let pages_for_current_order = 1 << order; - - debug_assert_eq!(frames.len() % pages_for_current_order, 0); - debug_assert!(base.is_aligned_to_order(order)); - - if !frames.is_empty() && order != MAX_ORDER && !base.is_aligned_to_order(order + 1) { - frames[0].next.store(order as usize, Ordering::Relaxed); - // The first section page is not aligned to the next order size. - - //info!("ORDER {order}: FIRST {base:?}"); - append_page(base, &frames[0], order); - - base = base.next_by(pages_for_current_order); - frames = &frames[pages_for_current_order..]; - } else { - //info!("ORDER {order}: FIRST SKIP"); - } - - if !frames.is_empty() - && order != MAX_ORDER - && !base.next_by(frames.len()).is_aligned_to_order(order + 1) + for section in &*sections { + if let Some(region) = region + && (section.base.physaddr.get() < region.base.data() + || section.base.physaddr.get() >= region.base.data() + region.size) { - // The last section page is not aligned to the next order size. - - let off = frames.len() - pages_for_current_order; - let final_page = base.next_by(off); - - frames[off].next.store(order as usize, Ordering::Relaxed); - - //info!("ORDER {order}: LAST {final_page:?}"); - append_page(final_page, &frames[off], order); - - frames = &frames[..off]; - } else { - //info!("ORDER {order}: LAST SKIP"); + continue; } - if frames.is_empty() { - break; - } + let mut base = section.base; + let mut frames = section.frames; + + for order in 0..=MAX_ORDER { + let pages_for_current_order = 1 << order; - if order == MAX_ORDER { debug_assert_eq!(frames.len() % pages_for_current_order, 0); - debug_assert!(base.is_aligned_to_order(MAX_ORDER)); + debug_assert!(base.is_aligned_to_order(order)); - for (off, info) in frames.iter().enumerate().step_by(pages_for_current_order) { - info.next.store(MAX_ORDER as usize, Ordering::Relaxed); - append_page(base.next_by(off), info, MAX_ORDER); + if !frames.is_empty() && order != MAX_ORDER && !base.is_aligned_to_order(order + 1) + { + frames[0].next.store(order as usize, Ordering::Relaxed); + // The first section page is not aligned to the next order size. + + //info!("ORDER {order}: FIRST {base:?}"); + append_page( + base, + &frames[0], + order, + &mut first_pages, + &mut last_pages, + allocator, + ); + + base = base.next_by(pages_for_current_order); + frames = &frames[pages_for_current_order..]; + } else { + //info!("ORDER {order}: FIRST SKIP"); + } + + if !frames.is_empty() + && order != MAX_ORDER + && !base.next_by(frames.len()).is_aligned_to_order(order + 1) + { + // The last section page is not aligned to the next order size. + + let off = frames.len() - pages_for_current_order; + let final_page = base.next_by(off); + + frames[off].next.store(order as usize, Ordering::Relaxed); + + //info!("ORDER {order}: LAST {final_page:?}"); + append_page( + final_page, + &frames[off], + order, + &mut first_pages, + &mut last_pages, + allocator, + ); + + frames = &frames[..off]; + } else { + //info!("ORDER {order}: LAST SKIP"); + } + + if frames.is_empty() { + break; + } + + if order == MAX_ORDER { + debug_assert_eq!(frames.len() % pages_for_current_order, 0); + debug_assert!(base.is_aligned_to_order(MAX_ORDER)); + + for (off, info) in frames.iter().enumerate().step_by(pages_for_current_order) { + info.next.store(MAX_ORDER as usize, Ordering::Relaxed); + append_page( + base.next_by(off), + info, + MAX_ORDER, + &mut first_pages, + &mut last_pages, + allocator, + ); + } } } + + //info!("SECTION from {:?}, {} pages, array at {:p}", section.base, section.frames.len(), section.frames); + } + for (order, tuple_opt) in last_pages.iter().enumerate() { + let Some((frame, info)) = tuple_opt else { + continue; + }; + debug_assert!(frame.is_aligned_to_order(order as u32)); + let free = info.as_free().unwrap(); + debug_assert_eq!(free.prev().order(), order as u32); + free.set_next(P2Frame::new(None, order as u32)); } - //info!("SECTION from {:?}, {} pages, array at {:p}", section.base, section.frames.len(), section.frames); - } - for (order, tuple_opt) in last_pages.iter().enumerate() { - let Some((frame, info)) = tuple_opt else { - continue; + first_pages + }; + + let free_lists; + if let Some(regions) = numa::memory_regions() { + let free_list_page = allocator + .allocate(FrameCount::new( + (numa::number_of_memory_regions() * size_of::>()) + .div_ceil(PAGE_SIZE), + )) + .expect("Failed to allocate free list page"); + let va = unsafe { RmmA::phys_to_virt(free_list_page).data() as *mut Mutex }; + free_lists = unsafe { slice::from_raw_parts_mut(va, numa::number_of_memory_regions()) }; + for (i, region) in regions.enumerate() { + let free_list = FreeList { + for_orders: free_list_fill(Some(region), &mut allocator) + .map(|pair| pair.map(|(frame, _)| frame)), + used_frames: 0, + upper_limit: region.base.add(region.size).data(), + lower_limit: region.base.data(), + }; + free_lists[i] = Mutex::new(free_list); + } + } else { + let free_list_page = allocator + .allocate(FrameCount::new( + size_of::>().div_ceil(PAGE_SIZE), + )) + .expect("Failed to allocate free list page"); + let va = unsafe { RmmA::phys_to_virt(free_list_page).data() as *mut Mutex }; + free_lists = unsafe { slice::from_raw_parts_mut(va, 1) }; + let free_list = FreeList { + for_orders: free_list_fill(None, &mut allocator) + .map(|pair| pair.map(|(frame, _)| frame)), + used_frames: 0, + upper_limit: usize::MAX, + lower_limit: 0, }; - debug_assert!(frame.is_aligned_to_order(order as u32)); - let free = info.as_free().unwrap(); - debug_assert_eq!(free.prev().order(), order as u32); - free.set_next(P2Frame::new(None, order as u32)); + free_lists[0] = Mutex::new(free_list); } - FREELIST.lock().for_orders = first_pages.map(|pair| pair.map(|(frame, _)| frame)); + FREE_LISTS.call_once(|| free_lists); //debug_freelist(); debug!("Initial freelist consistent"); diff --git a/src/numa.rs b/src/numa.rs index b3265bb88d..1102842226 100644 --- a/src/numa.rs +++ b/src/numa.rs @@ -111,6 +111,9 @@ impl Iterator for NumaMemoryIter { fn next(&mut self) -> Option { let mem = self.mem.get(self.i)?; + if mem.length == 0 { + return None; + } self.i += 1; Some(MemoryArea { base: PhysicalAddress::new(mem.start), diff --git a/src/startup/memory.rs b/src/startup/memory.rs index 1f23227ffb..2a9fa4b589 100644 --- a/src/startup/memory.rs +++ b/src/startup/memory.rs @@ -1,8 +1,7 @@ -#[cfg(feature = "numa")] -use crate::numa; use crate::{ arch::CurrentRmmArch, memory::PAGE_SIZE, + numa, startup::{memory::BootloaderMemoryKind::Null, KernelArgs}, }; use core::{ diff --git a/src/startup/mod.rs b/src/startup/mod.rs index 26700e3312..7b3f532fa4 100644 --- a/src/startup/mod.rs +++ b/src/startup/mod.rs @@ -13,7 +13,6 @@ use crate::{ sync::CleanLockToken, }; -#[cfg(feature = "numa")] use crate::numa; pub mod memory; @@ -189,7 +188,6 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { } } - #[cfg(feature = "numa")] numa::dump_info(); run_userspace(&mut token) } From fbfe439451d685fb31e468bc39bf0d86b498fe33 Mon Sep 17 00:00:00 2001 From: R Aadarsh Date: Thu, 9 Jul 2026 19:26:01 +0530 Subject: [PATCH 5/7] Fix riscv and x86 compilation failure --- src/acpi/mod.rs | 2 ++ src/arch/x86_shared/start.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index 8e742b5da8..fd1e88b948 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -23,9 +23,11 @@ mod rsdp; mod rsdt; mod rxsdt; pub mod sdt; +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] pub mod slit; #[cfg(target_arch = "aarch64")] mod spcr; +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] pub mod srat; mod xsdt; diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index 3a7abbdc39..295c273501 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -104,7 +104,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { // Initialize RMM #[cfg(target_arch = "x86")] - let bump_allocator = + let mut bump_allocator = crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000)); #[cfg(target_arch = "x86_64")] let mut bump_allocator = crate::startup::memory::init(&args, Some(0x100000), None); From bb06db93ce405a2171bc2916202bbc22588db73b Mon Sep 17 00:00:00 2001 From: R Aadarsh Date: Fri, 10 Jul 2026 15:39:54 +0530 Subject: [PATCH 6/7] Add support for querying numa info from userspace --- src/numa.rs | 38 ++++++++++++++++++++++++++++++++++++++ src/scheme/sys/mod.rs | 2 ++ 2 files changed, 40 insertions(+) diff --git a/src/numa.rs b/src/numa.rs index 1102842226..4844ed8ff7 100644 --- a/src/numa.rs +++ b/src/numa.rs @@ -9,6 +9,7 @@ use alloc::{sync::Arc, vec::Vec}; use hashbrown::HashMap; use rmm::{Arch, BumpAllocator, MemoryArea, PhysicalAddress}; use spin::once::Once; +use syscall::{Error, Result, ENODATA, EOPNOTSUPP}; pub const MAX_DOMAINS: usize = 128; @@ -184,3 +185,40 @@ pub fn nearest_preceding_memory_region(addr: usize, overlap: bool) -> Option<&'s }) .max_by_key(|e| e.start) } + +pub fn get_numa_info(token: &mut CleanLockToken) -> Result> { + let cpu_info = NUMA_CPUS + .get() + .ok_or(Error::new(EOPNOTSUPP))? + .iter() + .map(|e| e.to_ne_bytes()) + .flatten() + .collect::>(); + let mem_info = NUMA_MEMORY + .get() + .ok_or(Error::new(EOPNOTSUPP))? + .iter() + .map(|e| { + [ + e.start.to_ne_bytes(), + e.length.to_ne_bytes(), + (e.node_id as usize).to_ne_bytes(), + ] + }) + .flatten() + .flatten() + .collect::>(); + let mut numa_info = Vec::new(); + numa_info.extend(cpu_info); + numa_info.extend(mem_info); + Ok(numa_info) +} + +pub fn get_numa_distance_info(token: &mut CleanLockToken) -> Result> { + Ok(DISTANCES + .get() + .ok_or(Error::new(ENODATA))? + .iter() + .map(|e| *e) + .collect()) +} diff --git a/src/scheme/sys/mod.rs b/src/scheme/sys/mod.rs index fa44c95cbb..85aee697c1 100644 --- a/src/scheme/sys/mod.rs +++ b/src/scheme/sys/mod.rs @@ -76,6 +76,8 @@ const FILES: &[(&str, Kind)] = &[ ("iostat", Rd(iostat::resource)), ("irq", Rd(irq::resource)), ("log", Rd(log::resource)), + ("numa", Rd(crate::numa::get_numa_info)), + ("numa_dist", Rd(crate::numa::get_numa_distance_info)), ("syscall", Rd(syscall::resource)), ("uname", Rd(uname::resource)), ("env", Rd(|_| Ok(Vec::from(crate::startup::init_env())))), From 985bc2623fe390fb57f88dca34a9cc38e11cbb2a Mon Sep 17 00:00:00 2001 From: MJ Pooladkhay Date: Sat, 11 Jul 2026 01:13:36 +0100 Subject: [PATCH 7/7] Fix a race in get_round_robin_index. --- src/memory/mod.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 16c85d694b..c6f4e99216 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -93,12 +93,9 @@ pub fn allocate_frame() -> Option { } fn get_round_robin_index() -> usize { - static CURRENT_INDEX: AtomicU8 = AtomicU8::new(0); - if CURRENT_INDEX.load(Ordering::Acquire) as usize == FREE_LISTS.get().unwrap().len() { - CURRENT_INDEX.store(1, Ordering::Release); - return 0; - } - CURRENT_INDEX.fetch_add(1, Ordering::AcqRel) as usize + static CURRENT_INDEX: AtomicUsize = AtomicUsize::new(0); + let len = FREE_LISTS.get().unwrap().len(); + CURRENT_INDEX.fetch_add(1, Ordering::Relaxed) % len } // TODO: Flags, strategy