From bfbbec47a982626b7d482bc9e22387d89ae3cc5a Mon Sep 17 00:00:00 2001 From: R Aadarsh Date: Wed, 8 Jul 2026 21:23:22 +0530 Subject: [PATCH] Create seperate free lists for each NUMA memory regions --- Cargo.toml | 1 - src/acpi/mod.rs | 2 - src/arch/x86_shared/start.rs | 2 - src/main.rs | 1 - src/memory/mod.rs | 301 +++++++++++++++++++++++------------ src/numa.rs | 3 + src/startup/memory.rs | 3 +- src/startup/mod.rs | 2 - 8 files changed, 206 insertions(+), 109 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ea56939857..825b56a74a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,6 @@ default = [ "acpi", #"debugger", "multi_core", - # "numa", "serial_debug", "self_modifying", "x86_kvm_pv", diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index 34125f31b4..8e742b5da8 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -23,11 +23,9 @@ mod rsdp; mod rsdt; mod rxsdt; pub mod sdt; -#[cfg(feature = "numa")] pub mod slit; #[cfg(target_arch = "aarch64")] mod spcr; -#[cfg(feature = "numa")] pub mod srat; mod xsdt; diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index 9d8ec3bf21..3a7abbdc39 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -12,7 +12,6 @@ use crate::{ startup::KernelArgs, }; -#[cfg(feature = "numa")] use crate::numa; /// Test of zero values in BSS. @@ -117,7 +116,6 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { crate::acpi::init_before_mem(args.acpi_rsdp()); } - #[cfg(feature = "numa")] numa::init(&mut bump_allocator); crate::memory::init_mm(bump_allocator); diff --git a/src/main.rs b/src/main.rs index 23a60c006a..8380e02f94 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,7 +69,6 @@ mod log; mod memory; /// NUMA support -#[cfg(feature = "numa")] mod numa; /// Panic diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 6ce90a58eb..16c85d694b 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -5,21 +5,21 @@ use core::{ cell::SyncUnsafeCell, num::NonZeroUsize, ops::AddAssign, - sync::atomic::{AtomicUsize, Ordering}, + slice, + sync::atomic::{AtomicU8, AtomicUsize, Ordering}, }; pub use kernel_mapper::KernelMapper; -use spin::Mutex; +use spin::{once::Once, Mutex}; pub use crate::arch::CurrentRmmArch as RmmA; -#[cfg(feature = "numa")] -use crate::numa; use crate::{ context::{ self, memory::{AccessMode, PfError}, }, kernel_executable_offsets::{__usercopy_end, __usercopy_start}, + numa, sync::CleanLockToken, syscall::error::{Error, ENOMEM}, }; @@ -57,7 +57,15 @@ pub fn free_frames() -> usize { /// Get the number of frames used pub fn used_frames() -> usize { // TODO: Include bump allocator static pages? - FREELIST.lock().used_frames + FREE_LISTS + .get() + .unwrap() + .iter() + .map(|e| { + let l = e.lock(); + l.used_frames + }) + .sum() } pub fn total_frames() -> usize { // TODO: Include bump allocator static pages? @@ -66,19 +74,42 @@ pub fn total_frames() -> usize { /// Allocate a range of frames pub fn allocate_p2frame(order: u32) -> Option { - allocate_p2frame_complex(order, (), None, order).map(|(f, _)| f) + let initial_index = get_round_robin_index(); + let mut index = initial_index; + + loop { + if let Some(frame) = allocate_p2frame_complex(order, (), None, order, index).map(|(f, _)| f) + { + return Some(frame); + } + index = get_round_robin_index(); + if index == initial_index { + return None; + } + } } pub fn allocate_frame() -> Option { allocate_p2frame(0) } + +fn get_round_robin_index() -> usize { + static CURRENT_INDEX: AtomicU8 = AtomicU8::new(0); + if CURRENT_INDEX.load(Ordering::Acquire) as usize == FREE_LISTS.get().unwrap().len() { + CURRENT_INDEX.store(1, Ordering::Release); + return 0; + } + CURRENT_INDEX.fetch_add(1, Ordering::AcqRel) as usize +} + // TODO: Flags, strategy pub fn allocate_p2frame_complex( _req_order: u32, _flags: (), _strategy: Option<()>, min_order: u32, + index: usize, ) -> Option<(Frame, usize)> { - let mut freelist = FREELIST.lock(); + let mut freelist = FREE_LISTS.get().unwrap()[index].lock(); let (frame_order, frame) = freelist .for_orders @@ -151,8 +182,19 @@ pub fn allocate_p2frame_complex( Some((frame, PAGE_SIZE << min_order)) } +fn get_index_for_deallocation(addr: usize) -> Option { + for (i, list) in FREE_LISTS.get().unwrap().iter().enumerate() { + let l = list.lock(); + if addr >= l.lower_limit && addr < l.upper_limit { + return Some(i); + } + } + None +} + pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) { - let mut freelist = FREELIST.lock(); + let index = get_index_for_deallocation(orig_frame.physaddr.get()).expect("Expected an index"); + let mut freelist = FREE_LISTS.get().unwrap()[index].lock(); let initial_info = get_page_info(orig_frame) .unwrap_or_else(|| panic!("missing PageInfo for {orig_frame:?} being freed")); @@ -483,13 +525,12 @@ struct AllocatorData { } #[derive(Debug)] struct FreeList { + upper_limit: usize, + lower_limit: usize, for_orders: [Option; ORDER_COUNT as usize], used_frames: usize, } -static FREELIST: Mutex = Mutex::new(FreeList { - for_orders: [None; ORDER_COUNT as usize], - used_frames: 0, -}); +static FREE_LISTS: Once<&'static [Mutex]> = Once::new(); pub struct Section { base: Frame, @@ -506,16 +547,7 @@ const _: () = { #[cold] fn init_sections(mut allocator: BumpAllocator) { - let number_of_memory_regions = { - #[cfg(feature = "numa")] - { - numa::number_of_memory_regions() - } - #[cfg(not(feature = "numa"))] - { - 0 - } - }; + let number_of_memory_regions = numa::number_of_memory_regions(); let (free_areas, offset_into_first_free_area) = allocator.free_areas(); @@ -542,18 +574,15 @@ fn init_sections(mut allocator: BumpAllocator) { .next_multiple_of(MAX_SECTION_SIZE); let aligned_start = area.base.data() / MAX_SECTION_SIZE * MAX_SECTION_SIZE; - #[cfg(feature = "numa")] - { - if number_of_memory_regions > 0 { - if let Some(next_memory_region) = - numa::nearest_next_memory_region(area.base.data(), false) - && next_memory_region.start < area.base.add(area.size).data() - && next_memory_region.start + next_memory_region.length - > area.base.add(area.size).data() - && !next_memory_region.start.is_multiple_of(MAX_SECTION_SIZE) - { - return (aligned_end - aligned_start) / MAX_SECTION_SIZE + 1; - } + if number_of_memory_regions > 0 { + if let Some(next_memory_region) = + numa::nearest_next_memory_region(area.base.data(), false) + && next_memory_region.start < area.base.add(area.size).data() + && next_memory_region.start + next_memory_region.length + > area.base.add(area.size).data() + && !next_memory_region.start.is_multiple_of(MAX_SECTION_SIZE) + { + return (aligned_end - aligned_start) / MAX_SECTION_SIZE + 1; } } @@ -695,7 +724,6 @@ fn init_sections(mut allocator: BumpAllocator) { }; let mut i = 0; - #[cfg(feature = "numa")] if number_of_memory_regions > 0 && let Some(regions) = numa::memory_regions() { @@ -712,9 +740,6 @@ fn init_sections(mut allocator: BumpAllocator) { sections_fill(None, &mut i, false); } - #[cfg(not(feature = "numa"))] - sections_fill(None, &mut i, false); - let sections = &mut sections[..i]; sections.sort_unstable_by_key(|s| s.base); @@ -736,11 +761,12 @@ fn init_sections(mut allocator: BumpAllocator) { } } - let mut first_pages: [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] = - [None; ORDER_COUNT as usize]; - let mut last_pages = first_pages; - - let mut append_page = |page: Frame, info: &'static PageInfo, order| { + let mut append_page = |page: Frame, + info: &'static PageInfo, + order, + first_pages: &mut [Option<(Frame, &'static PageInfo)>; 11], + last_pages: &mut [Option<(Frame, &PageInfo)>; 11], + allocator: &mut BumpAllocator| { let this_page = (page, info); if page.base() < allocator.abs_offset() { @@ -776,76 +802,153 @@ fn init_sections(mut allocator: BumpAllocator) { }; } - for section in &*sections { - let mut base = section.base; - let mut frames = section.frames; + let mut free_list_fill = |region: Option, + allocator: &mut BumpAllocator| + -> [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] { + let mut first_pages: [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] = + [None; ORDER_COUNT as usize]; + let mut last_pages = first_pages; - for order in 0..=MAX_ORDER { - let pages_for_current_order = 1 << order; - - debug_assert_eq!(frames.len() % pages_for_current_order, 0); - debug_assert!(base.is_aligned_to_order(order)); - - if !frames.is_empty() && order != MAX_ORDER && !base.is_aligned_to_order(order + 1) { - frames[0].next.store(order as usize, Ordering::Relaxed); - // The first section page is not aligned to the next order size. - - //info!("ORDER {order}: FIRST {base:?}"); - append_page(base, &frames[0], order); - - base = base.next_by(pages_for_current_order); - frames = &frames[pages_for_current_order..]; - } else { - //info!("ORDER {order}: FIRST SKIP"); - } - - if !frames.is_empty() - && order != MAX_ORDER - && !base.next_by(frames.len()).is_aligned_to_order(order + 1) + for section in &*sections { + if let Some(region) = region + && (section.base.physaddr.get() < region.base.data() + || section.base.physaddr.get() >= region.base.data() + region.size) { - // The last section page is not aligned to the next order size. - - let off = frames.len() - pages_for_current_order; - let final_page = base.next_by(off); - - frames[off].next.store(order as usize, Ordering::Relaxed); - - //info!("ORDER {order}: LAST {final_page:?}"); - append_page(final_page, &frames[off], order); - - frames = &frames[..off]; - } else { - //info!("ORDER {order}: LAST SKIP"); + continue; } - if frames.is_empty() { - break; - } + let mut base = section.base; + let mut frames = section.frames; + + for order in 0..=MAX_ORDER { + let pages_for_current_order = 1 << order; - if order == MAX_ORDER { debug_assert_eq!(frames.len() % pages_for_current_order, 0); - debug_assert!(base.is_aligned_to_order(MAX_ORDER)); + debug_assert!(base.is_aligned_to_order(order)); - for (off, info) in frames.iter().enumerate().step_by(pages_for_current_order) { - info.next.store(MAX_ORDER as usize, Ordering::Relaxed); - append_page(base.next_by(off), info, MAX_ORDER); + if !frames.is_empty() && order != MAX_ORDER && !base.is_aligned_to_order(order + 1) + { + frames[0].next.store(order as usize, Ordering::Relaxed); + // The first section page is not aligned to the next order size. + + //info!("ORDER {order}: FIRST {base:?}"); + append_page( + base, + &frames[0], + order, + &mut first_pages, + &mut last_pages, + allocator, + ); + + base = base.next_by(pages_for_current_order); + frames = &frames[pages_for_current_order..]; + } else { + //info!("ORDER {order}: FIRST SKIP"); + } + + if !frames.is_empty() + && order != MAX_ORDER + && !base.next_by(frames.len()).is_aligned_to_order(order + 1) + { + // The last section page is not aligned to the next order size. + + let off = frames.len() - pages_for_current_order; + let final_page = base.next_by(off); + + frames[off].next.store(order as usize, Ordering::Relaxed); + + //info!("ORDER {order}: LAST {final_page:?}"); + append_page( + final_page, + &frames[off], + order, + &mut first_pages, + &mut last_pages, + allocator, + ); + + frames = &frames[..off]; + } else { + //info!("ORDER {order}: LAST SKIP"); + } + + if frames.is_empty() { + break; + } + + if order == MAX_ORDER { + debug_assert_eq!(frames.len() % pages_for_current_order, 0); + debug_assert!(base.is_aligned_to_order(MAX_ORDER)); + + for (off, info) in frames.iter().enumerate().step_by(pages_for_current_order) { + info.next.store(MAX_ORDER as usize, Ordering::Relaxed); + append_page( + base.next_by(off), + info, + MAX_ORDER, + &mut first_pages, + &mut last_pages, + allocator, + ); + } } } + + //info!("SECTION from {:?}, {} pages, array at {:p}", section.base, section.frames.len(), section.frames); + } + for (order, tuple_opt) in last_pages.iter().enumerate() { + let Some((frame, info)) = tuple_opt else { + continue; + }; + debug_assert!(frame.is_aligned_to_order(order as u32)); + let free = info.as_free().unwrap(); + debug_assert_eq!(free.prev().order(), order as u32); + free.set_next(P2Frame::new(None, order as u32)); } - //info!("SECTION from {:?}, {} pages, array at {:p}", section.base, section.frames.len(), section.frames); - } - for (order, tuple_opt) in last_pages.iter().enumerate() { - let Some((frame, info)) = tuple_opt else { - continue; + first_pages + }; + + let free_lists; + if let Some(regions) = numa::memory_regions() { + let free_list_page = allocator + .allocate(FrameCount::new( + (numa::number_of_memory_regions() * size_of::>()) + .div_ceil(PAGE_SIZE), + )) + .expect("Failed to allocate free list page"); + let va = unsafe { RmmA::phys_to_virt(free_list_page).data() as *mut Mutex }; + free_lists = unsafe { slice::from_raw_parts_mut(va, numa::number_of_memory_regions()) }; + for (i, region) in regions.enumerate() { + let free_list = FreeList { + for_orders: free_list_fill(Some(region), &mut allocator) + .map(|pair| pair.map(|(frame, _)| frame)), + used_frames: 0, + upper_limit: region.base.add(region.size).data(), + lower_limit: region.base.data(), + }; + free_lists[i] = Mutex::new(free_list); + } + } else { + let free_list_page = allocator + .allocate(FrameCount::new( + size_of::>().div_ceil(PAGE_SIZE), + )) + .expect("Failed to allocate free list page"); + let va = unsafe { RmmA::phys_to_virt(free_list_page).data() as *mut Mutex }; + free_lists = unsafe { slice::from_raw_parts_mut(va, 1) }; + let free_list = FreeList { + for_orders: free_list_fill(None, &mut allocator) + .map(|pair| pair.map(|(frame, _)| frame)), + used_frames: 0, + upper_limit: usize::MAX, + lower_limit: 0, }; - debug_assert!(frame.is_aligned_to_order(order as u32)); - let free = info.as_free().unwrap(); - debug_assert_eq!(free.prev().order(), order as u32); - free.set_next(P2Frame::new(None, order as u32)); + free_lists[0] = Mutex::new(free_list); } - FREELIST.lock().for_orders = first_pages.map(|pair| pair.map(|(frame, _)| frame)); + FREE_LISTS.call_once(|| free_lists); //debug_freelist(); debug!("Initial freelist consistent"); diff --git a/src/numa.rs b/src/numa.rs index b3265bb88d..1102842226 100644 --- a/src/numa.rs +++ b/src/numa.rs @@ -111,6 +111,9 @@ impl Iterator for NumaMemoryIter { fn next(&mut self) -> Option { let mem = self.mem.get(self.i)?; + if mem.length == 0 { + return None; + } self.i += 1; Some(MemoryArea { base: PhysicalAddress::new(mem.start), diff --git a/src/startup/memory.rs b/src/startup/memory.rs index 1f23227ffb..2a9fa4b589 100644 --- a/src/startup/memory.rs +++ b/src/startup/memory.rs @@ -1,8 +1,7 @@ -#[cfg(feature = "numa")] -use crate::numa; use crate::{ arch::CurrentRmmArch, memory::PAGE_SIZE, + numa, startup::{memory::BootloaderMemoryKind::Null, KernelArgs}, }; use core::{ diff --git a/src/startup/mod.rs b/src/startup/mod.rs index 26700e3312..7b3f532fa4 100644 --- a/src/startup/mod.rs +++ b/src/startup/mod.rs @@ -13,7 +13,6 @@ use crate::{ sync::CleanLockToken, }; -#[cfg(feature = "numa")] use crate::numa; pub mod memory; @@ -189,7 +188,6 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { } } - #[cfg(feature = "numa")] numa::dump_info(); run_userspace(&mut token) }