diff --git a/src/acpi/slit.rs b/src/acpi/slit.rs index 60d48e3e57..03c91597da 100644 --- a/src/acpi/slit.rs +++ b/src/acpi/slit.rs @@ -1,10 +1,13 @@ use crate::{ - acpi::sdt::Sdt, + acpi::{rxsdt::Rxsdt, sdt::Sdt, RXSDT_ENUM}, find_one_sdt, - numa::{self, NumaNode, NUMA_NODES}, + memory::{round_up_pages, PAGE_SIZE}, + numa::{self}, }; -use core::ops::Add; +use core::{ops::Add, slice}; use hashbrown::HashMap; +use rmm::{Arch, BumpAllocator, FrameAllocator, FrameCount}; +use spin::once::Once; #[derive(Debug)] pub struct Slit { @@ -21,27 +24,21 @@ impl Slit { address: (sdt.data_address() + 8) as *const u8, } } - pub fn init(&self, numa_nodes: &mut HashMap) { - let address = self.address; - let ndom = numa_nodes.len() as u32; + pub fn init(&self, allocator: &mut BumpAllocator) -> &'static mut [u8] { + unsafe { slice::from_raw_parts_mut(self.address as *mut u8, (self.no * self.no) as usize) } + } +} - for i in 0..ndom { - for j in i..ndom { - // ignore distances from a domain to itself, since it is always 10 - if i != j { - numa::set_distance(numa_nodes, i, j, unsafe { - *address.add((i + ndom * j) as usize) - }); - numa::set_distance(numa_nodes, j, i, unsafe { - *address.add((i + ndom * j) as usize) - }); - } +pub fn init(allocator: &mut BumpAllocator, distances: &Once<&'static [u8]>) { + if let Some(rxsdt) = RXSDT_ENUM.get() { + for sdt_addr in rxsdt.iter() { + let sdt = + unsafe { &*(crate::memory::RmmA::phys_to_virt(sdt_addr).data() as *const Sdt) }; + if &sdt.signature == b"SLIT" { + let slit = Slit::new(sdt); + distances.call_once(|| slit.init(allocator)); + return; } } } } - -pub fn init(numa_nodes: &mut HashMap) { - let slit = Slit::new(find_one_sdt!("SLIT")); - slit.init(numa_nodes); -} diff --git a/src/acpi/srat/aarch64.rs b/src/acpi/srat/aarch64.rs index 0b32481d20..e6b4e6363e 100644 --- a/src/acpi/srat/aarch64.rs +++ b/src/acpi/srat/aarch64.rs @@ -1,5 +1,8 @@ use crate::acpi::srat::Srat; -pub fn init_srat(srat: &Srat) { +pub fn init_srat( + allocator: &mut BumpAllocator, + srat: &Srat, +) -> (&'static [u32], &'static [u32], &'static [NumaMemory]) { // todo } diff --git a/src/acpi/srat/mod.rs b/src/acpi/srat/mod.rs index cf88935f00..d613f2c1aa 100644 --- a/src/acpi/srat/mod.rs +++ b/src/acpi/srat/mod.rs @@ -1,11 +1,13 @@ //! See use hashbrown::HashMap; +use rmm::{Arch, BumpAllocator}; +use spin::once::Once; use crate::{ - acpi::{find_sdt, sdt::Sdt, srat}, - find_one_sdt, - numa::{NumaNode, NUMA_NODES}, + acpi::{find_sdt, get_sdt_signature, rxsdt::Rxsdt, sdt::Sdt, srat, RXSDT_ENUM}, + find_one_sdt, memory, + numa::NumaMemory, }; #[cfg(target_arch = "aarch64")] @@ -22,9 +24,24 @@ pub struct Srat { entries: *const u8, } -pub fn init(numa_nodes: &mut HashMap) { - let srat = Srat::new(find_one_sdt!("SRAT")); - arch::init_srat(numa_nodes, &srat); +pub fn init( + allocator: &mut BumpAllocator, + map: &Once<&'static [u32]>, + cpus: &Once<&'static [u32]>, + mem: &Once<&'static [NumaMemory]>, +) { + if let Some(rxsdt) = RXSDT_ENUM.get() { + for sdt_addr in rxsdt.iter() { + let sdt = unsafe { &*(memory::RmmA::phys_to_virt(sdt_addr).data() as *const Sdt) }; + if &sdt.signature == b"SRAT" { + let (a, b, c) = arch::init_srat(allocator, &Srat::new(sdt)); + map.call_once(|| a); + cpus.call_once(|| b); + mem.call_once(|| c); + return; + } + } + } } impl Srat { diff --git a/src/acpi/srat/x86.rs b/src/acpi/srat/x86.rs index c1b73f973b..e65e71c4be 100644 --- a/src/acpi/srat/x86.rs +++ b/src/acpi/srat/x86.rs @@ -1,10 +1,13 @@ -use core::iter; +use core::{iter, slice}; use hashbrown::HashMap; +use rmm::{Arch, BumpAllocator, FrameAllocator}; use crate::{ acpi::srat::{to_usize, Srat, SratEntry}, - numa::{self, NumaNode, NUMA_NODES}, + cpu_set, + memory::{self, PAGE_SIZE}, + numa::{self, NumaMemory}, }; #[inline(always)] @@ -15,39 +18,147 @@ fn to_single_int(high: &[u8; 3], low: u8) -> u32 { u32::from_le_bytes(high_and_low) } -pub fn init_srat(numa_nodes: &mut HashMap, srat: &Srat) { +pub fn init_srat( + allocator: &mut BumpAllocator, + srat: &Srat, +) -> (&'static [u32], &'static [u32], &'static [NumaMemory]) { + let dom_node_map = allocator + .allocate(rmm::FrameCount::new( + memory::round_up_pages(numa::MAX_DOMAINS * size_of::()) / PAGE_SIZE, + )) + .expect("Failed to allocate memory for storing NUMA info"); + let mut mapper = unsafe { rmm::PageMapper::current(rmm::TableKind::Kernel, allocator) }; + let mut flags = rmm::PageFlags::::new(); + let flags = flags.write(true); + let dom_node_map_ptr = unsafe { + let (va, flush) = mapper + .map_linearly(dom_node_map, flags) + .expect("Failed to map NUMA info pages"); + flush.flush(); + va.data() as *mut u32 + }; + // Occupies 512 bytes (1/8th of a page) + let dom_node_map: &'static mut [u32] = + unsafe { slice::from_raw_parts_mut(dom_node_map_ptr, numa::MAX_DOMAINS) }; + dom_node_map.fill(u32::MAX); + + let mut cpu_count = 0; + let mut memory_count = 0; + + srat.into_iter().for_each(|e| match e { + SratEntry::LegacyProcessorLocalAffinity(legacy_processor_local_affinity) => cpu_count += 1, + SratEntry::MemoryAffinity(memory_affinity) => memory_count += 1, + SratEntry::ProcessorLocalAffinity(processor_local_affinity) => todo!(), + _ => (), + }); + + assert!( + memory_count <= numa::MAX_DOMAINS, + "Found {} memory blocks while only a maximum of {} are supported", + memory_count, + numa::MAX_DOMAINS + ); + + assert!( + cpu_count <= cpu_set::MAX_CPU_COUNT, + "Found more number of CPUs than supported" + ); + + // occupies 512 bytes (1/8th of a page) + let cpus: &'static mut [u32] = unsafe { + slice::from_raw_parts_mut( + dom_node_map_ptr.add(numa::MAX_DOMAINS) as *mut u32, + numa::MAX_DOMAINS, + ) + }; + + cpus.fill(u32::MAX); + + // total occupied till now: 1024 bytes, remaining 3072 bytes, can accomodate 128 memory entries + + let memories: &'static mut [NumaMemory] = unsafe { + slice::from_raw_parts_mut( + cpus.as_ptr().add(numa::MAX_DOMAINS) as *mut NumaMemory, + numa::MAX_DOMAINS, + ) + }; + + memories.fill(NumaMemory { + start: 0, + length: 0, + dom: 0, + _pad: [0; 4], + }); + for affinity in srat { match affinity { SratEntry::LegacyProcessorLocalAffinity(legacy_processor_local_affinity) => { - numa::add_cpu( - numa_nodes, - legacy_processor_local_affinity.apic_id as u32, - to_single_int( - &legacy_processor_local_affinity.proximity_domain_high, - legacy_processor_local_affinity.proximity_domain_low, - ), - ) + if legacy_processor_local_affinity.flags & 1 == 0 { + // processor disabled + continue; + } + let dom = to_single_int( + &legacy_processor_local_affinity.proximity_domain_high, + legacy_processor_local_affinity.proximity_domain_low, + ); + if dom_node_map[dom as usize] == u32::MAX { + let node_id = numa::assign_node_id(true); + dom_node_map[dom as usize] = node_id as u32; + } + cpus[legacy_processor_local_affinity.apic_id as usize] = dom_node_map[dom as usize]; } SratEntry::MemoryAffinity(memory_affinity) => { + if memory_affinity.flags & 1 == 0 { + // memory is not enabled + continue; + } + if memory_affinity.flags & (1 << 1) != 0 { + // memory is hot-pluggable + continue; + } + let dom = memory_affinity.proximity_domain; if memory_affinity.length_low == 0 { continue; } - numa::add_memory( - numa_nodes, - memory_affinity.proximity_domain, - to_usize( - memory_affinity.base_address_low, - memory_affinity.base_address_high, - ), - to_usize(memory_affinity.length_low, memory_affinity.length_high), + let start = to_usize( + memory_affinity.base_address_low, + memory_affinity.base_address_high, ); + let length = to_usize(memory_affinity.length_low, memory_affinity.length_high); + if dom_node_map[dom as usize] == u32::MAX { + let node_id = numa::assign_node_id(true); + dom_node_map[dom as usize] = node_id as u32; + } + memories[dom_node_map[dom as usize] as usize] = numa::NumaMemory { + start, + length, + dom: dom_node_map[dom as usize], + _pad: [0u8; 4], + }; + } + SratEntry::ProcessorLocalAffinity(processor_local_affinity) => { + if processor_local_affinity.flags & 1 == 0 { + // processor disabled + continue; + } + let dom = processor_local_affinity.proximity_domain; + if dom_node_map[dom as usize] == u32::MAX { + let node_id = numa::assign_node_id(true); + dom_node_map[dom as usize] = node_id as u32; + } + cpus[dom_node_map[dom as usize] as usize] = + processor_local_affinity.proximity_domain; } - SratEntry::ProcessorLocalAffinity(processor_local_affinity) => numa::add_cpu( - numa_nodes, - processor_local_affinity.x2apic_id, - processor_local_affinity.proximity_domain, - ), _ => continue, } } + let mut flags = rmm::PageFlags::::new(); + let flags = flags.write(false); + let flush = unsafe { + mapper + .remap(rmm::VirtualAddress::new(dom_node_map_ptr.addr()), flags) + .expect("Unable to make NUMA info page read-only") + }; + flush.flush(); + (dom_node_map, cpus, memories) } diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index 7c4ff43133..9d8ec3bf21 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -12,6 +12,9 @@ use crate::{ startup::KernelArgs, }; +#[cfg(feature = "numa")] +use crate::numa; + /// Test of zero values in BSS. static BSS_TEST_ZERO: SyncUnsafeCell = SyncUnsafeCell::new(0); /// Test of non-zero values in data. @@ -105,7 +108,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { let bump_allocator = crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000)); #[cfg(target_arch = "x86_64")] - let bump_allocator = crate::startup::memory::init(&args, Some(0x100000), None); + let mut bump_allocator = crate::startup::memory::init(&args, Some(0x100000), None); // Initialize paging paging::init(); @@ -114,6 +117,9 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { crate::acpi::init_before_mem(args.acpi_rsdp()); } + #[cfg(feature = "numa")] + numa::init(&mut bump_allocator); + crate::memory::init_mm(bump_allocator); #[cfg(target_arch = "x86_64")] diff --git a/src/numa.rs b/src/numa.rs index 33bc3b9ff9..df5a326fb9 100644 --- a/src/numa.rs +++ b/src/numa.rs @@ -5,14 +5,22 @@ use crate::{ }; use alloc::{sync::Arc, vec::Vec}; use hashbrown::HashMap; +use rmm::{Arch, BumpAllocator}; use spin::once::Once; -pub static NUMA_NODES: Once> = Once::new(); +pub const MAX_DOMAINS: usize = 128; -#[derive(Debug)] +static DOMAIN_NODE_MAP: Once<&'static [u32]> = Once::new(); +static NUMA_CPUS: Once<&'static [u32]> = Once::new(); +static NUMA_MEMORY: Once<&'static [NumaMemory]> = Once::new(); +static DISTANCES: Once<&'static [u8]> = Once::new(); + +#[derive(Debug, Clone)] pub struct NumaMemory { pub start: usize, pub length: usize, + pub dom: u32, + pub _pad: [u8; 4], } #[derive(Debug)] @@ -20,168 +28,63 @@ pub struct NumaCpu { pub id: u32, } -#[derive(Default, Debug)] -/// Represents a single NUMA logical node. A node is different from a domain. NUMA domain -/// refers to what exists physically. A NUMA node on the other hand is a logical one, with domains having -/// only CPUs or memory grouped together with other CPUs or memories. -/// -/// See the function `reorganise` below. -pub struct NumaNode { - cpus: Vec, - memory: Vec, - distances: Vec<(u32, u8)>, +pub fn init(allocator: &mut BumpAllocator) { + #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] + { + acpi::srat::init(allocator, &DOMAIN_NODE_MAP, &NUMA_CPUS, &NUMA_MEMORY); + acpi::slit::init(allocator, &DISTANCES); + } } -pub fn init() { - NUMA_NODES.call_once(|| { - let mut numa_nodes = HashMap::new(); - #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] - { - acpi::srat::init(&mut numa_nodes); - acpi::slit::init(&mut numa_nodes); - - sort_by_distances(&mut numa_nodes); - reorganise(&mut numa_nodes); - shrink(&mut numa_nodes); +pub fn assign_node_id(modify: bool) -> u8 { + static mut NODE_ID: u8 = 0; + if unsafe { NODE_ID } >= 128 { + panic!("Maximum number of domains supported is 128"); + } + unsafe { + NODE_ID += 1; + let return_value = NODE_ID - 1; + if !modify { + NODE_ID -= 1; } - - #[cfg(any(target_arch = "riscv64", target_arch = "aarch64"))] - { - // todo!() - } - - numa_nodes - }); + return_value + } } -pub fn add_cpu(numa_nodes: &mut HashMap, id: u32, node_id: u32) { - if let Some(node) = numa_nodes.get_mut(&id) { - node.cpus.push(NumaCpu { id }); +pub fn domain_to_node_id(domain_id: u32) -> Option { + Some(*DOMAIN_NODE_MAP.get()?.get(domain_id as usize)?) +} + +pub fn cpu_belongs_to_which_node(cpu_id: usize) -> Option { + Some(*NUMA_CPUS.get()?.get(cpu_id)?) +} + +/// A helper function that prints information about NUMA - available nodes, cpus and memory blocks in them +/// their starts and lengths +pub fn dump_info() { + if let Some(map) = DOMAIN_NODE_MAP.get() + && let Some(cpus) = NUMA_CPUS.get() + && let Some(memories) = NUMA_MEMORY.get() + { + println!("Number of NUMA nodes: {}", assign_node_id(false)); + for i in 0..cpus.len() { + if cpus[i] == u32::MAX { + continue; + } + println!("CPU {} : Node {}", i, cpus[i]) + } + for i in 0..memories.len() { + if memories[i].length == 0 { + continue; + } + println!( + "Memory Block starting at address {:#x} of size {:#x} bytes : Node {}", + memories[i].start, memories[i].length, memories[i].dom + ); + } } else { - let mut cpus = Vec::new(); - cpus.push(NumaCpu { id }); - numa_nodes.insert( - node_id, - NumaNode { - cpus, - memory: Vec::new(), - distances: Vec::new(), - }, + println!( + "The system has either no support for NUMA or there was an error during initialisation" ); } } - -pub fn add_memory( - numa_nodes: &mut HashMap, - node_id: u32, - start: usize, - length: usize, -) { - if let Some(node) = numa_nodes.get_mut(&node_id) { - node.memory.push(NumaMemory { start, length }); - } else { - let mut memory = Vec::new(); - memory.push(NumaMemory { start, length }); - numa_nodes.insert( - node_id, - NumaNode { - cpus: Vec::new(), - memory, - distances: Vec::new(), - }, - ); - } -} - -pub fn set_distance(nodes: &mut HashMap, src: u32, target: u32, distance: u8) { - let src = nodes.get_mut(&src).unwrap(); - src.distances.push((target, distance)); -} - -fn shrink(nodes: &mut HashMap) { - nodes.shrink_to_fit(); - - for (id, node) in nodes { - node.cpus.shrink_to_fit(); - node.distances.shrink_to_fit(); - node.memory.shrink_to_fit(); - } -} - -/// Reorganises CPUs and memories into nodes. If a NUMA domain has only a CPU but no memory, it is -/// put into a node with a memory that is nearest to it. Similarly, if a NUMA domain has only memory but no -/// CPUs, the memory is put into a node that has a CPU that is nearest to it. -/// -/// See the comment above the definition of `NumaNode`. -fn reorganise(nodes: &mut HashMap) { - let ids = nodes.keys().map(|e| *e).collect::>(); - - for id in ids { - let node = nodes.remove(&id).unwrap(); - - if node.cpus.len() == 0 { - assert!(node.memory.len() != 0); - put_for_adoption(nodes, node.distances, Some(node.memory), None, id); - } else if node.memory.len() == 0 { - put_for_adoption(nodes, node.distances, None, Some(node.cpus), id); - } else { - nodes.insert(id, node); - } - } -} - -fn put_for_adoption( - nodes: &mut HashMap, - distances: Vec<(u32, u8)>, - memories: Option>, - cpus: Option>, - orphan_node_id: u32, // id of the node containing only memory / CPU (orphan) -) { - if let Some(memories) = memories { - assert!(cpus.is_none()); - let foster_node = if !distances.is_empty() { - let (nearest_node_id, distance) = distances.first().unwrap(); - nodes.get_mut(nearest_node_id).unwrap() - } else { - let foster_node_id = { - let mut node_ids = nodes.keys(); - let foster_node = node_ids - .find(|node_id| **node_id != orphan_node_id) - .unwrap(); - *foster_node - }; - nodes.get_mut(&foster_node_id).unwrap() // panic not possible since there must be atleast one other domain with a cpu - }; - foster_node.memory.extend(memories); - } else if let Some(cpus) = cpus { - assert!(memories.is_none()); - let foster_node = if !distances.is_empty() { - let (nearest_node_id, distance) = distances.first().unwrap(); - nodes.get_mut(nearest_node_id).unwrap() - } else { - let foster_node_id = { - let mut node_ids = nodes.keys(); - let foster_node = node_ids - .find(|node_id| **node_id != orphan_node_id) - .unwrap(); - *foster_node - }; - nodes.get_mut(&foster_node_id).unwrap() // panic not possible since there must be atleast one other domain with memory - }; - foster_node.cpus.extend(cpus); - } else { - unreachable!() // this should never happen - }; - - for (_, node) in nodes { - if let Some(idx) = node.distances.iter().position(|e| e.0 == orphan_node_id) { - let _ = node.distances.remove(idx); - } - } -} - -fn sort_by_distances(nodes: &mut HashMap) { - for (id, node) in nodes { - node.distances.sort_by_key(|(_, e)| *e); - } -} diff --git a/src/startup/memory.rs b/src/startup/memory.rs index 6a19d90719..1f23227ffb 100644 --- a/src/startup/memory.rs +++ b/src/startup/memory.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "numa")] +use crate::numa; use crate::{ arch::CurrentRmmArch, memory::PAGE_SIZE, diff --git a/src/startup/mod.rs b/src/startup/mod.rs index 0ea4274a36..26700e3312 100644 --- a/src/startup/mod.rs +++ b/src/startup/mod.rs @@ -190,8 +190,7 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { } #[cfg(feature = "numa")] - numa::init(); - + numa::dump_info(); run_userspace(&mut token) }