diff --git a/Cargo.lock b/Cargo.lock index 966555f82d..d01e01ba48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,7 +104,7 @@ dependencies = [ [[package]] name = "kernel" -version = "0.5.12+rb0.3.0" +version = "0.5.12+rb0.3.1" dependencies = [ "arrayvec", "bitfield", @@ -201,7 +201,7 @@ checksum = "64072665120942deff5fd5425d6c1811b854f4939e7f1c01ce755f64432bbea7" [[package]] name = "redox_syscall" -version = "0.9.0+rb0.3.0" +version = "0.9.0+rb0.3.1" dependencies = [ "bitflags 2.13.0", ] @@ -420,4 +420,4 @@ dependencies = [ [[patch.unused]] name = "libredox" -version = "0.1.18+rb0.3.0" +version = "0.1.18+rb0.3.1" diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index fd1e88b948..83900eb226 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -17,6 +17,8 @@ use self::{hpet::Hpet, madt::Madt, rsdp::Rsdp, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sd #[cfg(target_arch = "aarch64")] mod gtdt; +pub mod fadt; +pub mod facs; pub mod hpet; pub mod madt; mod rsdp; diff --git a/src/acpi/slit.rs b/src/acpi/slit.rs new file mode 100644 index 0000000000..03c91597da --- /dev/null +++ b/src/acpi/slit.rs @@ -0,0 +1,44 @@ +use crate::{ + acpi::{rxsdt::Rxsdt, sdt::Sdt, RXSDT_ENUM}, + find_one_sdt, + memory::{round_up_pages, PAGE_SIZE}, + numa::{self}, +}; +use core::{ops::Add, slice}; +use hashbrown::HashMap; +use rmm::{Arch, BumpAllocator, FrameAllocator, FrameCount}; +use spin::once::Once; + +#[derive(Debug)] +pub struct Slit { + sdt: &'static Sdt, + no: u64, + address: *const u8, +} + +impl Slit { + pub fn new(sdt: &'static Sdt) -> Self { + Self { + sdt, + no: unsafe { *(sdt.data_address() as *const u64) }, + address: (sdt.data_address() + 8) as *const u8, + } + } + pub fn init(&self, allocator: &mut BumpAllocator) -> &'static mut [u8] { + unsafe { slice::from_raw_parts_mut(self.address as *mut u8, (self.no * self.no) as usize) } + } +} + +pub fn init(allocator: &mut BumpAllocator, distances: &Once<&'static [u8]>) { + if let Some(rxsdt) = RXSDT_ENUM.get() { + for sdt_addr in rxsdt.iter() { + let sdt = + unsafe { &*(crate::memory::RmmA::phys_to_virt(sdt_addr).data() as *const Sdt) }; + if &sdt.signature == b"SLIT" { + let slit = Slit::new(sdt); + distances.call_once(|| slit.init(allocator)); + return; + } + } + } +} diff --git a/src/acpi/srat/mod.rs b/src/acpi/srat/mod.rs new file mode 100644 index 0000000000..0768a78e9e --- /dev/null +++ b/src/acpi/srat/mod.rs @@ -0,0 +1,215 @@ +//! See + +use core::slice; + +use hashbrown::HashMap; +use rmm::{Arch, BumpAllocator, FrameAllocator}; +use spin::once::Once; + +use crate::{ + acpi::{find_sdt, get_sdt_signature, rxsdt::Rxsdt, sdt::Sdt, srat, RXSDT_ENUM}, + cpu_set::MAX_CPU_COUNT, + find_one_sdt, memory, + numa::{self, NumaMemory}, +}; + +#[cfg(target_arch = "aarch64")] +#[path = "aarch64.rs"] +mod arch; + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[path = "x86.rs"] +mod arch; + +#[repr(C, packed)] +pub struct Srat { + sdt: &'static Sdt, + entries: *const u8, +} + +pub fn init( + allocator: &mut BumpAllocator, + map: &Once<&'static [u32]>, + once_cpus: &Once<&'static [u32]>, + mem: &Once<&'static [NumaMemory]>, +) { + let dom_node_map = allocator + .allocate(rmm::FrameCount::new(1)) + .expect("Failed to allocate memory for storing NUMA info"); + + let dom_node_map_ptr = + unsafe { crate::memory::RmmA::phys_to_virt(dom_node_map).data() as *mut u32 }; + + // Occupies 512 bytes (1/8th of a page) + let dom_node_map: &'static mut [u32] = + unsafe { slice::from_raw_parts_mut(dom_node_map_ptr, numa::MAX_DOMAINS) }; + + // occupies 512 bytes (1/8th of a page) + let cpus: &'static mut [u32] = unsafe { + slice::from_raw_parts_mut( + dom_node_map_ptr.add(numa::MAX_DOMAINS) as *mut u32, + MAX_CPU_COUNT as usize, + ) + }; + + // total occupied till now: 1024 bytes, remaining 3072 bytes, can accomodate 128 memory entries + let memories: &'static mut [NumaMemory] = unsafe { + slice::from_raw_parts_mut( + cpus.as_ptr().add(numa::MAX_DOMAINS) as *mut NumaMemory, + numa::MAX_DOMAINS, + ) + }; + + dom_node_map.fill(u32::MAX); + cpus.fill(u32::MAX); + memories.fill(NumaMemory { + start: 0, + length: 0, + node_id: 0, + _pad: [0; 4], + }); + + if let Some(rxsdt) = RXSDT_ENUM.get() { + for sdt_addr in rxsdt.iter() { + let sdt = unsafe { &*(memory::RmmA::phys_to_virt(sdt_addr).data() as *const Sdt) }; + if &sdt.signature == b"SRAT" { + arch::init_srat(dom_node_map, cpus, memories, &Srat::new(sdt)); + map.call_once(|| dom_node_map); + once_cpus.call_once(|| cpus); + mem.call_once(|| memories); + return; + } + } + } +} + +impl Srat { + pub fn new(sdt: &'static Sdt) -> Self { + Self { + sdt, + entries: (sdt.data_address() + 12) as *const u8, + } + } +} + +impl<'a> IntoIterator for &'a Srat { + type Item = SratEntry; + + type IntoIter = SratIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + SratIter { i: 0, srat: self } + } +} + +pub struct SratIter<'a> { + i: u32, + srat: &'a Srat, +} + +impl<'a> Iterator for SratIter<'a> { + type Item = SratEntry; + + fn next(&mut self) -> Option { + while self.i < self.srat.sdt.data_len() as u32 { + let entry = unsafe { self.srat.entries.add(self.i as usize) }; + let entry_len = unsafe { *self.srat.entries.add(self.i as usize + 1) }; + + let entry = Some(match unsafe { *entry } { + 0 => SratEntry::LegacyProcessorLocalAffinity(unsafe { + assert!(entry_len as usize == size_of::() + 2); + *(entry.add(2) as *const LegacyProcessorLocalAffinity) + }), + + 1 => SratEntry::MemoryAffinity(unsafe { + assert!(entry_len as usize == size_of::() + 10); + *(entry.add(2) as *const MemoryAffinity) + }), + 2 => SratEntry::ProcessorLocalAffinity(unsafe { + assert!(entry_len as usize == size_of::() + 8); + *(entry.add(4) as *const ProcessorLocalAffinity) + }), + 3 => SratEntry::GiccAffinity(unsafe { + assert!(entry_len as usize == size_of::() + 2); + *(entry.add(2) as *const GiccAffinity) + }), + // ignore GIC ITS Affinity and Generic Initiator Affinity + _ => { + self.i += entry_len as u32; + continue; + } + }); + self.i += entry_len as u32; + return entry; + } + None + } +} + +#[derive(Debug, Clone, Copy)] +pub enum SratEntry { + LegacyProcessorLocalAffinity(LegacyProcessorLocalAffinity), + MemoryAffinity(MemoryAffinity), + ProcessorLocalAffinity(ProcessorLocalAffinity), + GiccAffinity(GiccAffinity), + // unimplemented: Gic Its Affinity and Generic Initiator Affinity + // our current focus is only on memory and cpus +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +/// For legacy xAPIC systems +struct LegacyProcessorLocalAffinity { + proximity_domain_low: u8, + apic_id: u8, + flags: u32, + sapic_id: u8, + proximity_domain_high: [u8; 3], + clock_domain: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +struct MemoryAffinity { + proximity_domain: u32, + _reserved0: u16, + base_address_low: u32, + base_address_high: u32, + length_low: u32, + length_high: u32, + _reserved1: u32, + flags: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +/// For x2APIC systems +struct ProcessorLocalAffinity { + proximity_domain: u32, + x2apic_id: u32, + flags: u32, + clock_domain: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +struct GiccAffinity { + proximity_domain: u32, + processor_uid: u32, + flags: u32, + clock_domain: u32, +} + +#[inline(always)] +pub(crate) fn to_usize(low: u32, high: u32) -> usize { + #[cfg(target_pointer_width = "32")] + return low as usize; + + #[cfg(target_pointer_width = "64")] + { + let mut low_and_high = [0u8; 8]; + low_and_high[0..=3].copy_from_slice(low.to_le_bytes().as_slice()); + low_and_high[4..=7].copy_from_slice(high.to_le_bytes().as_slice()); + usize::from_le_bytes(low_and_high) + } +} diff --git a/src/acpi/srat/x86.rs b/src/acpi/srat/x86.rs new file mode 100644 index 0000000000..f67407db25 --- /dev/null +++ b/src/acpi/srat/x86.rs @@ -0,0 +1,123 @@ +use core::{iter, slice}; + +use hashbrown::HashMap; +use rmm::{Arch, BumpAllocator, FrameAllocator, PhysicalAddress}; + +use crate::{ + acpi::srat::{to_usize, Srat, SratEntry}, + cpu_set, + memory::{self, PAGE_SIZE}, + numa::{self, assign_memory_id, NumaMemory}, +}; + +#[inline(always)] +fn to_single_int(high: &[u8; 3], low: u8) -> u32 { + let mut high_and_low = [0u8; 4]; + high_and_low[0] = low; + (high_and_low[1], high_and_low[2], high_and_low[3]) = (high[0], high[1], high[2]); + u32::from_le_bytes(high_and_low) +} + +pub fn init_srat( + dom_node_map: &mut [u32], + cpus: &mut [u32], + memories: &mut [NumaMemory], + srat: &Srat, +) { + let mut cpu_count = 0; + let mut memory_count = 0; + + srat.into_iter().for_each(|e| match e { + SratEntry::LegacyProcessorLocalAffinity(legacy_processor_local_affinity) => { + if legacy_processor_local_affinity.flags & 1 != 0 { + cpu_count += 1 + } + } + SratEntry::MemoryAffinity(memory_affinity) => { + if memory_affinity.flags & 1 != 0 && memory_affinity.flags & (1 << 1) == 0 { + memory_count += 1 + } + } + SratEntry::ProcessorLocalAffinity(processor_local_affinity) => { + if processor_local_affinity.flags & 1 != 0 { + cpu_count += 1 + } + } + _ => (), + }); + + assert!( + memory_count <= numa::MAX_DOMAINS, + "Found {} memory blocks while only a maximum of {} are supported", + memory_count, + numa::MAX_DOMAINS + ); + + assert!( + cpu_count <= cpu_set::MAX_CPU_COUNT, + "Found more number of CPUs than supported" + ); + + for affinity in srat { + match affinity { + SratEntry::LegacyProcessorLocalAffinity(legacy_processor_local_affinity) => { + if legacy_processor_local_affinity.flags & 1 == 0 { + // processor disabled + continue; + } + let dom = to_single_int( + &legacy_processor_local_affinity.proximity_domain_high, + legacy_processor_local_affinity.proximity_domain_low, + ); + if dom_node_map[dom as usize] == u32::MAX { + let node_id = numa::assign_node_id(true); + dom_node_map[dom as usize] = node_id as u32; + } + cpus[legacy_processor_local_affinity.apic_id as usize] = dom_node_map[dom as usize]; + } + SratEntry::MemoryAffinity(memory_affinity) => { + if memory_affinity.flags & 1 == 0 { + // memory is not enabled + continue; + } + if memory_affinity.flags & (1 << 1) != 0 { + // memory is hot-pluggable + continue; + } + let dom = memory_affinity.proximity_domain; + if memory_affinity.length_low == 0 { + continue; + } + let start = to_usize( + memory_affinity.base_address_low, + memory_affinity.base_address_high, + ); + let length = to_usize(memory_affinity.length_low, memory_affinity.length_high); + if dom_node_map[dom as usize] == u32::MAX { + let node_id = numa::assign_node_id(true); + dom_node_map[dom as usize] = node_id as u32; + } + let mem_id = assign_memory_id() as u32; + memories[mem_id as usize] = numa::NumaMemory { + start, + length, + node_id: dom_node_map[dom as usize], + _pad: [0u8; 4], + }; + } + SratEntry::ProcessorLocalAffinity(processor_local_affinity) => { + if processor_local_affinity.flags & 1 == 0 { + // processor disabled + continue; + } + let dom = processor_local_affinity.proximity_domain; + if dom_node_map[dom as usize] == u32::MAX { + let node_id = numa::assign_node_id(true); + dom_node_map[dom as usize] = node_id as u32; + } + cpus[processor_local_affinity.x2apic_id as usize] = dom_node_map[dom as usize]; + } + _ => continue, + } + } +} diff --git a/src/context/context.rs b/src/context/context.rs index 6ca468d8f4..4ee0e56302 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -1,6 +1,7 @@ use alloc::{collections::BTreeSet, sync::Arc, vec::Vec}; use arrayvec::ArrayString; use core::{ + cmp::Reverse, mem::{self, size_of, ManuallyDrop}, num::NonZeroUsize, sync::atomic::{AtomicU32, Ordering}, @@ -60,6 +61,9 @@ impl Status { pub fn is_soft_blocked(&self) -> bool { matches!(self, Self::Blocked) } + pub fn is_dead(&self) -> bool { + matches!(self, Self::Dead { .. }) + } } #[derive(Clone, Debug)] @@ -73,7 +77,7 @@ pub enum HardBlockedReason { NotYetStarted, } -const CONTEXT_NAME_CAPAC: usize = 32; +pub const CONTEXT_NAME_CAPAC: usize = 32; #[derive(Debug)] pub enum SyscallFrame { @@ -146,6 +150,16 @@ pub struct Context { pub fmap_ret: Option, /// Priority pub prio: usize, + /// Virtual Run Time + pub vtime: u64, + /// Virtual Deadline + pub vd: u64, + /// Remaining Slice of allocated time + pub rem_slice: u64, + /// Is currently active? + pub is_active: bool, + /// Key for the RunQueue + pub queue_key: Option<(u64, Reverse, u32)>, /// Process group ID. pub pgrp: usize, /// Session ID. @@ -239,6 +253,11 @@ impl Context { userspace: false, fmap_ret: None, prio: 20, + vtime: 0, + vd: 0, + rem_slice: 0, + is_active: false, + queue_key: None, being_sigkilled: false, owner_proc_id, diff --git a/src/context/memory.rs b/src/context/memory.rs index 93446ba7a7..39037db91a 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -5,7 +5,7 @@ use core::{ fmt::Debug, mem::ManuallyDrop, num::NonZeroUsize, - ops::Bound, + ops::{Bound, Deref}, sync::atomic::{AtomicU32, Ordering}, }; use rmm::{Arch as _, PageFlush}; @@ -768,6 +768,25 @@ impl AddrSpace { } } +pub struct AddrSpaceSwitchReadGuard { + pub lock: RwLockReadGuard<'static, L5, AddrSpace>, +} + +impl AddrSpaceSwitchReadGuard { + pub fn new(guard: RwLockReadGuard<'_, L5, AddrSpace>) -> Self { + Self { + lock: unsafe { core::mem::transmute(guard) }, + } + } +} +impl Deref for AddrSpaceSwitchReadGuard { + type Target = RwLockReadGuard<'static, L5, AddrSpace>; + + fn deref(&self) -> &Self::Target { + &self.lock + } +} + #[derive(Debug)] pub struct UserGrants { // Using a BTreeMap for its range method. diff --git a/src/percpu.rs b/src/percpu.rs index 011f889693..a5154c39de 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -12,7 +12,11 @@ use syscall::PtraceFlags; use crate::{ arch::device::ArchPercpuMisc, - context::{empty_cr3, memory::AddrSpaceWrapper, switch::ContextSwitchPercpu}, + context::{ + empty_cr3, + memory::{AddrSpaceSwitchReadGuard, AddrSpaceWrapper}, + switch::ContextSwitchPercpu, + }, cpu_set::{LogicalCpuId, MAX_CPU_COUNT}, cpu_stats::{CpuStats, CpuStatsData}, ptrace::Session, @@ -30,6 +34,7 @@ pub struct PercpuBlock { pub current_addrsp: RefCell>>, pub new_addrsp_tmp: Cell>>, + pub new_addrsp_guard: Cell>, pub wants_tlb_shootdown: AtomicBool, pub balance: Cell<[usize; 40]>, pub last_queue: Cell, @@ -242,6 +247,7 @@ impl PercpuBlock { switch_internals: ContextSwitchPercpu::default(), current_addrsp: RefCell::new(None), new_addrsp_tmp: Cell::new(None), + new_addrsp_guard: Cell::new(None), wants_tlb_shootdown: AtomicBool::new(false), balance: Cell::new([0; 40]), last_queue: Cell::new(39), diff --git a/src/startup/memory.rs b/src/startup/memory.rs index 2f78c98f75..6a19d90719 100644 --- a/src/startup/memory.rs +++ b/src/startup/memory.rs @@ -1,7 +1,6 @@ use crate::{ arch::CurrentRmmArch, memory::PAGE_SIZE, - numa, startup::{memory::BootloaderMemoryKind::Null, KernelArgs}, }; use core::{ @@ -395,7 +394,11 @@ unsafe fn map_memory(areas: &[MemoryArea], mut bump_allocator: &mut Bum } } -pub unsafe fn init(args: &KernelArgs, low_limit: Option, high_limit: Option) { +pub unsafe fn init( + args: &KernelArgs, + low_limit: Option, + high_limit: Option, +) -> BumpAllocator { register_memory_from_kernel_args(args); unsafe { @@ -442,7 +445,6 @@ pub unsafe fn init(args: &KernelArgs, low_limit: Option, high_limit: Opti // Create the physical memory map let offset = bump_allocator.offset(); info!("Permanently used: {} KB", offset.div_ceil(KILOBYTE)); - - crate::memory::init_mm(bump_allocator); + bump_allocator } } diff --git a/src/sync/ordered.rs b/src/sync/ordered.rs index 91d46158db..0b8ebda894 100644 --- a/src/sync/ordered.rs +++ b/src/sync/ordered.rs @@ -593,6 +593,17 @@ impl RwLock { RwLockUpgradableGuard { inner, lock_token } } + // Unsafe due to not using token, currently required by context::switch + pub unsafe fn try_write_arc(self: &Arc) -> Option> { + let Some(guard) = self.inner.try_write() else { + return None; + }; + core::mem::forget(guard); + Some(ArcRwLockWriteGuard { + rwlock: self.clone(), + }) + } + // Unsafe due to not using token, currently required by context::switch pub unsafe fn write_arc(self: &Arc) -> ArcRwLockWriteGuard { core::mem::forget(self.inner.write());