vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into kernel-0.3.0

Browse Source 
# Conflicts:
#	src/acpi/mod.rs
#	src/acpi/rsdt.rs
#	src/acpi/rxsdt.rs
#	src/acpi/srat/aarch64.rs
#	src/acpi/xsdt.rs
#	src/arch/x86_shared/start.rs
#	src/context/mod.rs
#	src/context/switch.rs
#	src/main.rs
#	src/memory/mod.rs
#	src/numa.rs
#	src/scheme/sys/mod.rs
#	src/startup/mod.rs
This commit is contained in:
vasilito
2026-07-11 11:31:19 +03:00
parent 8e8dcfb09d 985bc2623f
commit 7aa2a165df
17 changed files with 1129 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rmm/src/lib.rs
+2
View File
@@ -1,6 +1,8 @@
#![no_std]
#![allow(clippy::new_without_default)]
use core::ops::Add;
pub use crate::{allocator::*, arch::*, page::*};
mod allocator;
src/acpi/mod.rs
+32 -53
View File
@@ -1,29 +1,34 @@
//! # ACPI
//! Code to parse the ACPI tables
use alloc::{boxed::Box, string::String, vec::Vec};
use core::ptr::NonNull;
use alloc::{boxed::Box, string::String, vec::Vec};
use hashbrown::HashMap;
use spin::{Once, RwLock};
use crate::memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch};
use crate::{
acpi::rxsdt::RxsdtIter,
memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch},
};
use self::{hpet::Hpet, madt::Madt, rsdp::Rsdp, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sdt, xsdt::Xsdt};
#[cfg(target_arch = "aarch64")]
mod gtdt;
pub mod fadt;
pub mod facs;
pub mod hpet;
pub mod madt;
mod rsdp;
mod rsdt;
mod rxsdt;
pub mod sdt;
#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
pub mod slit;
#[cfg(target_arch = "aarch64")]
mod spcr;
#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
pub mod srat;
mod xsdt;
unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::memory::PageMapper) {
@@ -76,24 +81,20 @@ pub enum RxsdtEnum {
Xsdt(Xsdt),
}
impl Rxsdt for RxsdtEnum {
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
fn iter(&self) -> RxsdtIter {
match self {
Self::Rsdt(rsdt) => <Rsdt as Rxsdt>::iter(rsdt),
Self::Xsdt(xsdt) => <Xsdt as Rxsdt>::iter(xsdt),
Self::Rsdt(rsdt) => rsdt.iter(),
Self::Xsdt(xsdt) => xsdt.iter(),
}
}
}
pub static RXSDT_ENUM: Once<RxsdtEnum> = Once::new();
/// Parse the ACPI tables to gather CPU, interrupt, and timer information
pub unsafe fn init(already_supplied_rsdp: Option<NonNull<u8>>) {
/// Initialses the global `RXSDT_ENUM` if RSDT or XSDT was found and maps the SDT pages.
/// It does not perform any allocations
pub unsafe fn init_before_mem(already_supplied_rsdp: Option<NonNull<u8>>) {
unsafe {
{
let mut sdt_ptrs = SDT_POINTERS.write();
*sdt_ptrs = Some(HashMap::new());
}
// Search for RSDP
let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp);
@@ -138,6 +139,22 @@ pub unsafe fn init(already_supplied_rsdp: Option<NonNull<u8>>) {
for sdt in rxsdt.iter() {
get_sdt(sdt, &mut KernelMapper::lock_rw());
}
} else {
error!("NO RSDP FOUND");
return;
}
}
}
/// Parse the ACPI tables to gather CPU, interrupt, and timer information. The code performs allocations, so
/// it must be called only after the allocator is set up.
pub unsafe fn init_after_mem(already_supplied_rsdp: Option<NonNull<u8>>) {
if let Some(rxsdt) = RXSDT_ENUM.get() {
unsafe {
{
let mut sdt_ptrs = SDT_POINTERS.write();
*sdt_ptrs = Some(HashMap::new());
}
for sdt_address in rxsdt.iter() {
let sdt = &*(RmmA::phys_to_virt(sdt_address).data() as *const Sdt);
@@ -160,44 +177,6 @@ pub unsafe fn init(already_supplied_rsdp: Option<NonNull<u8>>) {
Hpet::init();
#[cfg(target_arch = "aarch64")]
gtdt::Gtdt::init();
// Phase II: parse the FADT to extract the PM1a_CNT
// and PM1a_STS port addresses used by the S3 entry
// path. Hardware-agnostic — works on any platform
// with a working FADT.
if let Some(fadt_sdts) = find_sdt("FACP").first() {
fadt::init(fadt_sdts);
} else {
warn!("ACPI: no FADT (FACP) found, S3 entry path disabled");
}
// Phase II.X.W: parse the FACS to extract the
// xfirmware_waking_vector. This is the address the
// platform firmware jumps to on S3 wake. The kernel's
// S3 resume trampoline in arch/x86_shared/s3_resume.rs
// is written to this address by acpid via the
// SetS3WakingVector AcPiVerb.
//
// The FACS is found via the FADT's x_firmware_ctrl
// field (64-bit) or firmware_ctrl field (32-bit).
// The FADT parser caches the FACS address. We map the
// FACS here because it is not listed in the RSDT/XSDT
// and therefore was not mapped during table discovery.
const FACS_MIN_SIZE: usize = 64;
let mut facs_addr = fadt::x_firmware_ctrl();
if facs_addr == 0 {
facs_addr = fadt::firmware_ctrl() as u64;
}
if facs_addr != 0 {
let facs_phys = PhysicalAddress::new(facs_addr as usize);
map_linearly(facs_phys, FACS_MIN_SIZE, &mut KernelMapper::lock_rw());
let facs_sdt = unsafe {
&*(RmmA::phys_to_virt(facs_phys).data() as *const Sdt)
};
facs::init(facs_sdt);
} else {
warn!("ACPI: no FACS found (neither x_firmware_ctrl nor firmware_ctrl), S3 resume path disabled");
}
} else {
error!("NO RSDP FOUND");
}
}
}
src/acpi/rsdt.rs
+7 -23
View File
@@ -2,6 +2,8 @@ use alloc::boxed::Box;
use core::convert::TryFrom;
use rmm::PhysicalAddress;
use crate::acpi::{rxsdt::RxsdtIter, RxsdtEnum};
use super::{rxsdt::Rxsdt, sdt::Sdt};
#[derive(Debug)]
@@ -24,29 +26,11 @@ impl Rsdt {
}
impl Rxsdt for Rsdt {
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
Box::new(RsdtIter { sdt: self.0, i: 0 })
}
}
pub struct RsdtIter {
sdt: &'static Sdt,
i: usize,
}
impl Iterator for RsdtIter {
type Item = PhysicalAddress;
fn next(&mut self) -> Option<Self::Item> {
if self.i < self.sdt.data_len() / size_of::<u32>() {
let item = unsafe {
(self.sdt.data_address() as *const u32)
.add(self.i)
.read_unaligned()
};
self.i += 1;
Some(PhysicalAddress::new(item as usize))
} else {
None
fn iter(&self) -> RxsdtIter {
RxsdtIter {
sdt: self.0,
i: 0,
rxsdt_enum: RxsdtEnum::Rsdt(Rsdt(self.0)),
}
}
}
src/acpi/rxsdt.rs
+31 -1
View File
@@ -1,6 +1,36 @@
use alloc::boxed::Box;
use rmm::PhysicalAddress;
use crate::acpi::{sdt::Sdt, RxsdtEnum};
pub trait Rxsdt {
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>>;
fn iter(&self) -> RxsdtIter;
}
pub struct RxsdtIter {
pub sdt: &'static Sdt,
pub i: usize,
pub rxsdt_enum: RxsdtEnum,
}
impl Iterator for RxsdtIter {
type Item = PhysicalAddress;
fn next(&mut self) -> Option<Self::Item> {
if self.i < self.sdt.data_len() / size_of::<u64>() {
let item = unsafe {
match self.rxsdt_enum {
RxsdtEnum::Rsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned(
(self.sdt.data_address() as *const u32).add(self.i),
) as usize),
RxsdtEnum::Xsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned(
(self.sdt.data_address() as *const u64).add(self.i),
) as usize),
}
};
self.i += 1;
Some(item)
} else {
None
}
}
}
src/acpi/srat/aarch64.rs
+89
View File
@@ -0,0 +1,89 @@
use core::{ops::Add, slice, u32};
use rmm::{Arch, BumpAllocator, FrameAllocator, FrameCount, PhysicalAddress};
use crate::{
acpi::srat::{to_usize, Srat, SratEntry},
cpu_set::MAX_CPU_COUNT,
memory::{round_up_pages, PAGE_SIZE},
numa::{self, assign_memory_id, assign_node_id, NumaMemory},
};
pub fn init_srat(dom_node_map: &mut [u32], cpus: &mut [u32], mem: &mut [NumaMemory], srat: &Srat) {
let mut cpu_count = 0;
let mut memory_count = 0;
srat.into_iter().for_each(|e| match e {
SratEntry::GiccAffinity(gicc_affinity) => {
if gicc_affinity.flags & 1 != 0 {
cpu_count += 1
}
}
SratEntry::MemoryAffinity(memory_affinity) => {
if memory_affinity.flags & 1 != 0 && memory_affinity.flags & (1 << 1) == 0 {
memory_count += 1
}
}
_ => (),
});
assert!(
memory_count <= numa::MAX_DOMAINS,
"Found {} memory blocks while only a maximum of {} are supported",
memory_count,
numa::MAX_DOMAINS
);
assert!(
cpu_count <= MAX_CPU_COUNT,
"Found more number of CPUs than supported"
);
for affinity in srat {
match affinity {
SratEntry::MemoryAffinity(memory_affinity) => {
let start = to_usize(
memory_affinity.base_address_low,
memory_affinity.base_address_high,
);
let length = to_usize(memory_affinity.length_low, memory_affinity.length_high);
if length == 0 {
continue;
}
if memory_affinity.flags & 1 == 0 {
// memory disabled
continue;
}
if memory_affinity.flags & (1 << 1) != 0 {
// memory hot-pluggable
continue;
}
if dom_node_map[memory_affinity.proximity_domain as usize] == u32::MAX {
let node = assign_node_id(true);
dom_node_map[memory_affinity.proximity_domain as usize] = node as u32;
}
let mem_id = assign_memory_id() as u32;
mem[mem_id as usize] = NumaMemory {
start,
length,
node_id: dom_node_map[memory_affinity.proximity_domain as usize],
_pad: [0u8; 4],
};
}
SratEntry::GiccAffinity(gicc_affinity) => {
if gicc_affinity.flags & 1 == 0 {
// disabled
continue;
}
let id = gicc_affinity.processor_uid;
let dom = gicc_affinity.proximity_domain;
if dom_node_map[dom as usize] == u32::MAX {
let node = assign_node_id(true);
dom_node_map[dom as usize] = node as u32;
}
cpus[id as usize] = dom_node_map[dom as usize];
}
_ => continue,
}
}
}
src/acpi/xsdt.rs
+7 -21
View File
@@ -2,6 +2,8 @@ use alloc::boxed::Box;
use core::convert::TryFrom;
use rmm::PhysicalAddress;
use crate::acpi::{rxsdt::RxsdtIter, RxsdtEnum};
use super::{rxsdt::Rxsdt, sdt::Sdt};
#[derive(Debug)]
@@ -24,27 +26,11 @@ impl Xsdt {
}
impl Rxsdt for Xsdt {
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
Box::new(XsdtIter { sdt: self.0, i: 0 })
}
}
pub struct XsdtIter {
sdt: &'static Sdt,
i: usize,
}
impl Iterator for XsdtIter {
type Item = PhysicalAddress;
fn next(&mut self) -> Option<Self::Item> {
if self.i < self.sdt.data_len() / size_of::<u64>() {
let item = unsafe {
core::ptr::read_unaligned((self.sdt.data_address() as *const u64).add(self.i))
};
self.i += 1;
Some(PhysicalAddress::new(item as usize))
} else {
None
fn iter(&self) -> RxsdtIter {
RxsdtIter {
sdt: self.0,
i: 0,
rxsdt_enum: RxsdtEnum::Xsdt(Xsdt(self.0)),
}
}
}
src/arch/x86_shared/start.rs
+14 -3
View File
@@ -12,6 +12,8 @@ use crate::{
startup::KernelArgs,
};
use crate::numa;
/// Test of zero values in BSS.
static BSS_TEST_ZERO: SyncUnsafeCell<usize> = SyncUnsafeCell::new(0);
/// Test of non-zero values in data.
@@ -102,13 +104,22 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
// Initialize RMM
#[cfg(target_arch = "x86")]
crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000));
let mut bump_allocator =
crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000));
#[cfg(target_arch = "x86_64")]
crate::startup::memory::init(&args, Some(0x100000), None);
let mut bump_allocator = crate::startup::memory::init(&args, Some(0x100000), None);
// Initialize paging
paging::init();
if cfg!(feature = "acpi") {
crate::acpi::init_before_mem(args.acpi_rsdp());
}
numa::init(&mut bump_allocator);
crate::memory::init_mm(bump_allocator);
#[cfg(target_arch = "x86_64")]
crate::arch::alternative::early_init(true);
@@ -130,7 +141,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
// Read ACPI tables, starts APs
if cfg!(feature = "acpi") {
crate::acpi::init(args.acpi_rsdp());
crate::acpi::init_after_mem(args.acpi_rsdp());
device::init_after_acpi();
}
crate::profiling::init();
src/context/context.rs
+1
View File
@@ -314,6 +314,7 @@ impl Context {
pub fn unblock_no_ipi(&mut self) -> bool {
if self.status.is_soft_blocked() {
self.status = Status::Runnable;
self.wake = None;
self.status_reason = "";
true
src/context/mod.rs
+39 -9
View File
@@ -3,10 +3,10 @@
//! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and [osdev](https://wiki.osdev.org/Context_Switching)
use alloc::{
collections::{BTreeSet, VecDeque},
collections::{BTreeMap, BTreeSet, VecDeque},
sync::{Arc, Weak},
};
use core::{num::NonZeroUsize, ops::Deref};
use core::{cmp::Reverse, num::NonZeroUsize, ops::Deref};
use crate::{
context::memory::AddrSpaceWrapper,
@@ -81,16 +81,30 @@ static RUN_CONTEXTS: Mutex<L1, RunContextData> = Mutex::new(RunContextData::new(
static IDLE_CONTEXTS: Mutex<L2, VecDeque<WeakContextRef>> = Mutex::new(VecDeque::new());
pub struct RunContextData {
set: [VecDeque<WeakContextRef>; 40],
// queue: VecDeque<WeakContextRef>,
queue: BTreeMap<(u64, Reverse<u64>, u32), (u64, u64, WeakContextRef)>, // ((vd, rem_slice, ctxt_id), (vtime, weight, context))
timers: BTreeSet<(u128, WeakContextRef)>, // (wake, context)
count: usize,
v: u64,
total_weight: u64,
min_vtime: u64,
}
impl RunContextData {
pub const fn new() -> Self {
const EMPTY_VEC: VecDeque<WeakContextRef> = VecDeque::new();
Self {
set: [EMPTY_VEC; 40],
queue: BTreeMap::new(),
timers: BTreeSet::new(),
count: 0,
v: 0,
total_weight: 0,
min_vtime: 0,
}
}
pub fn update_count(&mut self) -> usize {
self.count = self.queue.len();
self.count
}
}
/// Get the global schemes list, const
@@ -117,6 +131,10 @@ pub fn run_contexts(token: LockToken<'_, L0>) -> MutexGuard<'_, L1, RunContextDa
RUN_CONTEXTS.lock(token)
}
pub fn run_contexts_try(token: LockToken<'_, L0>) -> Option<MutexGuard<'_, L1, RunContextData>> {
RUN_CONTEXTS.try_lock(token)
}
pub fn init(token: &mut CleanLockToken) {
let owner = None; // kmain not owned by any fd
let mut context = Context::new(owner).expect("failed to create kmain context");
@@ -126,12 +144,18 @@ pub fn init(token: &mut CleanLockToken) {
context.name.clear();
context.name.push_str("[kmain]");
#[cfg(feature = "profiling")]
{
crate::profiling::DBG_ID_MAP
.write(token.token())
.insert(context.debug_id, context.name);
}
self::arch::EMPTY_CR3.call_once(|| RmmA::table(TableKind::User));
context.status = Status::Runnable;
context.running = true;
context.cpu_id = Some(crate::cpu_id());
context.start_time = crate::time::monotonic(token);
let context_lock = Arc::new(ContextLock::new(context));
@@ -224,9 +248,6 @@ pub fn spawn(
let stack = Kstack::new()?;
let mut context = Context::new(owner_proc_id)?;
context.start_time = crate::time::monotonic(token);
context.pgrp = context.pid;
context.session = context.pid;
let _ = context.set_addr_space(Some(AddrSpaceWrapper::new()?), token.downgrade());
context
@@ -235,6 +256,7 @@ pub fn spawn(
context.kstack = Some(stack);
context.userspace = userspace_allowed;
context.queue_key = Some((context.vd, Reverse(context.rem_slice), context.debug_id));
let context_lock = Arc::new(ContextLock::new(context));
let context_ref = ContextRef(Arc::clone(&context_lock));
@@ -326,3 +348,11 @@ impl Drop for PreemptGuardL2<'_> {
self.context.write(self.token.token()).preempt_locks -= 1;
}
}
pub fn get_contexts_stats(token: &mut CleanLockToken) -> (usize, usize, usize) {
let alive = contexts(token.downgrade()).len();
let running = run_contexts(token.token()).count;
let blocked = idle_contexts(token.downgrade()).len();
(alive, running, blocked)
}
src/context/switch.rs
+323 -127
View File
@@ -4,20 +4,25 @@
use crate::{
context::{
self, arch, idle_contexts, idle_contexts_try, run_contexts, ArcContextLockWriteGuard,
Context, ContextLock, WeakContextRef,
self, arch, idle_contexts, idle_contexts_try, memory::AddrSpaceSwitchReadGuard,
run_contexts, run_contexts_try, ArcContextLockWriteGuard, Context, ContextLock,
WeakContextRef,
},
cpu_set::LogicalCpuId,
cpu_stats::{self, CpuState},
percpu::PercpuBlock,
sync::{ArcRwLockWriteGuard, CleanLockToken, L4},
};
use alloc::{sync::Arc, vec::Vec};
use alloc::sync::Arc;
use core::{
cell::{Cell, RefCell},
hint, mem,
cmp::Reverse,
hint, matches, mem,
option::Option::{None, Some},
sync::atomic::Ordering,
u64,
};
use smallvec::SmallVec;
use syscall::PtraceFlags;
enum UpdateResult {
@@ -26,13 +31,18 @@ enum UpdateResult {
Blocked,
}
// A simple geometric series where value[i] ~= value[i - 1] * 1.25
// A simple geometric series where value[i] ~= value[i + 1] * 1.25
const SCHED_PRIO_TO_WEIGHT: [usize; 40] = [
88761, 71755, 56483, 46273, 36291, 29154, 23254, 18705, 14949, 11916, 9548, 7620, 6100, 4904,
3906, 3121, 2501, 1991, 1586, 1277, 1024, 820, 655, 526, 423, 335, 272, 215, 172, 137, 110, 87,
70, 56, 45, 36, 29, 23, 18, 15,
];
const SCALE: u128 = 1 << 40;
const TICK_INTERVAL: u64 = 3; // Approx 6.75 ms
const BASE_SLICE_TICKS: u64 = TICK_INTERVAL * 3; // Approx 20.25 ms
const NANOS_PER_TICK: u128 = 2_250_000; // 2.25 ms
/// Determines if a given context is eligible to be scheduled on a given CPU (in
/// principle, the current CPU).
///
@@ -96,7 +106,9 @@ pub fn tick(token: &mut CleanLockToken) {
ticks_cell.set(new_ticks);
// Trigger a context switch after every 3 ticks (approx. 6.75 ms).
if new_ticks >= 3 {
if new_ticks >= TICK_INTERVAL as usize
&& arch::CONTEXT_SWITCH_LOCK.load(Ordering::Relaxed) == false
{
switch(token);
crate::context::signal::signal_handler(token);
}
@@ -175,15 +187,80 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
}
// Alarm (previously in update_runnable)
let wakeups = wakeup_contexts(token, switch_time);
let mut wakeups = wakeup_contexts(token);
let mut push_idle: SmallVec<[WeakContextRef; 16]> = SmallVec::new();
if let Some(mut run_contexts) = run_contexts_try(token.token()) {
// Pop Timers
while let Some((wake, _)) = run_contexts.timers.first() {
if *wake > switch_time {
break;
}
if let Some((_, context_ref)) = run_contexts.timers.pop_first() {
wakeups.push(context_ref);
}
}
}
if wakeups.len() > 0 {
let mut run_contexts = run_contexts(token.token());
for (prio, context_lock) in wakeups {
run_contexts.set[prio].push_back(context_lock);
for context_ref in wakeups {
let Some(context_lock) = context_ref.upgrade() else {
continue;
};
let Some(mut guard) = (unsafe { context_lock.try_write_arc() }) else {
push_idle.push(context_ref);
continue;
};
let new_vtime = guard.vtime.max(run_contexts.v);
guard.vtime = new_vtime;
let weight = SCHED_PRIO_TO_WEIGHT[guard.prio] as u64;
let scaled_slice = (BASE_SLICE_TICKS as u128 * SCALE) / weight as u128;
if !guard.is_active {
guard.is_active = true;
run_contexts.total_weight += weight;
}
guard.vd = new_vtime + scaled_slice as u64;
guard.rem_slice = BASE_SLICE_TICKS * SCALE as u64;
let key = (guard.vd, Reverse(guard.rem_slice), guard.debug_id);
guard.queue_key = Some(key);
drop(guard);
run_contexts
.queue
.insert(key, (new_vtime, weight, context_ref));
}
}
{
let mut idle_list = idle_contexts(token.downgrade());
for context_ref in push_idle {
idle_list.push_back(context_ref);
}
}
/* // uncomment to debug contexts count
let cpu_count = crate::cpu_count() as usize;
let len_idle = idle_contexts(token.downgrade()).len();
let all_contexts = context::contexts(token.downgrade())
.len()
.saturating_sub(cpu_count); // ignore kmain
print!(
"\r TIME {}.{} IDLE {} WAKEUPS {} ALL {} ",
switch_time / 1000_000_000,
(switch_time / 100_000_000) % 10,
len_idle,
wakeups_len,
all_contexts
);
*/
let cpu_id = crate::cpu_id();
// Update per-cpu times
@@ -192,21 +269,19 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
let was_idle = percpu.stats.add_time(percpu_ms) == CpuState::Idle as u8;
percpu.switch_internals.switch_time.set(switch_time);
let switch_context_opt = match select_next_context(
let switch_context_opt = select_next_context(
token,
percpu,
cpu_id,
switch_time,
percpu_nanos,
was_idle,
&mut prev_context_guard,
) {
Ok(opt) => opt,
Err(early_ret) => return early_ret,
};
);
// Switch process states, TSS stack pointer, and store new context ID
match switch_context_opt {
Some(mut next_context_guard) => {
Some((mut next_context_guard, addr_space_guard)) => {
// Update context states and prepare for the switch.
let prev_context = &mut *prev_context_guard;
let next_context = &mut *next_context_guard;
@@ -221,13 +296,7 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
// Update times
if !was_idle {
let delta = switch_time.saturating_sub(prev_context.switch_time);
prev_context.cpu_time += delta;
if prev_context.userspace {
prev_context.utime += delta;
} else {
prev_context.stime += delta;
}
prev_context.cpu_time += switch_time.saturating_sub(prev_context.switch_time);
}
next_context.switch_time = switch_time;
if next_context.userspace {
@@ -272,6 +341,14 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
prev_context.inside_syscall =
percpu.inside_syscall.replace(next_context.inside_syscall);
#[cfg(feature = "profiling")]
{
percpu
.switch_internals
.current_dbg_id
.store(next_context.debug_id, Ordering::Relaxed);
}
#[cfg(feature = "syscall_debug")]
{
prev_context.syscall_debug_info = percpu
@@ -286,7 +363,11 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
.being_sigkilled
.set(next_context.being_sigkilled);
// Anything implement Drop must be manually dropped now
drop(prev_context_lock);
unsafe {
percpu.new_addrsp_guard.set(addr_space_guard);
arch::switch_to(prev_context, next_context);
}
@@ -308,9 +389,11 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
}
}
fn wakeup_contexts(token: &mut CleanLockToken, switch_time: u128) -> Vec<(usize, WeakContextRef)> {
// TODO: Optimise this somehow. Perhaps using a separate timer queue?
let mut wakeups = Vec::new();
fn wakeup_contexts(
token: &mut CleanLockToken,
) -> SmallVec<[WeakContextRef; 16]> {
// TODO: Optimise this somehow
let mut wakeups = SmallVec::new();
let current_context = context::current();
let Some(idle_contexts) = idle_contexts_try(token.downgrade()) else {
// other cpus may spawning or killing contexts so let's skip wakeups to avoid contention
@@ -333,21 +416,14 @@ fn wakeup_contexts(token: &mut CleanLockToken, switch_time: u128) -> Vec<(usize,
idle_contexts.push_back(context_ref);
continue;
};
if guard.status.is_soft_blocked() {
if let Some(wake) = guard.wake {
if switch_time >= wake {
let prio = guard.prio;
drop(guard);
wakeups.push((prio, context_ref));
continue;
}
}
if guard.status.is_dead() {
// TODO: who hold this dead context?
continue;
}
if guard.status.is_runnable() && !guard.running {
let prio = guard.prio;
drop(guard);
wakeups.push((prio, context_ref));
wakeups.push(context_ref);
continue;
}
@@ -357,137 +433,250 @@ fn wakeup_contexts(token: &mut CleanLockToken, switch_time: u128) -> Vec<(usize,
wakeups
}
/// This is the scheduler function which currently utilises Deficit Weighted Round Robin Scheduler
/// This is the scheduler function which currently utilises EEVDF Scheduler
fn select_next_context(
token: &mut CleanLockToken,
percpu: &PercpuBlock,
cpu_id: LogicalCpuId,
switch_time: u128,
elapsed_time: u64,
was_idle: bool,
prev_context_guard: &mut ArcRwLockWriteGuard<L4, Context>,
) -> Result<Option<ArcContextLockWriteGuard>, SwitchResult> {
) -> Option<(ArcContextLockWriteGuard, Option<AddrSpaceSwitchReadGuard>)> {
let contexts_data = run_contexts(token.token());
let (mut contexts_data, mut token) = contexts_data.into_split();
let contexts_list = &mut contexts_data.set;
let idle_context = percpu.switch_internals.idle_context();
let mut balance = percpu.balance.get();
let mut i = percpu.last_queue.get() % 40;
// Lock the previous context.
let prev_context_lock = crate::context::current();
let is_idle = Arc::ptr_eq(&prev_context_lock, &idle_context);
let prev_runnable = !is_idle && prev_context_guard.status.is_runnable();
let is_timer = prev_context_guard.wake.is_some();
let mut empty_queues = 0;
let mut total_iters = 0;
let mut next_context_guard_opt = None;
let elapsed_ticks = elapsed_time as u128 * SCALE / NANOS_PER_TICK;
let total_contexts: usize = contexts_list.iter().map(|q| q.len()).sum();
let mut skipped_contexts = 0;
if prev_runnable {
let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64;
prev_context_guard.rem_slice = prev_context_guard
.rem_slice
.saturating_sub((elapsed_ticks) as u64);
let scaled_task = elapsed_ticks / weight as u128;
prev_context_guard.vtime += scaled_task as u64;
'priority: loop {
i = (i + 1) % 40;
total_iters += 1;
// The least prioritised queue takes <5000 iters to build up
// balance = sched_prio_to_weight[20], if we have already spent
// that many iters and not found any context, it is better to just
// skip for now
if total_iters >= 5000 {
break 'priority;
if prev_context_guard.vtime < contexts_data.v {
prev_context_guard.vtime = contexts_data.v;
}
if skipped_contexts > total_contexts && total_contexts > 0 {
break 'priority;
let is_yield = (elapsed_time as u128) < (TICK_INTERVAL as u128 * NANOS_PER_TICK) / 2;
if is_yield {
let unconsumed = prev_context_guard.rem_slice as u128;
let penalty = unconsumed / weight as u128;
prev_context_guard.vtime += penalty as u64;
prev_context_guard.rem_slice = 0;
}
let contexts = contexts_list
.get_mut(i)
.expect("i should be between [0, 39]!");
if prev_context_guard.rem_slice == 0 {
prev_context_guard.rem_slice = BASE_SLICE_TICKS * SCALE as u64;
let scaled_slice = (BASE_SLICE_TICKS as u128 * SCALE) / weight as u128;
prev_context_guard.vd = prev_context_guard.vtime + scaled_slice as u64;
}
} else if !is_idle {
if prev_context_guard.is_active {
prev_context_guard.is_active = false;
let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64;
contexts_data.total_weight = contexts_data.total_weight.saturating_sub(weight);
}
prev_context_guard.rem_slice = 0;
if contexts.is_empty() {
empty_queues += 1;
if empty_queues >= 40 {
// If all queues are empty, just break out
break 'priority;
}
continue;
if let Some(wake) = prev_context_guard.wake {
contexts_data
.timers
.insert((wake, WeakContextRef(Arc::downgrade(&prev_context_lock))));
}
}
let mut eligible_best = None;
let mut prev_is_eligible = false;
let mut ineligible_best = None;
let mut ineligible_min_vtime = u64::MAX;
let mut ineligible_vd = u64::MAX;
if prev_runnable {
if prev_context_guard.vtime <= contexts_data.v {
prev_is_eligible = true;
} else {
empty_queues = 0;
ineligible_min_vtime = prev_context_guard.vtime;
ineligible_vd = prev_context_guard.vd;
}
}
if balance[i] < SCHED_PRIO_TO_WEIGHT[20] {
// This queue does not have enough balance to run,
// increment the balance!
balance[i] += SCHED_PRIO_TO_WEIGHT[i];
// New BTreeMap based walk
let mut weight_change: u64 = 0;
let mut contexts_to_remove: SmallVec<[(u64, Reverse<u64>, u32); 16]> = SmallVec::new();
for ((vd, rem_slice, ctxt_id), (vtime, context_weight, context_ref)) in
contexts_data.queue.iter()
{
if *vtime > ineligible_min_vtime && *vtime > contexts_data.v {
continue;
}
let len = contexts.len();
for _ in 0..len {
let (next_context_ref, next_context_lock) = match contexts.pop_front() {
Some(lock) => match lock.upgrade() {
Some(new_lock) => (lock, new_lock),
None => {
skipped_contexts += 1;
continue; // Ghost Process, just continue
}
},
None => break, // Empty Queue
};
let Some(context_lock) = context_ref.upgrade() else {
weight_change += *context_weight as u64;
contexts_to_remove.push((*vd, *rem_slice, *ctxt_id));
continue;
};
if Arc::ptr_eq(&next_context_lock, &prev_context_lock) {
contexts.push_back(next_context_ref);
continue;
}
if Arc::ptr_eq(&next_context_lock, &idle_context) {
contexts.push_back(next_context_ref);
continue;
}
let mut next_context_guard = unsafe { next_context_lock.write_arc() };
if Arc::ptr_eq(&context_lock, &idle_context)
|| Arc::ptr_eq(&context_lock, &prev_context_lock)
{
//weight_change += *context_weight as u64;
//contexts_to_remove.push((*vd, *rem_slice, *ctxt_id));
continue;
}
// Is this context runnable on this CPU?
let sw = unsafe { update_runnable(&mut next_context_guard, cpu_id, switch_time) };
if let UpdateResult::CanSwitch = sw {
next_context_guard_opt = Some(next_context_guard);
balance[i] -= SCHED_PRIO_TO_WEIGHT[20];
break 'priority;
let Some(mut guard) = (unsafe { context_lock.try_write_arc() }) else {
continue;
};
let sw = unsafe { update_runnable(&mut guard, cpu_id, switch_time) };
if matches!(sw, UpdateResult::Blocked) {
if guard.is_active {
guard.is_active = false;
weight_change += context_weight;
}
guard.rem_slice = 0;
guard.queue_key = None;
contexts_to_remove.push((*vd, *rem_slice, *ctxt_id));
drop(guard);
// Reenqueue should be handled by unblock
idle_contexts(token.token()).push_back(context_ref.clone());
continue;
}
if !matches!(sw, UpdateResult::CanSwitch) {
continue;
}
let mut best_addr_space = None;
if let Some(addr_space) = &guard.addr_space {
let mut t = unsafe { CleanLockToken::new() };
if let Some(addr) = addr_space.inner.try_read(t.token()) {
best_addr_space = Some(AddrSpaceSwitchReadGuard::new(addr));
} else {
if matches!(sw, UpdateResult::Blocked) {
idle_contexts(token.token()).push_back(next_context_ref);
} else {
contexts.push_back(next_context_ref);
};
skipped_contexts += 1;
continue;
}
}
if skipped_contexts >= total_contexts {
break 'priority;
if *vtime <= contexts_data.v {
// Eligible
eligible_best = Some((guard, best_addr_space));
break;
} else {
// Ineligible
if *vtime < ineligible_min_vtime {
ineligible_min_vtime = *vtime;
ineligible_vd = *vd;
if let Some((old_guard, old_addr_space)) = ineligible_best {
drop(old_guard);
drop(old_addr_space);
}
ineligible_best = Some((guard, best_addr_space));
}
}
}
percpu.balance.set(balance);
percpu.last_queue.set(i);
if !Arc::ptr_eq(&prev_context_lock, &idle_context) {
// Send the old process to the back of the line (if it is still runnable)
let prev_ctx = WeakContextRef(Arc::downgrade(&prev_context_lock));
if prev_context_guard.status.is_runnable() {
let prio = prev_context_guard.prio;
contexts_list[prio].push_back(prev_ctx);
} else {
idle_contexts(token.token()).push_back(prev_ctx);
contexts_data.total_weight = contexts_data.total_weight.saturating_sub(weight_change);
for old_key in contexts_to_remove {
contexts_data.queue.remove(&old_key);
}
// No eligible context was found
if !(prev_is_eligible || eligible_best.is_some()) && ineligible_min_vtime != u64::MAX {
contexts_data.v = ineligible_min_vtime; // Advance V
let prev_is_earliest = prev_runnable && prev_context_guard.vtime <= ineligible_min_vtime;
if prev_is_earliest {
eligible_best = None;
} else if ineligible_best.is_some() {
let prev_has_slice = prev_runnable && prev_context_guard.rem_slice > 0;
if prev_has_slice && prev_context_guard.vd <= ineligible_vd {
eligible_best = None;
} else {
eligible_best = ineligible_best.take();
}
}
} else if prev_is_eligible && eligible_best.is_some() {
if let Some((ref guard, _)) = eligible_best {
if prev_context_guard.vd < guard.vd
|| (prev_context_guard.vd == guard.vd
&& prev_context_guard.rem_slice > guard.rem_slice)
{
eligible_best = None;
}
}
}
if let Some(next_context_guard) = next_context_guard_opt {
// We found a new process!
return Ok(Some(next_context_guard));
} else {
if !was_idle && !Arc::ptr_eq(&prev_context_lock, &idle_context) {
// We switch into the idle context
Ok(Some(unsafe { idle_context.write_arc() }))
let mut final_winner = None;
if let Some((mut chosen_guard, addr_space)) = eligible_best {
if let Some(key) = chosen_guard.queue_key.take() {
contexts_data.queue.remove(&key);
}
final_winner = Some((chosen_guard, addr_space));
}
if final_winner.is_some() || prev_runnable {
if contexts_data.total_weight > 0 {
let v_advance = elapsed_ticks as u128 / contexts_data.total_weight as u128;
contexts_data.v += v_advance as u64;
}
if let Some((chosen_guard, addr_space)) = final_winner {
if prev_runnable {
let (vd, rem_slice, ctxt_id, vtime) = (
prev_context_guard.vd,
prev_context_guard.rem_slice,
prev_context_guard.debug_id,
prev_context_guard.vtime,
);
prev_context_guard.queue_key = Some((vd, Reverse(rem_slice), ctxt_id));
let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64;
contexts_data.queue.insert(
(vd, Reverse(rem_slice), ctxt_id),
(
vtime,
weight,
WeakContextRef(Arc::downgrade(&prev_context_lock)),
),
);
} else if !is_idle && !is_timer {
idle_contexts(token.token())
.push_back(WeakContextRef(Arc::downgrade(&prev_context_lock)));
}
return Some((chosen_guard, addr_space));
} else {
// We found no other process to run.
Ok(None)
return None;
}
} else {
if !is_idle && !is_timer {
idle_contexts(token.token())
.push_back(WeakContextRef(Arc::downgrade(&prev_context_lock)));
}
let prev_is_dead = !is_idle && !prev_context_guard.status.is_runnable();
if (!was_idle || prev_is_dead) && !is_idle {
return Some(unsafe { (idle_context.write_arc(), None) });
} else {
return None;
}
}
}
@@ -503,6 +692,10 @@ pub struct ContextSwitchPercpu {
current_ctxt: RefCell<Option<Arc<ContextLock>>>,
// TODO: just access current_ctxt directly?
#[cfg(feature = "profiling")]
pub(crate) current_dbg_id: core::sync::atomic::AtomicU32,
/// The idle process.
idle_ctxt: RefCell<Option<Arc<ContextLock>>>,
pub(crate) being_sigkilled: Cell<bool>,
@@ -517,6 +710,9 @@ impl ContextSwitchPercpu {
current_ctxt: RefCell::new(None),
idle_ctxt: RefCell::new(None),
being_sigkilled: Cell::new(false),
#[cfg(feature = "profiling")]
current_dbg_id: core::sync::atomic::AtomicU32::new(!0),
}
}
src/main.rs
+4 -3
View File
@@ -3,10 +3,8 @@
//! The Redox OS Kernel is a microkernel that supports `x86_64` systems and
//! provides Unix-like syscalls for primarily Rust applications
#![feature(asm_cfg)] // Stabilized in 1.93
#![feature(if_let_guard)]
#![feature(int_roundings)]
#![feature(iter_next_chunk)]
#![cfg_attr(dtb, feature(iter_next_chunk))]
#![feature(sync_unsafe_cell)]
#![feature(btree_cursors)]
#![cfg_attr(not(test), no_std)]
@@ -70,6 +68,9 @@ mod log;
/// Memory management
mod memory;
/// NUMA support
mod numa;
/// Panic
mod panic;
src/memory/mod.rs
+343 -142
View File
@@ -4,11 +4,13 @@
use core::{
cell::SyncUnsafeCell,
num::NonZeroUsize,
sync::atomic::{AtomicUsize, Ordering},
ops::AddAssign,
slice,
sync::atomic::{AtomicU8, AtomicUsize, Ordering},
};
pub use kernel_mapper::KernelMapper;
use spin::Mutex;
use spin::{once::Once, Mutex};
pub use crate::arch::CurrentRmmArch as RmmA;
use crate::{
@@ -17,11 +19,12 @@ use crate::{
memory::{AccessMode, PfError},
},
kernel_executable_offsets::{__usercopy_end, __usercopy_start},
numa,
sync::CleanLockToken,
syscall::error::{Error, ENOMEM},
};
pub use rmm::{Arch as RmmArch, PageFlags, PhysicalAddress, TableKind, VirtualAddress};
use rmm::{BumpAllocator, FrameAllocator, FrameCount, FrameUsage};
use rmm::{BumpAllocator, FrameAllocator, FrameCount, FrameUsage, MemoryArea};
mod kernel_mapper;
pub mod page;
@@ -54,7 +57,15 @@ pub fn free_frames() -> usize {
/// Get the number of frames used
pub fn used_frames() -> usize {
// TODO: Include bump allocator static pages?
FREELIST.lock().used_frames
FREE_LISTS
.get()
.unwrap()
.iter()
.map(|e| {
let l = e.lock();
l.used_frames
})
.sum()
}
pub fn total_frames() -> usize {
// TODO: Include bump allocator static pages?
@@ -63,19 +74,39 @@ pub fn total_frames() -> usize {
/// Allocate a range of frames
pub fn allocate_p2frame(order: u32) -> Option<Frame> {
allocate_p2frame_complex(order, (), None, order).map(|(f, _)| f)
let initial_index = get_round_robin_index();
let mut index = initial_index;
loop {
if let Some(frame) = allocate_p2frame_complex(order, (), None, order, index).map(|(f, _)| f)
{
return Some(frame);
}
index = get_round_robin_index();
if index == initial_index {
return None;
}
}
}
pub fn allocate_frame() -> Option<Frame> {
allocate_p2frame(0)
}
fn get_round_robin_index() -> usize {
static CURRENT_INDEX: AtomicUsize = AtomicUsize::new(0);
let len = FREE_LISTS.get().unwrap().len();
CURRENT_INDEX.fetch_add(1, Ordering::Relaxed) % len
}
// TODO: Flags, strategy
pub fn allocate_p2frame_complex(
_req_order: u32,
_flags: (),
_strategy: Option<()>,
min_order: u32,
index: usize,
) -> Option<(Frame, usize)> {
let mut freelist = FREELIST.lock();
let mut freelist = FREE_LISTS.get().unwrap()[index].lock();
let (frame_order, frame) = freelist
.for_orders
@@ -112,7 +143,9 @@ pub fn allocate_p2frame_complex(
freelist.for_orders[frame_order as usize] = next_free.frame();
// TODO: Is this LIFO cache optimal?
//info!("MIN{min_order}FRAMEORD{frame_order}");
// if min_order > 0 {
// info!("MIN {min_order} FRAMEORD {frame_order}");
// }
for order in (min_order..frame_order).rev() {
//info!("SPLIT ORDER {order}");
let order_page_count = 1 << order;
@@ -146,8 +179,19 @@ pub fn allocate_p2frame_complex(
Some((frame, PAGE_SIZE << min_order))
}
fn get_index_for_deallocation(addr: usize) -> Option<usize> {
for (i, list) in FREE_LISTS.get().unwrap().iter().enumerate() {
let l = list.lock();
if addr >= l.lower_limit && addr < l.upper_limit {
return Some(i);
}
}
None
}
pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) {
let mut freelist = FREELIST.lock();
let index = get_index_for_deallocation(orig_frame.physaddr.get()).expect("Expected an index");
let mut freelist = FREE_LISTS.get().unwrap()[index].lock();
let initial_info = get_page_info(orig_frame)
.unwrap_or_else(|| panic!("missing PageInfo for {orig_frame:?} being freed"));
@@ -231,7 +275,9 @@ pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) {
old_head_info.set_prev(P2Frame::new(Some(new_head), largest_order));
}
//info!("FREED {frame:?}+2^{order}");
// if order > 0 {
// info!("FREED {current:?}+2^{order}");
// }
freelist.used_frames -= 1 << order;
}
@@ -476,13 +522,12 @@ struct AllocatorData {
}
#[derive(Debug)]
struct FreeList {
upper_limit: usize,
lower_limit: usize,
for_orders: [Option<Frame>; ORDER_COUNT as usize],
used_frames: usize,
}
static FREELIST: Mutex<FreeList> = Mutex::new(FreeList {
for_orders: [None; ORDER_COUNT as usize],
used_frames: 0,
});
static FREE_LISTS: Once<&'static [Mutex<FreeList>]> = Once::new();
pub struct Section {
base: Frame,
@@ -499,6 +544,8 @@ const _: () = {
#[cold]
fn init_sections(mut allocator: BumpAllocator<RmmA>) {
let number_of_memory_regions = numa::number_of_memory_regions();
let (free_areas, offset_into_first_free_area) = allocator.free_areas();
let free_areas_iter = || {
@@ -524,6 +571,18 @@ fn init_sections(mut allocator: BumpAllocator<RmmA>) {
.next_multiple_of(MAX_SECTION_SIZE);
let aligned_start = area.base.data() / MAX_SECTION_SIZE * MAX_SECTION_SIZE;
if number_of_memory_regions > 0 {
if let Some(next_memory_region) =
numa::nearest_next_memory_region(area.base.data(), false)
&& next_memory_region.start < area.base.add(area.size).data()
&& next_memory_region.start + next_memory_region.length
> area.base.add(area.size).data()
&& !next_memory_region.start.is_multiple_of(MAX_SECTION_SIZE)
{
return (aligned_end - aligned_start) / MAX_SECTION_SIZE + 1;
}
}
(aligned_end - aligned_start) / MAX_SECTION_SIZE
})
.sum();
@@ -543,76 +602,141 @@ fn init_sections(mut allocator: BumpAllocator<RmmA>) {
let mut iter = free_areas_iter().peekable();
let mut i = 0;
let mut sections_fill = |region: Option<MemoryArea>,
i: &mut usize, // out parameter
force: bool|
-> Option<MemoryArea> {
let mut iter = free_areas_iter().peekable();
while let Some(mut memory_map_area) = iter.next() {
// TODO: NonZeroUsize
// TODO: x86_32 fails without this check
if memory_map_area.size == 0 {
continue;
}
assert_ne!(
memory_map_area.size, 0,
"RMM should enforce areas are not zeroed"
);
// TODO: Should RMM do this?
while let Some(next_area) = iter.peek()
&& next_area.base == memory_map_area.base.add(memory_map_area.size)
{
memory_map_area.size += next_area.size;
let _ = iter.next();
}
assert_eq!(
memory_map_area.base.data() % PAGE_SIZE,
0,
"RMM should enforce area alignment"
);
assert_eq!(
memory_map_area.size % PAGE_SIZE,
0,
"RMM should enforce area length alignment"
);
let mut pages_left = memory_map_area.size.div_floor(PAGE_SIZE);
let mut base = Frame::containing(memory_map_area.base);
while pages_left > 0 {
let page_info_max_count = core::cmp::min(pages_left, MAX_SECTION_PAGE_COUNT);
let pages_to_next_section =
(MAX_SECTION_SIZE - (base.base().data() % MAX_SECTION_SIZE)) / PAGE_SIZE;
let page_info_count = core::cmp::min(page_info_max_count, pages_to_next_section);
let page_info_array_size_pages =
(page_info_count * size_of::<PageInfo>()).div_ceil(PAGE_SIZE);
let page_info_array = unsafe {
let base = allocator
.allocate(FrameCount::new(page_info_array_size_pages))
.expect("failed to allocate page info array");
core::slice::from_raw_parts_mut(
RmmA::phys_to_virt(base).data() as *mut PageInfo,
page_info_count,
)
};
for p in &*page_info_array {
assert_eq!(p.next.load(Ordering::Relaxed), 0);
assert_eq!(p.refcount.load(Ordering::Relaxed), 0);
while let Some(mut memory_map_area) = iter.next() {
if !{
if let Some(region) = region
&& !force
{
memory_map_area.base >= region.base
&& memory_map_area.base.data() < region.base.data() + region.size
} else {
true
}
} {
continue;
}
sections[i] = Section {
base,
frames: page_info_array,
};
i += 1;
// TODO: NonZeroUsize
pages_left -= page_info_count;
base = base.next_by(page_info_count);
// TODO: x86_32 fails without this check
if memory_map_area.size == 0 {
continue;
}
assert_ne!(
memory_map_area.size, 0,
"RMM should enforce areas are not of length 0"
);
// TODO: Should RMM do this?
while let Some(next_area) = iter.peek()
&& next_area.base == memory_map_area.base.add(memory_map_area.size)
&& if let Some(region) = region {
next_area.base >= region.base
&& next_area.base.data() < region.base.data() + region.size
&& !force
} else {
true
}
{
memory_map_area.size += next_area.size;
let _ = iter.next();
}
assert_eq!(
memory_map_area.base.data() % PAGE_SIZE,
0,
"RMM should enforce area alignment"
);
assert_eq!(
memory_map_area.size % PAGE_SIZE,
0,
"RMM should enforce area length alignment"
);
let mut pages_left = memory_map_area.size.div_floor(PAGE_SIZE);
let mut base = Frame::containing(memory_map_area.base);
let mut return_value = None;
while pages_left > 0 {
let page_info_max_count = core::cmp::min(pages_left, MAX_SECTION_PAGE_COUNT);
let pages_to_next_section =
(MAX_SECTION_SIZE - (base.base().data() % MAX_SECTION_SIZE)) / PAGE_SIZE;
let mut page_info_count =
core::cmp::min(page_info_max_count, pages_to_next_section);
if !force
&& let Some(region) = region
&& base.base() >= region.base
&& base.base().data() < region.base.data() + region.size
&& (page_info_count * PAGE_SIZE
> (region.base.data() + region.size - base.base().data()))
{
page_info_count =
(region.base.data() + region.size - base.base().data()) / PAGE_SIZE;
return_value = Some(MemoryArea {
base: PhysicalAddress::new(region.base.data() + region.size),
size: (base.physaddr.get() + pages_left * PAGE_SIZE)
- (region.base.data() + region.size),
});
}
let page_info_array_size_pages =
(page_info_count * size_of::<PageInfo>()).div_ceil(PAGE_SIZE);
let page_info_array = unsafe {
let base = allocator
.allocate(FrameCount::new(page_info_array_size_pages))
.expect("failed to allocate page info array");
core::slice::from_raw_parts_mut(
RmmA::phys_to_virt(base).data() as *mut PageInfo,
page_info_count,
)
};
for p in &*page_info_array {
assert_eq!(p.next.load(Ordering::Relaxed), 0);
assert_eq!(p.refcount.load(Ordering::Relaxed), 0);
}
sections[*i] = Section {
base,
frames: page_info_array,
};
i.add_assign(1);
pages_left -= page_info_count;
base = base.next_by(page_info_count);
if let Some(return_value) = return_value {
return Some(return_value);
}
}
}
return None;
};
let mut i = 0;
if number_of_memory_regions > 0
&& let Some(regions) = numa::memory_regions()
{
for region in regions {
let mut sections_fill_result = Some(region);
let mut force = false;
while let Some(mut region) = sections_fill_result {
sections_fill_result = sections_fill(Some(region), &mut i, force);
force = true;
}
}
} else {
sections_fill(None, &mut i, false);
}
let sections = &mut sections[..i];
sections.sort_unstable_by_key(|s| s.base);
@@ -634,11 +758,12 @@ fn init_sections(mut allocator: BumpAllocator<RmmA>) {
}
}
let mut first_pages: [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] =
[None; ORDER_COUNT as usize];
let mut last_pages = first_pages;
let mut append_page = |page: Frame, info: &'static PageInfo, order| {
let mut append_page = |page: Frame,
info: &'static PageInfo,
order,
first_pages: &mut [Option<(Frame, &'static PageInfo)>; 11],
last_pages: &mut [Option<(Frame, &PageInfo)>; 11],
allocator: &mut BumpAllocator<RmmA>| {
let this_page = (page, info);
if page.base() < allocator.abs_offset() {
@@ -674,76 +799,153 @@ fn init_sections(mut allocator: BumpAllocator<RmmA>) {
};
}
for section in &*sections {
let mut base = section.base;
let mut frames = section.frames;
let mut free_list_fill = |region: Option<MemoryArea>,
allocator: &mut BumpAllocator<RmmA>|
-> [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] {
let mut first_pages: [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] =
[None; ORDER_COUNT as usize];
let mut last_pages = first_pages;
for order in 0..=MAX_ORDER {
let pages_for_current_order = 1 << order;
debug_assert_eq!(frames.len() % pages_for_current_order, 0);
debug_assert!(base.is_aligned_to_order(order));
if !frames.is_empty() && order != MAX_ORDER && !base.is_aligned_to_order(order + 1) {
frames[0].next.store(order as usize, Ordering::Relaxed);
// The first section page is not aligned to the next order size.
//info!("ORDER {order}: FIRST {base:?}");
append_page(base, &frames[0], order);
base = base.next_by(pages_for_current_order);
frames = &frames[pages_for_current_order..];
} else {
//info!("ORDER {order}: FIRST SKIP");
}
if !frames.is_empty()
&& order != MAX_ORDER
&& !base.next_by(frames.len()).is_aligned_to_order(order + 1)
for section in &*sections {
if let Some(region) = region
&& (section.base.physaddr.get() < region.base.data()
|| section.base.physaddr.get() >= region.base.data() + region.size)
{
// The last section page is not aligned to the next order size.
let off = frames.len() - pages_for_current_order;
let final_page = base.next_by(off);
frames[off].next.store(order as usize, Ordering::Relaxed);
//info!("ORDER {order}: LAST {final_page:?}");
append_page(final_page, &frames[off], order);
frames = &frames[..off];
} else {
//info!("ORDER {order}: LAST SKIP");
continue;
}
if frames.is_empty() {
break;
}
let mut base = section.base;
let mut frames = section.frames;
for order in 0..=MAX_ORDER {
let pages_for_current_order = 1 << order;
if order == MAX_ORDER {
debug_assert_eq!(frames.len() % pages_for_current_order, 0);
debug_assert!(base.is_aligned_to_order(MAX_ORDER));
debug_assert!(base.is_aligned_to_order(order));
for (off, info) in frames.iter().enumerate().step_by(pages_for_current_order) {
info.next.store(MAX_ORDER as usize, Ordering::Relaxed);
append_page(base.next_by(off), info, MAX_ORDER);
if !frames.is_empty() && order != MAX_ORDER && !base.is_aligned_to_order(order + 1)
{
frames[0].next.store(order as usize, Ordering::Relaxed);
// The first section page is not aligned to the next order size.
//info!("ORDER {order}: FIRST {base:?}");
append_page(
base,
&frames[0],
order,
&mut first_pages,
&mut last_pages,
allocator,
);
base = base.next_by(pages_for_current_order);
frames = &frames[pages_for_current_order..];
} else {
//info!("ORDER {order}: FIRST SKIP");
}
if !frames.is_empty()
&& order != MAX_ORDER
&& !base.next_by(frames.len()).is_aligned_to_order(order + 1)
{
// The last section page is not aligned to the next order size.
let off = frames.len() - pages_for_current_order;
let final_page = base.next_by(off);
frames[off].next.store(order as usize, Ordering::Relaxed);
//info!("ORDER {order}: LAST {final_page:?}");
append_page(
final_page,
&frames[off],
order,
&mut first_pages,
&mut last_pages,
allocator,
);
frames = &frames[..off];
} else {
//info!("ORDER {order}: LAST SKIP");
}
if frames.is_empty() {
break;
}
if order == MAX_ORDER {
debug_assert_eq!(frames.len() % pages_for_current_order, 0);
debug_assert!(base.is_aligned_to_order(MAX_ORDER));
for (off, info) in frames.iter().enumerate().step_by(pages_for_current_order) {
info.next.store(MAX_ORDER as usize, Ordering::Relaxed);
append_page(
base.next_by(off),
info,
MAX_ORDER,
&mut first_pages,
&mut last_pages,
allocator,
);
}
}
}
//info!("SECTION from {:?}, {} pages, array at {:p}", section.base, section.frames.len(), section.frames);
}
for (order, tuple_opt) in last_pages.iter().enumerate() {
let Some((frame, info)) = tuple_opt else {
continue;
};
debug_assert!(frame.is_aligned_to_order(order as u32));
let free = info.as_free().unwrap();
debug_assert_eq!(free.prev().order(), order as u32);
free.set_next(P2Frame::new(None, order as u32));
}
//info!("SECTION from {:?}, {} pages, array at {:p}", section.base, section.frames.len(), section.frames);
}
for (order, tuple_opt) in last_pages.iter().enumerate() {
let Some((frame, info)) = tuple_opt else {
continue;
first_pages
};
let free_lists;
if let Some(regions) = numa::memory_regions() {
let free_list_page = allocator
.allocate(FrameCount::new(
(numa::number_of_memory_regions() * size_of::<Mutex<FreeList>>())
.div_ceil(PAGE_SIZE),
))
.expect("Failed to allocate free list page");
let va = unsafe { RmmA::phys_to_virt(free_list_page).data() as *mut Mutex<FreeList> };
free_lists = unsafe { slice::from_raw_parts_mut(va, numa::number_of_memory_regions()) };
for (i, region) in regions.enumerate() {
let free_list = FreeList {
for_orders: free_list_fill(Some(region), &mut allocator)
.map(|pair| pair.map(|(frame, _)| frame)),
used_frames: 0,
upper_limit: region.base.add(region.size).data(),
lower_limit: region.base.data(),
};
free_lists[i] = Mutex::new(free_list);
}
} else {
let free_list_page = allocator
.allocate(FrameCount::new(
size_of::<Mutex<FreeList>>().div_ceil(PAGE_SIZE),
))
.expect("Failed to allocate free list page");
let va = unsafe { RmmA::phys_to_virt(free_list_page).data() as *mut Mutex<FreeList> };
free_lists = unsafe { slice::from_raw_parts_mut(va, 1) };
let free_list = FreeList {
for_orders: free_list_fill(None, &mut allocator)
.map(|pair| pair.map(|(frame, _)| frame)),
used_frames: 0,
upper_limit: usize::MAX,
lower_limit: 0,
};
debug_assert!(frame.is_aligned_to_order(order as u32));
let free = info.as_free().unwrap();
debug_assert_eq!(free.prev().order(), order as u32);
free.set_next(P2Frame::new(None, order as u32));
free_lists[0] = Mutex::new(free_list);
}
FREELIST.lock().for_orders = first_pages.map(|pair| pair.map(|(frame, _)| frame));
FREE_LISTS.call_once(|| free_lists);
//debug_freelist();
debug!("Initial freelist consistent");
@@ -1027,10 +1229,9 @@ pub fn page_fault_handler(
let mut token = unsafe { CleanLockToken::new() };
match context::memory::try_correcting_page_tables(faulting_page, mode, &mut token) {
Ok(()) => return Ok(()),
Err(PfError::Oom) => {
return Err(Segv);
}
Err(PfError::Segv | PfError::RecursionLimitExceeded | PfError::NonfatalInternalError) => (),
Err(PfError::Oom) => todo!("oom"),
Err(PfError::Segv | PfError::RecursionLimitExceeded) => (),
Err(PfError::NonfatalInternalError) => todo!(),
}
}
src/numa.rs
+224
View File
@@ -0,0 +1,224 @@
use core::ops::Add;
use crate::{
acpi,
cpu_set::LogicalCpuId,
sync::{CleanLockToken, Mutex, L0},
};
use alloc::{sync::Arc, vec::Vec};
use hashbrown::HashMap;
use rmm::{Arch, BumpAllocator, MemoryArea, PhysicalAddress};
use spin::once::Once;
use syscall::{Error, Result, ENODATA, EOPNOTSUPP};
pub const MAX_DOMAINS: usize = 128;
static DOMAIN_NODE_MAP: Once<&'static [u32]> = Once::new();
static NUMA_CPUS: Once<&'static [u32]> = Once::new();
static NUMA_MEMORY: Once<&'static [NumaMemory]> = Once::new();
static DISTANCES: Once<&'static [u8]> = Once::new();
#[derive(Debug, Clone)]
pub struct NumaMemory {
pub start: usize,
pub length: usize,
pub node_id: u32,
pub _pad: [u8; 4],
}
#[derive(Debug)]
pub struct NumaCpu {
pub id: u32,
}
pub fn init<A: Arch>(allocator: &mut BumpAllocator<A>) {
#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
{
acpi::srat::init(allocator, &DOMAIN_NODE_MAP, &NUMA_CPUS, &NUMA_MEMORY);
acpi::slit::init(allocator, &DISTANCES);
}
}
pub fn assign_node_id(modify: bool) -> u8 {
static mut NODE_ID: u8 = 0;
if unsafe { NODE_ID } >= 128 {
panic!("Maximum number of domains supported is 128");
}
unsafe {
NODE_ID += 1;
let return_value = NODE_ID - 1;
if !modify {
NODE_ID -= 1;
}
return_value
}
}
pub fn assign_memory_id() -> u8 {
static mut MEMORY_ID: u8 = 0;
if unsafe { MEMORY_ID } >= 128 {
panic!("Maximum number of memory regions supported is 128");
}
let old = unsafe { MEMORY_ID };
unsafe { MEMORY_ID = MEMORY_ID.add(1) };
old
}
pub fn domain_to_node_id(domain_id: u32) -> Option<u32> {
Some(*DOMAIN_NODE_MAP.get()?.get(domain_id as usize)?)
}
pub fn cpu_belongs_to_which_node(cpu_id: usize) -> Option<u32> {
Some(*NUMA_CPUS.get()?.get(cpu_id)?)
}
/// A helper function that prints information about NUMA - available nodes, cpus and memory blocks in them
/// their starts and lengths
pub fn dump_info() {
if let Some(map) = DOMAIN_NODE_MAP.get()
&& let Some(cpus) = NUMA_CPUS.get()
&& let Some(memories) = NUMA_MEMORY.get()
{
println!("Number of NUMA nodes: {}", assign_node_id(false));
for i in 0..cpus.len() {
if cpus[i] == u32::MAX {
continue;
}
println!("CPU {} : Node {}", i, cpus[i])
}
for i in 0..memories.len() {
if memories[i].length == 0 {
continue;
}
println!(
"Memory Block starting at address {:#x} of size {:#x} bytes : Node {}",
memories[i].start, memories[i].length, memories[i].node_id
);
}
} else {
println!(
"The system has either no support for NUMA or there was an error during initialisation"
);
}
}
pub struct NumaMemoryIter {
i: usize,
mem: &'static [NumaMemory],
}
impl Iterator for NumaMemoryIter {
type Item = MemoryArea;
fn next(&mut self) -> Option<Self::Item> {
let mem = self.mem.get(self.i)?;
if mem.length == 0 {
return None;
}
self.i += 1;
Some(MemoryArea {
base: PhysicalAddress::new(mem.start),
size: mem.length,
})
}
}
impl NumaMemoryIter {
/// Skips an arbitrarily chosen `i`th element. Unlike `skip`, which skips the first `n` elements,
/// `iskip` can ignore non-consecutive elements
pub fn iskip(&self, addr: usize) -> Option<usize> {
let i = self.mem.binary_search_by_key(&addr, |e| e.start).ok()?;
Some(i)
}
}
pub fn number_of_memory_regions() -> usize {
if let Some(mem) = NUMA_MEMORY.get() {
mem.iter()
.map(|e| if e.length != 0 { 1 } else { 0 })
.sum::<usize>()
} else {
0 // TODO: or should 1 be returned?
}
}
pub fn memory_regions() -> Option<NumaMemoryIter> {
if let Some(mem) = NUMA_MEMORY.get() {
Some(NumaMemoryIter { i: 0, mem })
} else {
None
}
}
pub fn nearest_next_memory_region(addr: usize, overlap: bool) -> Option<&'static NumaMemory> {
NUMA_MEMORY
.get()?
.iter()
.filter_map(|e| {
if if overlap {
e.start >= addr
} else {
e.start > addr
} {
Some(e)
} else {
None
}
})
.min_by_key(|e| e.start)
}
pub fn nearest_preceding_memory_region(addr: usize, overlap: bool) -> Option<&'static NumaMemory> {
NUMA_MEMORY
.get()?
.iter()
.filter_map(|e| {
if if overlap {
e.start <= addr
} else {
e.start < addr
} {
Some(e)
} else {
None
}
})
.max_by_key(|e| e.start)
}
pub fn get_numa_info(token: &mut CleanLockToken) -> Result<Vec<u8>> {
let cpu_info = NUMA_CPUS
.get()
.ok_or(Error::new(EOPNOTSUPP))?
.iter()
.map(|e| e.to_ne_bytes())
.flatten()
.collect::<Vec<u8>>();
let mem_info = NUMA_MEMORY
.get()
.ok_or(Error::new(EOPNOTSUPP))?
.iter()
.map(|e| {
[
e.start.to_ne_bytes(),
e.length.to_ne_bytes(),
(e.node_id as usize).to_ne_bytes(),
]
})
.flatten()
.flatten()
.collect::<Vec<u8>>();
let mut numa_info = Vec::new();
numa_info.extend(cpu_info);
numa_info.extend(mem_info);
Ok(numa_info)
}
pub fn get_numa_distance_info(token: &mut CleanLockToken) -> Result<Vec<u8>> {
Ok(DISTANCES
.get()
.ok_or(Error::new(ENODATA))?
.iter()
.map(|e| *e)
.collect())
}
src/scheme/mod.rs
-2
View File
@@ -6,8 +6,6 @@
//! The kernel validates paths and file descriptors before they are passed to schemes,
//! also stripping the scheme identifier of paths if necessary.
// TODO: Move handling of the global namespace to userspace.
use alloc::{
sync::{Arc, Weak},
vec::Vec,
src/scheme/sys/mod.rs
+2
View File
@@ -112,6 +112,8 @@ const FILES: &[(&str, Kind)] = &[
("irq", Rd(irq::resource)),
("log", Rd(log::resource)),
("mem", Rd(mem::resource)),
("numa", Rd(crate::numa::get_numa_info)),
("numa_dist", Rd(crate::numa::get_numa_distance_info)),
("syscall", Rd(syscall::resource)),
("uname", Rd(uname::resource)),
("env", Rd(|_| Ok(Vec::from(crate::startup::init_env())))),
src/startup/memory.rs
+1
View File
@@ -1,6 +1,7 @@
use crate::{
arch::CurrentRmmArch,
memory::PAGE_SIZE,
numa,
startup::{memory::BootloaderMemoryKind::Null, KernelArgs},
};
use core::{
src/startup/mod.rs
+10 -14
View File
@@ -1,18 +1,20 @@
use core::{
hint, slice,
hint,
ptr::NonNull,
slice,
sync::atomic::{AtomicBool, Ordering},
};
use core::ptr::NonNull;
use crate::{
arch::interrupt,
context,
context::switch::SwitchResult,
context::{self, switch::SwitchResult},
memory::{PhysicalAddress, RmmA, RmmArch},
profiling, scheme,
sync::CleanLockToken,
};
use crate::numa;
pub mod memory;
#[repr(C, packed(8))]
@@ -105,7 +107,7 @@ impl KernelArgs {
)
};
if data.starts_with(b"RSD PTR ") {
Some(NonNull::new(data.as_ptr() as *mut u8).unwrap())
Some(NonNull::from_ref(data).cast())
} else {
None
}
@@ -186,6 +188,7 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! {
}
}
numa::dump_info();
run_userspace(&mut token)
}
@@ -230,15 +233,8 @@ fn run_userspace(token: &mut CleanLockToken) -> ! {
interrupt::enable_and_nop();
}
SwitchResult::AllContextsIdle => {
// Enable interrupts, then enter the deepest MWAIT
// C-state (C6/C7/C8/C9/C10/S0iX). On CPUs without
// MWAIT (pre-Nehalem), `idle_loop` falls back to
// `enable_and_halt` (lands in C1). The MWAIT path
// enables Arrow Lake-H to actually reach S0i2/S0i3
// substates and dramatically reduce idle power on the
// LG Gram 2025; without it the kernel only lands in
// C1 and the CPU stays relatively warm.
interrupt::idle_loop();
// Enable interrupts, then halt CPU (to save power) until the next interrupt is actually fired.
interrupt::enable_and_halt();
}
}
}