From 1e670645e452f2d9734f3f1ffc084d31c67b94b8 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Tue, 1 Aug 2023 11:22:42 +0200 Subject: [PATCH] Bypass kernel heap for storing PageInfo. Since these are stored in arrays that are typically 512 KiB each (i.e. which in turn typically store metadata for 128 MiB), they will certainly be a page-size multiple, and large contiguous allocations are allowed at least at boot time. Not only is the linked-list allocator less efficient for that, but this change will also help reduce TLB overhead. --- src/common/mod.rs | 10 ++++++---- src/memory/mod.rs | 45 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/common/mod.rs b/src/common/mod.rs index 7081135cb8..712cb1c5ba 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,3 +1,5 @@ +use core::alloc::Allocator; + use alloc::boxed::Box; use alloc::vec::Vec; @@ -32,13 +34,13 @@ macro_rules! dbg { }; } -pub fn try_new_vec_with_exact_size(len: usize) -> Result, Enomem> { - let mut vec = Vec::new(); +pub fn try_new_vec_with_exact_size(len: usize, alloc: A) -> Result, Enomem> { + let mut vec = Vec::::new_in(alloc); vec.try_reserve_exact(len).map_err(|_| Enomem)?; Ok(vec.into()) } -pub fn try_box_slice_new(value: impl FnMut() -> T, len: usize) -> Result, Enomem> { - let mut vec = try_new_vec_with_exact_size(len)?; +pub fn try_box_slice_new(value: impl FnMut() -> T, len: usize, alloc: A) -> Result, Enomem> { + let mut vec = try_new_vec_with_exact_size(len, alloc)?; vec.resize_with(len, value); Ok(vec.into()) } diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 7c94081616..31123ab0a9 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -1,17 +1,17 @@ //! # Memory management //! Some code was borrowed from [Phil Opp's Blog](http://os.phil-opp.com/allocating-frames.html) +use core::ptr::NonNull; use core::{cmp, mem}; use core::num::NonZeroUsize; use core::sync::atomic::{AtomicUsize, Ordering}; use crate::arch::rmm::LockedAllocator; use crate::common::try_box_slice_new; -use crate::context; -use crate::context::memory::{init_frame, AccessMode, PfError}; +use crate::context::{self, memory::{init_frame, AccessMode, PfError}}; use crate::kernel_executable_offsets::{__usercopy_start, __usercopy_end}; use crate::paging::Page; -pub use crate::paging::{PAGE_SIZE, PhysicalAddress}; +pub use crate::paging::{PAGE_SIZE, PhysicalAddress, RmmA, RmmArch}; use crate::rmm::areas; use alloc::boxed::Box; @@ -223,7 +223,8 @@ bitflags::bitflags! { } } -// TODO: Very read-heavy RwLock? ArcSwap? +// TODO: Very read-heavy RwLock? ArcSwap? Store the struct in percpu, and in the *very* unlikely +// event of hotplugging, do IPIs to force all CPUs to update the sections. // // XXX: Is it possible to safely initialize an empty boxed slice from a const context? //pub static SECTIONS: RwLock> = RwLock::new(Box::new([])); @@ -238,6 +239,38 @@ pub const MAX_SECTION_SIZE_BITS: u32 = 27; pub const MAX_SECTION_SIZE: usize = 1 << MAX_SECTION_SIZE_BITS; pub const MAX_SECTION_PAGE_COUNT: usize = MAX_SECTION_SIZE / PAGE_SIZE; +const _: () = { + assert!(mem::size_of::().is_power_of_two()); +}; + +/// Allocator that bypasses the kernel heap, instead allocating directly from physical memory. +pub struct DirectAllocator; + +unsafe impl core::alloc::Allocator for DirectAllocator { + unsafe fn deallocate(&self, ptr: core::ptr::NonNull, layout: core::alloc::Layout) { + // TODO: virt_to_phys + let phys = (ptr.as_ptr() as usize) - RmmA::PHYS_OFFSET; + let frame = Frame::containing_address(PhysicalAddress::new(phys)); + + deallocate_frames(frame, layout.size().div_ceil(PAGE_SIZE)); + } + // TODO: Allow zeroing out frames to be optional in RMM? + fn allocate_zeroed(&self, layout: core::alloc::Layout) -> Result, core::alloc::AllocError> { + assert!(layout.align() <= PAGE_SIZE); + + let phys = allocate_frames(layout.size().div_ceil(PAGE_SIZE)).ok_or(core::alloc::AllocError)?; + + Ok(unsafe { + let virt = RmmA::phys_to_virt(phys.start_address()).data() as *mut u8; + + NonNull::new_unchecked(core::ptr::slice_from_raw_parts_mut(virt as *mut u8, layout.size())) + }) + } + fn allocate(&self, layout: core::alloc::Layout) -> Result, core::alloc::AllocError> { + self.allocate_zeroed(layout) + } +} + #[cold] pub fn init_mm() { let mut guard = SECTIONS.write(); @@ -250,6 +283,7 @@ pub fn init_mm() { assert_ne!(memory_map_area.size, 0, "RMM should enforce areas are not zeroed"); // TODO: Would it make sense to naturally align the sections? + // TODO: Should RMM do this? while let Some(next_area) = iter.peek() && next_area.base == memory_map_area.base.add(memory_map_area.size) { memory_map_area.size += next_area.size; @@ -267,8 +301,7 @@ pub fn init_mm() { sections.push(Section { base, - // TODO: zeroed rather than PageInfo::new()? - frames: Box::leak(try_box_slice_new(PageInfo::new, section_page_count).expect("failed to allocate pages array")), + frames: Box::leak(try_box_slice_new(|| PageInfo::new(), section_page_count, DirectAllocator).expect("failed to allocate static frame sections")), }); pages_left -= section_page_count;