From 0e8c187a5030f6820b0f725e2ab0cc18ad056f4e Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Wed, 25 Oct 2023 18:43:37 +0200 Subject: [PATCH] WIP: Implement more of deallocate_p2frame. --- src/arch/x86_64/paging/mod.rs | 1 + src/arch/x86_shared/idt.rs | 64 +++++----- src/memory/mod.rs | 218 +++++++++++++++++++++++++++++++--- src/percpu.rs | 4 +- 4 files changed, 236 insertions(+), 51 deletions(-) diff --git a/src/arch/x86_64/paging/mod.rs b/src/arch/x86_64/paging/mod.rs index 00a930bed6..b41f165ff5 100644 --- a/src/arch/x86_64/paging/mod.rs +++ b/src/arch/x86_64/paging/mod.rs @@ -25,6 +25,7 @@ pub mod mapper; /// Size of pages pub const PAGE_SIZE: usize = RmmA::PAGE_SIZE; +pub const PAGE_MASK: usize = RmmA::PAGE_OFFSET_MASK; /// Setup page attribute table #[cold] diff --git a/src/arch/x86_shared/idt.rs b/src/arch/x86_shared/idt.rs index f15a982a28..7c349093cb 100644 --- a/src/arch/x86_shared/idt.rs +++ b/src/arch/x86_shared/idt.rs @@ -18,10 +18,7 @@ use crate::{interrupt::*, ipi::IpiKind, cpu_set::LogicalCpuId}; use spin::RwLock; -pub static mut INIT_IDTR: DescriptorTablePointer = DescriptorTablePointer { - limit: 0, - base: 0 as *const X86IdtEntry, -}; +pub static mut INIT_IDT: [IdtEntry; 32] = [IdtEntry::new(); 32]; pub type IdtEntries = [IdtEntry; 256]; pub type IdtReservations = [AtomicU32; 8]; @@ -144,7 +141,37 @@ macro_rules! use_default_irqs( ); pub unsafe fn init() { - dtables::lidt(&INIT_IDTR); + set_exceptions(&mut INIT_IDT); + dtables::lidt(&DescriptorTablePointer::new(&INIT_IDT)); +} + +fn set_exceptions(idt: &mut [IdtEntry]) { + // Set up exceptions + idt[0].set_func(exception::divide_by_zero); + idt[1].set_func(exception::debug); + idt[2].set_func(exception::non_maskable); + idt[3].set_func(exception::breakpoint); + idt[3].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT); + idt[4].set_func(exception::overflow); + idt[5].set_func(exception::bound_range); + idt[6].set_func(exception::invalid_opcode); + idt[7].set_func(exception::device_not_available); + idt[8].set_func(exception::double_fault); + // 9 no longer available + idt[10].set_func(exception::invalid_tss); + idt[11].set_func(exception::segment_not_present); + idt[12].set_func(exception::stack_segment); + idt[13].set_func(exception::protection); + idt[14].set_func(exception::page); + // 15 reserved + idt[16].set_func(exception::fpu_fault); + idt[17].set_func(exception::alignment_check); + idt[18].set_func(exception::machine_check); + idt[19].set_func(exception::simd); + idt[20].set_func(exception::virtualization); + // 21 through 29 reserved + idt[30].set_func(exception::security); + // 31 reserved } const fn new_idt_reservations() -> [AtomicU32; 8] { @@ -224,35 +251,10 @@ pub unsafe fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt) { index }; - // Set up exceptions - current_idt[0].set_func(exception::divide_by_zero); - current_idt[1].set_func(exception::debug); - current_idt[2].set_func(exception::non_maskable); + set_exceptions(current_idt); current_idt[2].set_ist(backup_ist); - current_idt[3].set_func(exception::breakpoint); - current_idt[3].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT); - current_idt[4].set_func(exception::overflow); - current_idt[5].set_func(exception::bound_range); - current_idt[6].set_func(exception::invalid_opcode); - current_idt[7].set_func(exception::device_not_available); - current_idt[8].set_func(exception::double_fault); current_idt[8].set_ist(backup_ist); - // 9 no longer available - current_idt[10].set_func(exception::invalid_tss); - current_idt[11].set_func(exception::segment_not_present); - current_idt[12].set_func(exception::stack_segment); - current_idt[13].set_func(exception::protection); - current_idt[14].set_func(exception::page); - // 15 reserved - current_idt[16].set_func(exception::fpu_fault); - current_idt[17].set_func(exception::alignment_check); - current_idt[18].set_func(exception::machine_check); current_idt[18].set_ist(backup_ist); - current_idt[19].set_func(exception::simd); - current_idt[20].set_func(exception::virtualization); - // 21 through 29 reserved - current_idt[30].set_func(exception::security); - // 31 reserved #[cfg(target_arch = "x86_64")] assert_eq!( diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 242d2906f4..ee34130ac3 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -8,10 +8,12 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; -pub use crate::paging::{PhysicalAddress, RmmA, RmmArch, PAGE_SIZE}; -use crate::paging::Page; +use spin::Mutex; + use crate::context::{self, memory::{AccessMode, PfError}}; use crate::kernel_executable_offsets::{__usercopy_start, __usercopy_end}; +use crate::paging::Page; +pub use crate::paging::{PAGE_SIZE, PAGE_MASK, PhysicalAddress, RmmA, RmmArch}; use rmm::{ FrameAllocator, FrameCount, VirtualAddress, TableKind, BumpAllocator, @@ -46,29 +48,139 @@ pub fn allocate_frame() -> Option { allocate_frames(1) } pub fn allocate_frames_complex(count: usize, flags: (), strategy: Option<()>, min: usize) -> Option<(Frame, usize)> { - todo!() + // TODO: Split into sub-power of two allocations. + let min_order = min.next_power_of_two().trailing_zeros(); + let _req_order = count.next_power_of_two().trailing_zeros(); + + let mut freelist = FREELIST.lock(); + + let Some((frame_order, frame)) = freelist[min_order as usize..].iter().enumerate().find_map(|(i, f)| f.map(|f| (i, f))) else { + // TODO: For larger sizes than the max order, split into power of two allocations. + return None + }; + + let info = get_page_info(frame).unwrap_or_else(|| panic!("no page info for allocated frame {frame:?}")); + let next_free = info.next(); + log::info!("FREE {frame:?} ORDER {frame_order} NEXT_FREE {next_free:?}"); + + freelist[frame_order] = next_free.frame(); + + // TODO: Is this LIFO cache optimal? + for order in (min..frame_order).rev() { + let order_page_count = 1 << order; + + let hi = frame.next_by(order_page_count); + log::info!("SPLIT INTO {frame:?}:{hi:?} ORDER {order}"); + + if let Some(old_head) = freelist[order].replace(hi) { + let hi_info = get_page_info(hi).expect("no page info for allocated frame"); + hi_info.set_next(P2Frame::new(Some(old_head), order as u32)); + } + } + + log::info!("ALLOCATED {frame:?}+2^{min_order}"); + Some((frame, PAGE_SIZE << min_order)) +} + +/// Deallocate a range of frames +pub unsafe fn deallocate_frames(frame: Frame, count: usize) { + let max_order = core::cmp::min(MAX_ORDER, count.next_power_of_two().trailing_zeros()); + + let (first_aligned, chunk_order, number_of_chunks) = (0..=max_order).rev().find_map(|order| { + let bytes_for_order = PAGE_SIZE << order; + let first_aligned = frame.start_address().data().next_multiple_of(bytes_for_order); + let last_aligned = (frame.start_address().data() + count * PAGE_SIZE) / bytes_for_order * bytes_for_order; + let chunks = (last_aligned - first_aligned) / bytes_for_order; + + (first_aligned < last_aligned).then_some((first_aligned, order, chunks)) + }).expect("must succeed at least for order=0"); + + for i in 0..number_of_chunks { + deallocate_p2frame(Frame::containing_address(PhysicalAddress::new(first_aligned + i * (PAGE_SIZE << chunk_order))), chunk_order); + } + + let first_aligned_frame = Frame::containing_address(PhysicalAddress::new(first_aligned)); + let lo_subblock_page_count = first_aligned_frame.offset_from(frame); + let hi_subblock_page_count = count - (number_of_chunks << chunk_order) - lo_subblock_page_count; + deallocate_frames(frame, lo_subblock_page_count); + deallocate_frames(first_aligned_frame.next_by(number_of_chunks << chunk_order), hi_subblock_page_count); +} + +unsafe fn deallocate_p2frame(mut frame: Frame, order: u32) { + let mut freelist = FREELIST.lock(); + + for merge_order in order..=MAX_ORDER { + // Because there's a PageInfo, this frame must be allocator-owned. We need to be very + // careful with who owns this page, as the refcount can be anything from 0 (undefined) to + // 2^addrwidth - 1. However, allocation and deallocation must be synchronized (the "next" + // word of the PageInfo). + + let Some(neighbor) = frame.neighbor(merge_order) else { + break; + }; + + let Some(next_info) = get_page_info(neighbor) else { + // The frame that was deallocated, was the end of a contiguous allocator memory range. + break; + }; + + // Whether or not there's a linked frame, the order is nevertheless stored. + if next_info.next().order() != merge_order { + break; + } + + frame = frame.align_down_to_order(order) + .expect("must succeed since the neighbor p2frame existed"); + } + + log::info!("FREED {frame:?}+2^{order}"); +} + +pub unsafe fn deallocate_frame(frame: Frame) { + deallocate_p2frame(frame, 0) } const ORDER_COUNT: u32 = 11; +const MAX_ORDER: u32 = ORDER_COUNT - 1; pub struct FreeList { for_orders: [Option; ORDER_COUNT as usize], } -/// Deallocate a range of frames frame -pub unsafe fn deallocate_frames(frame: Frame, count: usize) { - todo!() -} -pub unsafe fn deallocate_frame(frame: Frame) { - deallocate_frames(frame, 1) -} - #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct Frame { // On x86/x86_64, all memory below 1 MiB is reserved, and although some frames in that range // may end up in the paging code, it's very unlikely that frame 0x0 would. physaddr: NonZeroUsize, } + +/// Option combined with power-of-two size. +#[derive(Clone, Copy)] +struct P2Frame(usize); +impl P2Frame { + fn new(frame: Option, order: u32) -> Self { + Self( + frame.map_or(0, |f| f.physaddr.get()) | (order as usize), + ) + } + fn get(self) -> (Option, u32) { + let page_off_mask = PAGE_SIZE - 1; + (NonZeroUsize::new(self.0 & !page_off_mask).map(|physaddr| Frame { physaddr }), (self.0 & page_off_mask) as u32) + } + fn frame(self) -> Option { + self.get().0 + } + fn order(self) -> u32 { + self.get().1 + } +} +impl core::fmt::Debug for P2Frame { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let (frame, order) = self.get(); + write!(f, "[frame at {frame:?}] order {order}") + } +} + impl core::fmt::Debug for Frame { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( @@ -89,7 +201,7 @@ impl Frame { /// Create a frame containing `address` pub fn containing(address: PhysicalAddress) -> Frame { Frame { - physaddr: NonZeroUsize::new(address.data()).expect("frame 0x0 is reserved"), + physaddr: NonZeroUsize::new(address.data() & !PAGE_MASK).expect("frame 0x0 is reserved"), } } // TODO: Remove @@ -114,12 +226,30 @@ impl Frame { .expect("overflow in Frame::next_by"), } } + pub fn prev_by(self, n: usize) -> Self { + Self { + physaddr: self.physaddr.get().checked_sub(n.checked_mul(PAGE_SIZE).expect("unreasonable n")).and_then(NonZeroUsize::new).expect("overflow in Frame::prev_by"), + } + } + pub fn neighbor(self, order: u32) -> Option { + if self.is_aligned_to_order(order) { + Some(self.next_by(1 << order)) + } else { + self.align_down_to_order(order) + } + } + pub fn align_down_to_order(self, order: u32) -> Option { + Some(Self { physaddr: NonZeroUsize::new(self.physaddr.get() / (PAGE_SIZE << order) * (PAGE_SIZE << order))? }) + } pub fn offset_from(self, from: Self) -> usize { self.physaddr .get() .checked_sub(from.physaddr.get()) .expect("overflow in Frame::offset_from") / PAGE_SIZE } + pub fn is_aligned_to_order(self, order: u32) -> bool { + self.start_address().data() % (PAGE_SIZE << order) == 0 + } } #[derive(Debug)] @@ -180,11 +310,9 @@ pub struct PageInfo { /// shared if set, and CoW if unset. The flag is not meaningful when the refcount is 0 or 1. pub refcount: AtomicUsize, - // TODO: Needs to be atomic, or we can introduce some form of lock. - // // TODO: Add one flag indicating whether the page contents is zeroed? Or should this primarily // be managed by the memory allocator first? - pub flags: FrameFlags, + pub next: AtomicUsize, } const RC_SHARED_NOT_COW: usize = 1 << (usize::BITS - 1); @@ -204,6 +332,7 @@ struct AllocatorData { // TODO: Memory hotplugging? sections: &'static [Section], } +static FREELIST: Mutex<[Option; ORDER_COUNT as usize]> = Mutex::new([None; ORDER_COUNT as usize]); pub struct Section { base: Frame, @@ -276,7 +405,7 @@ fn init_sections(mut allocator: BumpAllocator) { let page_info_array_size_pages = (page_info_count * mem::size_of::()).div_ceil(PAGE_SIZE); let page_info_array = unsafe { let base = allocator.allocate(FrameCount::new(page_info_array_size_pages)).expect("failed to allocate page info array"); - core::slice::from_raw_parts_mut(base.data() as *mut PageInfo, page_info_count) + core::slice::from_raw_parts_mut(RmmA::phys_to_virt(base).data() as *mut PageInfo, page_info_count) }; sections[i] = Section { @@ -290,16 +419,63 @@ fn init_sections(mut allocator: BumpAllocator) { } } + let mut first_pages: [Option<(Frame, &'static PageInfo)>; ORDER_COUNT as usize] = [None; ORDER_COUNT as usize]; + let mut last_pages = first_pages; + + let mut append_page = |page, info, order| { + let this_page = (page, info); + let last_page = last_pages[order as usize].replace(this_page); + + if let Some((_, last_page_info)) = last_page { + last_page_info.set_next(P2Frame::new(Some(page), order)); + } else { + first_pages[order as usize] = Some(this_page); + } + }; + for section in &*sections { - //log::info!("SECTION from {:?}, {} pages", section.base, section.frames.len()); + let mut base = section.base; + let mut frames = section.frames; + + for order in 0..=MAX_ORDER { + let pages_for_current_order = 1 << order; + + if !frames.is_empty() && (order == MAX_ORDER || !base.is_aligned_to_order(order + 1)) { + // The first section page is not aligned to the next order size. + + log::info!("ORDER {order}: FIRST {base:?}"); + append_page(base, &frames[0], order); + + base = base.next_by(pages_for_current_order << order); + frames = &frames[pages_for_current_order..]; + } else { + log::info!("ORDER {order}: FIRST SKIP"); + } + + if let Some(off2) = frames.len().checked_sub(2 * pages_for_current_order) && (order == MAX_ORDER || !base.next_by(off2).is_aligned_to_order(order + 1)) { + let off = frames.len() - pages_for_current_order; + + log::info!("ORDER {order}: LAST {base:?}"); + append_page(base.next_by(off), &frames[off], order); + + // The last section page is not aligned to the next order size. + frames = &frames[..frames.len() - pages_for_current_order]; + } else { + log::info!("ORDER {order}: LAST SKIP"); + } + } + + log::info!("SECTION from {:?}, {} pages", section.base, section.frames.len()); } + *FREELIST.lock() = first_pages.map(|pair| pair.map(|(frame, _)| frame)); + sections.sort_unstable_by_key(|s| s.base); unsafe { ALLOCATOR_DATA = AllocatorData { sections }; } - loop {} + //loop {} } #[cold] @@ -322,6 +498,12 @@ pub enum AddRefError { SharedToCow, } impl PageInfo { + fn next(&self) -> P2Frame { + P2Frame(self.next.load(Ordering::Relaxed)) + } + fn set_next(&self, next: P2Frame) { + self.next.store(next.0, Ordering::Relaxed) + } pub fn add_ref(&self, kind: RefKind) -> Result<(), AddRefError> { match (self.refcount(), kind) { (RefCount::Zero, _) => self.refcount.store(1, Ordering::Relaxed), diff --git a/src/percpu.rs b/src/percpu.rs index e8bf8eaedc..e6d8e1fbba 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -32,7 +32,7 @@ pub struct PercpuBlock { #[cfg(feature = "profiling")] pub profiling: Option<&'static crate::profiling::RingBuffer>, - pub freelist: crate::memory::FreeList, + //pub freelist: crate::memory::FreeList, pub ptrace_flags: Cell, pub ptrace_session: RefCell>>, @@ -141,7 +141,7 @@ impl PercpuBlock { ptrace_flags: Cell::new(Default::default()), ptrace_session: RefCell::new(None), inside_syscall: Cell::new(false), - freelist: FreeList::new(), + //freelist: FreeList::new(), #[cfg(feature = "syscall_debug")] syscall_debug_info: Cell::new(SyscallDebugInfo::default()),