diff --git a/rmm/.gitignore b/rmm/.gitignore new file mode 100644 index 0000000000..06aba01b65 --- /dev/null +++ b/rmm/.gitignore @@ -0,0 +1,2 @@ +Cargo.lock +/target diff --git a/rmm/Cargo.toml b/rmm/Cargo.toml new file mode 100644 index 0000000000..d5755aae45 --- /dev/null +++ b/rmm/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rmm" +version = "0.1.0" +authors = ["Jeremy Soller "] +edition = "2024" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +default = ["std"] +std = [] diff --git a/rmm/LICENSE b/rmm/LICENSE new file mode 100644 index 0000000000..c7a7f3c3fe --- /dev/null +++ b/rmm/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jeremy Soller + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/rmm/README.md b/rmm/README.md new file mode 100644 index 0000000000..f475fba520 --- /dev/null +++ b/rmm/README.md @@ -0,0 +1,4 @@ +# Redox Memory Management + +This is a Rust crate to provide abstractions for hardware memory management. It +also contains a mechanism for testing memory management with software emulation. \ No newline at end of file diff --git a/rmm/src/allocator/frame/buddy.rs b/rmm/src/allocator/frame/buddy.rs new file mode 100644 index 0000000000..def1c87a84 --- /dev/null +++ b/rmm/src/allocator/frame/buddy.rs @@ -0,0 +1,303 @@ +use core::{marker::PhantomData, mem}; + +use crate::{ + Arch, BumpAllocator, FrameAllocator, FrameCount, FrameUsage, PhysicalAddress, VirtualAddress, +}; + +#[repr(transparent)] +struct BuddyUsage(u8); + +#[repr(C, packed)] +struct BuddyEntry { + base: PhysicalAddress, + size: usize, + // Number of first free page + skip: usize, + // Count of used pages + used: usize, + phantom: PhantomData, +} + +impl Clone for BuddyEntry { + fn clone(&self) -> Self { + Self { + base: self.base, + size: self.size, + skip: self.skip, + used: self.used, + phantom: PhantomData, + } + } +} +impl Copy for BuddyEntry {} + +impl BuddyEntry { + fn empty() -> Self { + Self { + base: PhysicalAddress::new(0), + size: 0, + skip: 0, + used: 0, + phantom: PhantomData, + } + } + + #[inline(always)] + fn pages(&self) -> usize { + self.size >> A::PAGE_SHIFT + } + + fn usage_pages(&self) -> usize { + let bytes = self.pages() * mem::size_of::(); + // Round bytes used for usage to next page + (bytes + A::PAGE_OFFSET_MASK) >> A::PAGE_SHIFT + } + + unsafe fn usage_addr(&self, page: usize) -> Option { + unsafe { + if page < self.pages() { + let phys = self.base.add(page * mem::size_of::()); + Some(A::phys_to_virt(phys)) + } else { + None + } + } + } + + unsafe fn usage(&self, page: usize) -> Option { + unsafe { + let addr = self.usage_addr(page)?; + Some(A::read(addr)) + } + } + + unsafe fn set_usage(&self, page: usize, usage: BuddyUsage) -> Option<()> { + unsafe { + let addr = self.usage_addr(page)?; + Some(A::write(addr, usage)) + } + } +} + +pub struct BuddyAllocator { + table_virt: VirtualAddress, + phantom: PhantomData, +} + +impl BuddyAllocator { + const BUDDY_ENTRIES: usize = A::PAGE_SIZE / mem::size_of::>(); + + pub unsafe fn new(mut bump_allocator: BumpAllocator) -> Option { + unsafe { + // Allocate buddy table + let table_phys = bump_allocator.allocate_one()?; + let table_virt = A::phys_to_virt(table_phys); + for i in 0..(A::PAGE_SIZE / mem::size_of::>()) { + let virt = table_virt.add(i * mem::size_of::>()); + A::write(virt, BuddyEntry::::empty()); + } + + let allocator = Self { + table_virt, + phantom: PhantomData, + }; + + // Add areas to buddy table, combining areas when possible, and skipping frames used + // by the bump allocator + let mut offset = bump_allocator.offset(); + for old_area in bump_allocator.areas().iter() { + let mut area = old_area.clone(); + if offset >= area.size { + offset -= area.size; + continue; + } else if offset > 0 { + area.base = area.base.add(offset); + area.size -= offset; + offset = 0; + } + for i in 0..(A::PAGE_SIZE / mem::size_of::>()) { + let virt = table_virt.add(i * mem::size_of::>()); + let mut entry = A::read::>(virt); + let inserted = if area.base.add(area.size) == { entry.base } { + // Combine entry at start + entry.base = area.base; + entry.size += area.size; + true + } else if area.base == entry.base.add(entry.size) { + // Combine entry at end + entry.size += area.size; + true + } else if entry.size == 0 { + // Create new entry + entry.base = area.base; + entry.size = area.size; + true + } else { + false + }; + if inserted { + A::write(virt, entry); + break; + } + } + } + + //TODO: sort areas? + + // Allocate buddy maps + for i in 0..Self::BUDDY_ENTRIES { + let virt = table_virt.add(i * mem::size_of::>()); + let mut entry = A::read::>(virt); + + // Only set up entries that have enough space for their own usage map + let usage_pages = entry.usage_pages(); + if entry.pages() > usage_pages { + // Mark all usage bytes as unused + let usage_start = entry.usage_addr(0)?; + for page in 0..usage_pages { + A::write_bytes(usage_start.add(page << A::PAGE_SHIFT), 0, A::PAGE_SIZE); + } + + // Mark bytes used for usage as used + for page in 0..usage_pages { + entry.set_usage(page, BuddyUsage(1))?; + } + } + + // Skip the pages used for usage + entry.skip = usage_pages; + + // Set used pages to pages used for usage + entry.used = usage_pages; + + // Write updated entry + A::write(virt, entry); + } + + Some(allocator) + } + } +} + +impl FrameAllocator for BuddyAllocator { + unsafe fn allocate(&mut self, count: FrameCount) -> Option { + unsafe { + if self.table_virt.data() == 0 { + return None; + } + + for entry_i in 0..Self::BUDDY_ENTRIES { + let virt = self + .table_virt + .add(entry_i * mem::size_of::>()); + let mut entry = A::read::>(virt); + + let mut free_page = entry.skip; + let mut free_count = 0; + for page in entry.skip..entry.pages() { + let usage = entry.usage(page)?; + if usage.0 == 0 { + free_count += 1; + + if free_count == count.data() { + break; + } + } else { + free_page = page + 1; + free_count = 0; + } + } + + if free_count == count.data() { + for page in free_page..free_page + free_count { + // Update usage + let mut usage = entry.usage(page)?; + usage.0 += 1; + entry.set_usage(page, usage); + + // Zero page + let page_phys = entry.base.add(page << A::PAGE_SHIFT); + let page_virt = A::phys_to_virt(page_phys); + A::write_bytes(page_virt, 0, A::PAGE_SIZE); + } + + // Update skip if necessary + if entry.skip == free_page { + entry.skip = free_page + free_count; + } + + // Update used page count + entry.used += free_count; + + // Write updated entry + A::write(virt, entry); + + return Some(entry.base.add(free_page << A::PAGE_SHIFT)); + } + } + + None + } + } + + unsafe fn free(&mut self, base: PhysicalAddress, count: FrameCount) { + unsafe { + if self.table_virt.data() == 0 { + return; + } + + let size = count.data() * A::PAGE_SIZE; + for i in 0..Self::BUDDY_ENTRIES { + let virt = self.table_virt.add(i * mem::size_of::>()); + let mut entry = A::read::>(virt); + + if base >= { entry.base } && base.add(size) <= entry.base.add(entry.size) { + let start_page = (base.data() - { entry.base }.data()) >> A::PAGE_SHIFT; + for page in start_page..start_page + count.data() { + let mut usage = entry.usage(page).expect("failed to get usage during free"); + + if usage.0 > 0 { + usage.0 -= 1; + } else { + panic!("tried to free already free frame"); + } + + // If page was freed + if usage.0 == 0 { + // Update skip if necessary + if page < entry.skip { + entry.skip = page; + } + + // Update used page count + entry.used -= 1; + } + + entry + .set_usage(page, usage) + .expect("failed to set usage during free"); + } + + // Write updated entry + A::write(virt, entry); + + return; + } + } + } + } + + unsafe fn usage(&self) -> FrameUsage { + unsafe { + let mut total = 0; + let mut used = 0; + for i in 0..Self::BUDDY_ENTRIES { + let virt = self.table_virt.add(i * mem::size_of::>()); + let entry = A::read::>(virt); + total += entry.size >> A::PAGE_SHIFT; + used += entry.used; + } + FrameUsage::new(FrameCount::new(used), FrameCount::new(total)) + } + } +} diff --git a/rmm/src/allocator/frame/bump.rs b/rmm/src/allocator/frame/bump.rs new file mode 100644 index 0000000000..9043b181ae --- /dev/null +++ b/rmm/src/allocator/frame/bump.rs @@ -0,0 +1,79 @@ +use core::marker::PhantomData; + +use crate::{Arch, FrameAllocator, FrameCount, FrameUsage, MemoryArea, PhysicalAddress}; + +#[derive(Debug)] +pub struct BumpAllocator { + orig_areas: (&'static [MemoryArea], usize), + cur_areas: (&'static [MemoryArea], usize), + _marker: PhantomData A>, +} + +impl BumpAllocator { + pub fn new(mut areas: &'static [MemoryArea], mut offset: usize) -> Self { + while let Some(first) = areas.first() + && first.size <= offset + { + offset -= first.size; + areas = &areas[1..]; + } + + Self { + orig_areas: (areas, offset), + cur_areas: (areas, offset), + _marker: PhantomData, + } + } + pub fn areas(&self) -> &'static [MemoryArea] { + self.orig_areas.0 + } + /// Returns one semifree and the fully free areas. The offset is the number of bytes after + /// which the first area is free. + pub fn free_areas(&self) -> (&'static [MemoryArea], usize) { + self.cur_areas + } + pub fn abs_offset(&self) -> PhysicalAddress { + let (areas, off) = self.cur_areas; + areas + .first() + .map_or(PhysicalAddress::new(0), |a| a.base.add(off)) + } + pub fn offset(&self) -> usize { + (unsafe { self.usage().total().data() - self.usage().free().data() }) * A::PAGE_SIZE + } +} + +impl FrameAllocator for BumpAllocator { + unsafe fn allocate(&mut self, count: FrameCount) -> Option { + unsafe { + let req_size = count.data() * A::PAGE_SIZE; + + let block = loop { + let area = self.cur_areas.0.first()?; + let off = self.cur_areas.1; + if area.size - off < req_size { + self.cur_areas = (&self.cur_areas.0[1..], 0); + continue; + } + self.cur_areas.1 += req_size; + + break area.base.add(off); + }; + A::write_bytes(A::phys_to_virt(block), 0, req_size); + Some(block) + } + } + + unsafe fn free(&mut self, _address: PhysicalAddress, _count: FrameCount) { + unimplemented!("BumpAllocator::free not implemented"); + } + + unsafe fn usage(&self) -> FrameUsage { + let total = self.orig_areas.0.iter().map(|a| a.size).sum::() - self.orig_areas.1; + let free = self.cur_areas.0.iter().map(|a| a.size).sum::() - self.cur_areas.1; + FrameUsage::new( + FrameCount::new((total - free) / A::PAGE_SIZE), + FrameCount::new(total / A::PAGE_SIZE), + ) + } +} diff --git a/rmm/src/allocator/frame/mod.rs b/rmm/src/allocator/frame/mod.rs new file mode 100644 index 0000000000..1260def01b --- /dev/null +++ b/rmm/src/allocator/frame/mod.rs @@ -0,0 +1,83 @@ +use crate::PhysicalAddress; + +pub use self::{buddy::*, bump::*}; + +mod buddy; +mod bump; + +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct FrameCount(usize); + +impl FrameCount { + pub fn new(count: usize) -> Self { + Self(count) + } + + pub fn data(&self) -> usize { + self.0 + } +} + +#[derive(Debug)] +pub struct FrameUsage { + used: FrameCount, + total: FrameCount, +} + +impl FrameUsage { + pub fn new(used: FrameCount, total: FrameCount) -> Self { + Self { used, total } + } + + pub fn used(&self) -> FrameCount { + self.used + } + + pub fn free(&self) -> FrameCount { + FrameCount(self.total.0 - self.used.0) + } + + pub fn total(&self) -> FrameCount { + self.total + } +} + +pub trait FrameAllocator { + unsafe fn allocate(&mut self, count: FrameCount) -> Option; + + unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount); + + unsafe fn allocate_one(&mut self) -> Option { + unsafe { self.allocate(FrameCount::new(1)) } + } + + unsafe fn free_one(&mut self, address: PhysicalAddress) { + unsafe { + self.free(address, FrameCount::new(1)); + } + } + + unsafe fn usage(&self) -> FrameUsage; +} + +impl FrameAllocator for &mut T +where + T: FrameAllocator, +{ + unsafe fn allocate(&mut self, count: FrameCount) -> Option { + unsafe { T::allocate(self, count) } + } + unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount) { + unsafe { T::free(self, address, count) } + } + unsafe fn allocate_one(&mut self) -> Option { + unsafe { T::allocate_one(self) } + } + unsafe fn free_one(&mut self, address: PhysicalAddress) { + unsafe { T::free_one(self, address) } + } + unsafe fn usage(&self) -> FrameUsage { + unsafe { T::usage(self) } + } +} diff --git a/rmm/src/allocator/mod.rs b/rmm/src/allocator/mod.rs new file mode 100644 index 0000000000..0027153038 --- /dev/null +++ b/rmm/src/allocator/mod.rs @@ -0,0 +1,3 @@ +pub use self::frame::*; + +mod frame; diff --git a/rmm/src/arch/aarch64.rs b/rmm/src/arch/aarch64.rs new file mode 100644 index 0000000000..118ac956ea --- /dev/null +++ b/rmm/src/arch/aarch64.rs @@ -0,0 +1,129 @@ +use core::arch::asm; + +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; + +#[derive(Clone, Copy)] +pub struct AArch64Arch; + +impl Arch for AArch64Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each + const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3 + + //TODO + const ENTRY_ADDRESS_WIDTH: usize = 40; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT + | 1 << 1 // Page flag + | 1 << 10 // Access flag + | Self::ENTRY_FLAG_NO_GLOBAL; + const ENTRY_FLAG_DEFAULT_TABLE: usize + = Self::ENTRY_FLAG_PRESENT + | Self::ENTRY_FLAG_READWRITE + | 1 << 1 // Table flag + | 1 << 10 // Access flag + ; + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_READONLY: usize = 1 << 7; + const ENTRY_FLAG_READWRITE: usize = 0; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 6; + // This sets both userspace and privileged execute never + //TODO: Separate the two? + const ENTRY_FLAG_NO_EXEC: usize = 0b11 << 53; + const ENTRY_FLAG_EXEC: usize = 0; + const ENTRY_FLAG_GLOBAL: usize = 0; + const ENTRY_FLAG_NO_GLOBAL: usize = 1 << 11; + const ENTRY_FLAG_WRITE_COMBINING: usize = 0; + + const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!("AArch64Arch::init unimplemented"); + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + unsafe { + asm!(" + dsb ishst + tlbi vaae1is, {} + dsb ish + isb + ", in(reg) (address.data() >> Self::PAGE_SHIFT)); + } + } + + #[inline(always)] + unsafe fn invalidate_all() { + unsafe { + asm!( + " + dsb ishst + tlbi vmalle1is + dsb ish + isb + " + ); + } + } + + #[inline(always)] + unsafe fn table(table_kind: TableKind) -> PhysicalAddress { + unsafe { + let address: usize; + match table_kind { + TableKind::User => { + asm!("mrs {0}, ttbr0_el1", out(reg) address); + } + TableKind::Kernel => { + asm!("mrs {0}, ttbr1_el1", out(reg) address); + } + } + PhysicalAddress::new(address) + } + } + + #[inline(always)] + unsafe fn set_table(table_kind: TableKind, address: PhysicalAddress) { + unsafe { + match table_kind { + TableKind::User => { + asm!("msr ttbr0_el1, {0}", in(reg) address.data()); + } + TableKind::Kernel => { + asm!("msr ttbr1_el1, {0}", in(reg) address.data()); + } + } + Self::invalidate_all(); + } + } + + fn virt_is_valid(_address: VirtualAddress) -> bool { + //TODO: what makes an address valid on aarch64? + true + } +} + +#[cfg(test)] +mod tests { + use super::AArch64Arch; + use crate::Arch; + + #[test] + fn constants() { + assert_eq!(AArch64Arch::PAGE_SIZE, 4096); + assert_eq!(AArch64Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(AArch64Arch::PAGE_ADDRESS_SHIFT, 48); + assert_eq!(AArch64Arch::PAGE_ADDRESS_SIZE, 0x0001_0000_0000_0000); + assert_eq!(AArch64Arch::PAGE_ADDRESS_MASK, 0x0000_FFFF_FFFF_F000); + assert_eq!(AArch64Arch::PAGE_ENTRY_SIZE, 8); + assert_eq!(AArch64Arch::PAGE_ENTRIES, 512); + assert_eq!(AArch64Arch::PAGE_ENTRY_MASK, 0x1FF); + assert_eq!(AArch64Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); + + assert_eq!(AArch64Arch::ENTRY_ADDRESS_SIZE, 0x0000_0100_0000_0000); + assert_eq!(AArch64Arch::ENTRY_ADDRESS_MASK, 0x0000_00FF_FFFF_FFFF); + assert_eq!(AArch64Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + + assert_eq!(AArch64Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); + } +} diff --git a/rmm/src/arch/emulate.rs b/rmm/src/arch/emulate.rs new file mode 100644 index 0000000000..5e8bdb3c2a --- /dev/null +++ b/rmm/src/arch/emulate.rs @@ -0,0 +1,347 @@ +use std::{collections::BTreeMap, marker::PhantomData, mem, ptr, sync::Mutex}; + +use crate::{ + arch::x86_64::X8664Arch, page::PageFlags, Arch, MemoryArea, PageEntry, PhysicalAddress, + TableKind, VirtualAddress, MEGABYTE, +}; + +#[derive(Clone, Copy)] +pub struct EmulateArch; + +impl Arch for EmulateArch { + const PAGE_SHIFT: usize = X8664Arch::PAGE_SHIFT; + const PAGE_ENTRY_SHIFT: usize = X8664Arch::PAGE_ENTRY_SHIFT; + const PAGE_LEVELS: usize = X8664Arch::PAGE_LEVELS; + + const ENTRY_ADDRESS_SHIFT: usize = X8664Arch::ENTRY_ADDRESS_SHIFT; + const ENTRY_FLAG_DEFAULT_PAGE: usize = X8664Arch::ENTRY_FLAG_DEFAULT_PAGE; + const ENTRY_FLAG_DEFAULT_TABLE: usize = X8664Arch::ENTRY_FLAG_DEFAULT_TABLE; + const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_READONLY: usize = X8664Arch::ENTRY_FLAG_READONLY; + const ENTRY_FLAG_READWRITE: usize = X8664Arch::ENTRY_FLAG_READWRITE; + const ENTRY_FLAG_PAGE_USER: usize = X8664Arch::ENTRY_FLAG_PAGE_USER; + const ENTRY_FLAG_NO_EXEC: usize = X8664Arch::ENTRY_FLAG_NO_EXEC; + const ENTRY_FLAG_EXEC: usize = X8664Arch::ENTRY_FLAG_EXEC; + + const PHYS_OFFSET: usize = X8664Arch::PHYS_OFFSET; + + const ENTRY_FLAG_GLOBAL: usize = X8664Arch::ENTRY_FLAG_GLOBAL; + const ENTRY_FLAG_NO_GLOBAL: usize = X8664Arch::ENTRY_FLAG_NO_GLOBAL; + + const ENTRY_ADDRESS_WIDTH: usize = X8664Arch::ENTRY_ADDRESS_WIDTH; + + const ENTRY_FLAG_WRITE_COMBINING: usize = X8664Arch::ENTRY_FLAG_WRITE_COMBINING; + + unsafe fn init() -> &'static [MemoryArea] { + unsafe { + // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) + let mut machine = Machine::new(MEMORY_SIZE); + + // PML4 index 256 (PHYS_OFFSET) link to PDP + let pml4 = 0; + let pdp = pml4 + Self::PAGE_SIZE; + let flags = Self::ENTRY_FLAG_READWRITE | Self::ENTRY_FLAG_PRESENT; + machine.write_phys::( + PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), + pdp | flags, + ); + + // PDP link to PD + let pd = pdp + Self::PAGE_SIZE; + machine.write_phys::(PhysicalAddress::new(pdp), pd | flags); + + // PD link to PT + let pt = pd + Self::PAGE_SIZE; + machine.write_phys::(PhysicalAddress::new(pd), pt | flags); + + // PT links to frames + for i in 0..Self::PAGE_ENTRIES { + let page = i * Self::PAGE_SIZE; + machine.write_phys::( + PhysicalAddress::new(pt + i * Self::PAGE_ENTRY_SIZE), + page | flags, + ); + } + + *MACHINE.lock().unwrap() = Some(machine); + + // Set table to pml4 + EmulateArch::set_table(TableKind::Kernel, PhysicalAddress::new(pml4)); + + &MEMORY_AREAS + } + } + + #[inline(always)] + unsafe fn read(address: VirtualAddress) -> T { + MACHINE.lock().unwrap().as_ref().unwrap().read(address) + } + + #[inline(always)] + unsafe fn write(address: VirtualAddress, value: T) { + MACHINE + .lock() + .unwrap() + .as_mut() + .unwrap() + .write(address, value) + } + + #[inline(always)] + unsafe fn write_bytes(address: VirtualAddress, value: u8, count: usize) { + MACHINE + .lock() + .unwrap() + .as_mut() + .unwrap() + .write_bytes(address, value, count) + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + MACHINE + .lock() + .unwrap() + .as_mut() + .unwrap() + .invalidate(address); + } + + #[inline(always)] + unsafe fn invalidate_all() { + MACHINE.lock().unwrap().as_mut().unwrap().invalidate_all(); + } + + #[inline(always)] + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { + MACHINE.lock().unwrap().as_mut().unwrap().get_table() + } + + #[inline(always)] + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { + MACHINE.lock().unwrap().as_mut().unwrap().set_table(address); + } + fn virt_is_valid(_address: VirtualAddress) -> bool { + // TODO: Don't see why an emulated arch would have any problems with canonicalness... + true + } +} + +const MEMORY_SIZE: usize = 64 * MEGABYTE; +static MEMORY_AREAS: [MemoryArea; 2] = [ + MemoryArea { + base: PhysicalAddress::new(EmulateArch::PAGE_SIZE * 4), // Initial PML4, PDP, PD, and PT wasted + size: MEMORY_SIZE / 2 - EmulateArch::PAGE_SIZE * 4, + }, + // Second area for debugging + MemoryArea { + base: PhysicalAddress::new(MEMORY_SIZE / 2), + size: MEMORY_SIZE / 2, + }, +]; + +static MACHINE: Mutex>> = Mutex::new(None); + +struct Machine { + memory: Box<[u8]>, + map: BTreeMap>, + table_addr: PhysicalAddress, + phantom: PhantomData, +} + +impl Machine { + fn new(memory_size: usize) -> Self { + Self { + memory: vec![0; memory_size].into_boxed_slice(), + map: BTreeMap::new(), + table_addr: PhysicalAddress::new(0), + phantom: PhantomData, + } + } + + fn read_phys(&self, phys: PhysicalAddress) -> T { + let size = mem::size_of::(); + if phys.add(size).data() <= self.memory.len() { + unsafe { ptr::read(self.memory.as_ptr().add(phys.data()) as *const T) } + } else { + panic!( + "read_phys: 0x{:X} size 0x{:X} outside of memory", + phys.data(), + size + ); + } + } + + fn write_phys(&mut self, phys: PhysicalAddress, value: T) { + let size = mem::size_of::(); + if phys.add(size).data() <= self.memory.len() { + unsafe { + ptr::write(self.memory.as_mut_ptr().add(phys.data()) as *mut T, value); + } + } else { + panic!( + "write_phys: 0x{:X} size 0x{:X} outside of memory", + phys.data(), + size + ); + } + } + + fn write_phys_bytes(&mut self, phys: PhysicalAddress, value: u8, count: usize) { + if phys.add(count).data() <= self.memory.len() { + unsafe { + ptr::write_bytes(self.memory.as_mut_ptr().add(phys.data()), value, count); + } + } else { + panic!( + "write_phys_bytes: 0x{:X} count 0x{:X} outside of memory", + phys.data(), + count + ); + } + } + + fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, PageFlags)> { + let virt_data = virt.data(); + let page = virt_data & A::PAGE_ADDRESS_MASK; + let offset = virt_data & A::PAGE_OFFSET_MASK; + let entry = self.map.get(&VirtualAddress::new(page))?; + Some((entry.address().ok()?.add(offset), entry.flags())) + } + + fn read(&self, virt: VirtualAddress) -> T { + //TODO: allow reading past page boundaries + let virt_data = virt.data(); + let size = mem::size_of::(); + if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) { + panic!( + "read: 0x{:X} size 0x{:X} passes page boundary", + virt_data, size + ); + } + + if let Some((phys, _flags)) = self.translate(virt) { + self.read_phys(phys) + } else { + panic!("read: 0x{:X} size 0x{:X} not present", virt_data, size); + } + } + + fn write(&mut self, virt: VirtualAddress, value: T) { + //TODO: allow writing past page boundaries + let virt_data = virt.data(); + let size = mem::size_of::(); + if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) { + panic!( + "write: 0x{:X} size 0x{:X} passes page boundary", + virt_data, size + ); + } + + if let Some((phys, flags)) = self.translate(virt) { + if flags.has_write() { + self.write_phys(phys, value); + } else { + panic!("write: 0x{:X} size 0x{:X} not writable", virt_data, size); + } + } else { + panic!("write: 0x{:X} size 0x{:X} not present", virt_data, size); + } + } + + fn write_bytes(&mut self, virt: VirtualAddress, value: u8, count: usize) { + //TODO: allow writing past page boundaries + let virt_data = virt.data(); + if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (count - 1)) & A::PAGE_ADDRESS_MASK) + { + panic!( + "write_bytes: 0x{:X} count 0x{:X} passes page boundary", + virt_data, count + ); + } + + if let Some((phys, flags)) = self.translate(virt) { + if flags.has_write() { + self.write_phys_bytes(phys, value, count); + } else { + panic!( + "write_bytes: 0x{:X} count 0x{:X} not writable", + virt_data, count + ); + } + } else { + panic!( + "write_bytes: 0x{:X} count 0x{:X} not present", + virt_data, count + ); + } + } + + fn invalidate(&mut self, _address: VirtualAddress) { + unimplemented!("EmulateArch::invalidate not implemented"); + } + + //TODO: cleanup + fn invalidate_all(&mut self) { + self.map.clear(); + + // PML4 + let a4 = self.table_addr.data(); + for i4 in 0..A::PAGE_ENTRIES { + let e3 = self.read_phys::(PhysicalAddress::new(a4 + i4 * A::PAGE_ENTRY_SIZE)); + let f3 = e3 & A::ENTRY_FLAGS_MASK; + if f3 & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } + + // Page directory pointer + let a3 = ((e3 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT; + for i3 in 0..A::PAGE_ENTRIES { + let e2 = + self.read_phys::(PhysicalAddress::new(a3 + i3 * A::PAGE_ENTRY_SIZE)); + let f2 = e2 & A::ENTRY_FLAGS_MASK; + if f2 & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } + + // Page directory + let a2 = ((e2 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT; + for i2 in 0..A::PAGE_ENTRIES { + let e1 = + self.read_phys::(PhysicalAddress::new(a2 + i2 * A::PAGE_ENTRY_SIZE)); + let f1 = e1 & A::ENTRY_FLAGS_MASK; + if f1 & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } + + // Page table + let a1 = + ((e1 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT; + for i1 in 0..A::PAGE_ENTRIES { + let e = self + .read_phys::(PhysicalAddress::new(a1 + i1 * A::PAGE_ENTRY_SIZE)); + let f = e & A::ENTRY_FLAGS_MASK; + if f & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } + + // Page + let page = (i4 << 39) | (i3 << 30) | (i2 << 21) | (i1 << 12); + //println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); + self.map + .insert(VirtualAddress::new(page), PageEntry::from_data(e)); + } + } + } + } + } + + fn get_table(&self) -> PhysicalAddress { + self.table_addr + } + + fn set_table(&mut self, address: PhysicalAddress) { + self.table_addr = address; + self.invalidate_all(); + } +} diff --git a/rmm/src/arch/mod.rs b/rmm/src/arch/mod.rs new file mode 100644 index 0000000000..a5f92a80bc --- /dev/null +++ b/rmm/src/arch/mod.rs @@ -0,0 +1,104 @@ +use core::ptr; + +use crate::{MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; + +//TODO: Support having all page tables compile on all architectures +#[cfg(all(feature = "std", target_pointer_width = "64"))] +pub use self::emulate::EmulateArch; +#[cfg(target_pointer_width = "32")] +pub use self::x86::X86Arch; +#[cfg(target_pointer_width = "64")] +pub use self::{ + aarch64::AArch64Arch, + riscv64::{RiscV64Sv39Arch, RiscV64Sv48Arch}, + x86_64::X8664Arch, +}; + +#[cfg(target_pointer_width = "64")] +mod aarch64; +#[cfg(all(feature = "std", target_pointer_width = "64"))] +mod emulate; +#[cfg(target_pointer_width = "64")] +mod riscv64; +#[cfg(target_pointer_width = "32")] +mod x86; +#[cfg(target_pointer_width = "64")] +mod x86_64; + +pub trait Arch: Clone + Copy { + const PAGE_SHIFT: usize; + const PAGE_ENTRY_SHIFT: usize; + const PAGE_LEVELS: usize; + + const ENTRY_ADDRESS_WIDTH: usize; // Number of bits of physical address in PTE + const ENTRY_ADDRESS_SHIFT: usize = Self::PAGE_SHIFT; // Offset of physical address in PTE + const ENTRY_FLAG_DEFAULT_PAGE: usize; + const ENTRY_FLAG_DEFAULT_TABLE: usize; + const ENTRY_FLAG_PRESENT: usize; + const ENTRY_FLAG_READONLY: usize; + const ENTRY_FLAG_READWRITE: usize; + const ENTRY_FLAG_PAGE_USER: usize; // Leaf table user page flag + const ENTRY_FLAG_TABLE_USER: usize = Self::ENTRY_FLAG_PAGE_USER; // Directory user page table flag + const ENTRY_FLAG_NO_EXEC: usize; + const ENTRY_FLAG_EXEC: usize; + const ENTRY_FLAG_GLOBAL: usize; + const ENTRY_FLAG_NO_GLOBAL: usize; + const ENTRY_FLAG_WRITE_COMBINING: usize; + + const PHYS_OFFSET: usize; + + const PAGE_SIZE: usize = 1 << Self::PAGE_SHIFT; + const PAGE_OFFSET_MASK: usize = Self::PAGE_SIZE - 1; + const PAGE_ADDRESS_SHIFT: usize = Self::PAGE_LEVELS * Self::PAGE_ENTRY_SHIFT + Self::PAGE_SHIFT; + const PAGE_ADDRESS_SIZE: u64 = 1 << (Self::PAGE_ADDRESS_SHIFT as u64); + const PAGE_ADDRESS_MASK: usize = (Self::PAGE_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize; + const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT); + const PAGE_ENTRIES: usize = 1 << Self::PAGE_ENTRY_SHIFT; + const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; + const PAGE_NEGATIVE_MASK: usize = !(Self::PAGE_ADDRESS_SIZE - 1) as usize; + + const ENTRY_ADDRESS_SIZE: usize = 1 << Self::ENTRY_ADDRESS_WIDTH; // size of addressable physical memory, in pages + const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - 1; // Mask of physical address, starting at 0th bit + const ENTRY_FLAGS_MASK: usize = !(Self::ENTRY_ADDRESS_MASK << Self::ENTRY_ADDRESS_SHIFT); + + unsafe fn init() -> &'static [MemoryArea]; + + #[inline(always)] + unsafe fn read(address: VirtualAddress) -> T { + unsafe { ptr::read(address.data() as *const T) } + } + + #[inline(always)] + unsafe fn write(address: VirtualAddress, value: T) { + unsafe { ptr::write(address.data() as *mut T, value) } + } + + #[inline(always)] + unsafe fn write_bytes(address: VirtualAddress, value: u8, count: usize) { + unsafe { ptr::write_bytes(address.data() as *mut u8, value, count) } + } + + unsafe fn invalidate(address: VirtualAddress); + + #[inline(always)] + unsafe fn invalidate_all() { + unsafe { + //TODO: this stub only works on x86_64, maybe make the arch implement this? + Self::set_table(TableKind::User, Self::table(TableKind::User)); + } + } + + unsafe fn table(table_kind: TableKind) -> PhysicalAddress; + + unsafe fn set_table(table_kind: TableKind, address: PhysicalAddress); + + #[inline(always)] + unsafe fn phys_to_virt(phys: PhysicalAddress) -> VirtualAddress { + match phys.data().checked_add(Self::PHYS_OFFSET) { + Some(some) => VirtualAddress::new(some), + None => panic!("phys_to_virt({:#x}) overflow", phys.data()), + } + } + + fn virt_is_valid(address: VirtualAddress) -> bool; +} diff --git a/rmm/src/arch/riscv64/mod.rs b/rmm/src/arch/riscv64/mod.rs new file mode 100644 index 0000000000..894ef349e4 --- /dev/null +++ b/rmm/src/arch/riscv64/mod.rs @@ -0,0 +1,5 @@ +pub use sv39::RiscV64Sv39Arch; +pub use sv48::RiscV64Sv48Arch; + +mod sv39; +mod sv48; diff --git a/rmm/src/arch/riscv64/sv39.rs b/rmm/src/arch/riscv64/sv39.rs new file mode 100644 index 0000000000..79d1a94484 --- /dev/null +++ b/rmm/src/arch/riscv64/sv39.rs @@ -0,0 +1,130 @@ +use core::arch::asm; + +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; + +#[derive(Clone, Copy)] +pub struct RiscV64Sv39Arch; + +pub const ACCESSED: usize = 1 << 6; +pub const DIRTY: usize = 1 << 7; + +impl Arch for RiscV64Sv39Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each + const PAGE_LEVELS: usize = 3; // L0, L1, L2 + + const ENTRY_ADDRESS_WIDTH: usize = 44; + const ENTRY_ADDRESS_SHIFT: usize = 10; + + const ENTRY_FLAG_DEFAULT_PAGE: usize = + Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY | ACCESSED | DIRTY; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_READONLY: usize = 1 << 1; + const ENTRY_FLAG_READWRITE: usize = 3 << 1; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; + const ENTRY_FLAG_TABLE_USER: usize = 0; + const ENTRY_FLAG_NO_EXEC: usize = 0; + const ENTRY_FLAG_EXEC: usize = 1 << 3; + const ENTRY_FLAG_GLOBAL: usize = 1 << 5; + const ENTRY_FLAG_NO_GLOBAL: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 0; + + const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000; + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!("RiscV64Sv39Arch::init unimplemented"); + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + unsafe { + asm!("sfence.vma {}", in(reg) address.data()); + } + } + + #[inline(always)] + unsafe fn invalidate_all() { + unsafe { + asm!("sfence.vma"); + } + } + + #[inline(always)] + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { + unsafe { + let satp: usize; + asm!("csrr {0}, satp", out(reg) satp); + PhysicalAddress::new( + (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN + ) + } + } + + #[inline(always)] + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { + unsafe { + let satp = (8 << 60) | // Sv39 MODE + (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) + asm!("csrw satp, {0}", in(reg) satp); + Self::invalidate_all(); + } + } + + fn virt_is_valid(address: VirtualAddress) -> bool { + let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1); + let masked = address.data() & mask; + + masked == mask || masked == 0 + } +} + +#[cfg(test)] +mod tests { + use super::RiscV64Sv39Arch; + use crate::Arch; + + #[test] + fn constants() { + assert_eq!(RiscV64Sv39Arch::PAGE_SIZE, 4096); + assert_eq!(RiscV64Sv39Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(RiscV64Sv39Arch::PAGE_ADDRESS_SHIFT, 39); + assert_eq!(RiscV64Sv39Arch::PAGE_ADDRESS_SIZE, 0x0000_0080_0000_0000); + assert_eq!(RiscV64Sv39Arch::PAGE_ADDRESS_MASK, 0x0000_007F_FFFF_F000); + assert_eq!(RiscV64Sv39Arch::PAGE_ENTRY_SIZE, 8); + assert_eq!(RiscV64Sv39Arch::PAGE_ENTRIES, 512); + assert_eq!(RiscV64Sv39Arch::PAGE_ENTRY_MASK, 0x1FF); + assert_eq!(RiscV64Sv39Arch::PAGE_NEGATIVE_MASK, 0xFFFF_FF80_0000_0000); + + assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_SIZE, 0x0000_1000_0000_0000); + assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_MASK, 0x0000_0FFF_FFFF_FFFF); + assert_eq!(RiscV64Sv39Arch::ENTRY_FLAGS_MASK, 0xFFC0_0000_0000_03FF); + + assert_eq!(RiscV64Sv39Arch::PHYS_OFFSET, 0xFFFF_FFC0_0000_0000); + } + #[test] + fn is_canonical() { + use super::VirtualAddress; + + #[track_caller] + fn yes(addr: usize) { + assert!(RiscV64Sv39Arch::virt_is_valid(VirtualAddress::new(addr))); + } + #[track_caller] + fn no(addr: usize) { + assert!(!RiscV64Sv39Arch::virt_is_valid(VirtualAddress::new(addr))); + } + + yes(0xFFFF_FFFF_FFFF_FFFF); + yes(0xFFFF_FFF0_1337_1337); + no(0x0000_0F00_0000_0000); + no(0x1337_0000_0000_0000); + no(1 << 38); + yes(1 << 37); + + // Check for off-by-one errors. + yes(0xFFFF_FFC0_0000_0000 | (1 << 37)); + yes(0xFFFF_FFE0_0000_0000 | (1 << 37)); + no(0xFFFF_FF80_0000_0000 | (1 << 37)); + } +} diff --git a/rmm/src/arch/riscv64/sv48.rs b/rmm/src/arch/riscv64/sv48.rs new file mode 100644 index 0000000000..8d6e2faf0c --- /dev/null +++ b/rmm/src/arch/riscv64/sv48.rs @@ -0,0 +1,124 @@ +use core::arch::asm; + +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; + +#[derive(Clone, Copy)] +pub struct RiscV64Sv48Arch; + +impl Arch for RiscV64Sv48Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each + const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3 + + const ENTRY_ADDRESS_WIDTH: usize = 44; + const ENTRY_ADDRESS_SHIFT: usize = 10; + + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_READONLY: usize = 1 << 1; + const ENTRY_FLAG_READWRITE: usize = 3 << 1; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; + const ENTRY_FLAG_TABLE_USER: usize = 0; + const ENTRY_FLAG_NO_EXEC: usize = 0; + const ENTRY_FLAG_EXEC: usize = 1 << 3; + const ENTRY_FLAG_GLOBAL: usize = 1 << 5; + const ENTRY_FLAG_NO_GLOBAL: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 0; + + const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!("RiscV64Sv48Arch::init unimplemented"); + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + unsafe { + asm!("sfence.vma {}", in(reg) address.data()); + } + } + + #[inline(always)] + unsafe fn invalidate_all() { + unsafe { + asm!("sfence.vma"); + } + } + + #[inline(always)] + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { + unsafe { + let satp: usize; + asm!("csrr {0}, satp", out(reg) satp); + PhysicalAddress::new( + (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN + ) + } + } + + #[inline(always)] + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { + unsafe { + let satp = (9 << 60) | // Sv48 MODE + (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) + asm!("csrw satp, {0}", in(reg) satp); + Self::invalidate_all(); + } + } + + fn virt_is_valid(address: VirtualAddress) -> bool { + // RISC-V SV48 uses 48-bit sign-extended addresses, identical to 4-level paging on x86_64. + let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1); + let masked = address.data() & mask; + + masked == mask || masked == 0 + } +} + +#[cfg(test)] +mod tests { + use super::RiscV64Sv48Arch; + use crate::Arch; + + #[test] + fn constants() { + assert_eq!(RiscV64Sv48Arch::PAGE_SIZE, 4096); + assert_eq!(RiscV64Sv48Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(RiscV64Sv48Arch::PAGE_ADDRESS_SHIFT, 48); + assert_eq!(RiscV64Sv48Arch::PAGE_ADDRESS_SIZE, 0x0001_0000_0000_0000); + assert_eq!(RiscV64Sv48Arch::PAGE_ADDRESS_MASK, 0x0000_FFFF_FFFF_F000); + assert_eq!(RiscV64Sv48Arch::PAGE_ENTRY_SIZE, 8); + assert_eq!(RiscV64Sv48Arch::PAGE_ENTRIES, 512); + assert_eq!(RiscV64Sv48Arch::PAGE_ENTRY_MASK, 0x1FF); + assert_eq!(RiscV64Sv48Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); + + assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_SIZE, 0x0000_1000_0000_0000); + assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_MASK, 0x0000_0FFF_FFFF_FFFF); + assert_eq!(RiscV64Sv48Arch::ENTRY_FLAGS_MASK, 0xFFC0_0000_0000_03FF); + + assert_eq!(RiscV64Sv48Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); + } + #[test] + fn is_canonical() { + use super::VirtualAddress; + + // Close to identical when compared to x86_64 test. + fn yes(address: usize) { + assert!(RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new(address))); + } + fn no(address: usize) { + assert!(!RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new( + address + ))); + } + + yes(0xFFFF_8000_1337_1337); + yes(0xFFFF_FFFF_FFFF_FFFF); + yes(0x0000_0000_0000_0042); + yes(0x0000_7FFF_FFFF_FFFF); + no(0x1337_0000_0000_0000); + no(0x1337_8000_0000_0000); + no(0x0000_8000_0000_0000); + } +} diff --git a/rmm/src/arch/x86.rs b/rmm/src/arch/x86.rs new file mode 100644 index 0000000000..7e078a2fc0 --- /dev/null +++ b/rmm/src/arch/x86.rs @@ -0,0 +1,80 @@ +//TODO: USE PAE +use core::arch::asm; + +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; + +#[derive(Clone, Copy)] +pub struct X86Arch; + +impl Arch for X86Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 10; // 1024 entries, 4 bytes each + const PAGE_LEVELS: usize = 2; // PD, PT + + const ENTRY_ADDRESS_WIDTH: usize = 20; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READWRITE; + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_READONLY: usize = 0; + const ENTRY_FLAG_READWRITE: usize = 1 << 1; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 2; + // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; + const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + const ENTRY_FLAG_NO_GLOBAL: usize = 0; + const ENTRY_FLAG_NO_EXEC: usize = 0; // NOT AVAILABLE UNLESS PAE IS USED! + const ENTRY_FLAG_EXEC: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 1 << 7; + + const PHYS_OFFSET: usize = 0x8000_0000; + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!("X86Arch::init unimplemented"); + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + asm!("invlpg [{0}]", in(reg) address.data()); + } + + #[inline(always)] + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { + let address: usize; + asm!("mov {0}, cr3", out(reg) address); + PhysicalAddress::new(address) + } + + #[inline(always)] + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { + asm!("mov cr3, {0}", in(reg) address.data()); + } + + fn virt_is_valid(_address: VirtualAddress) -> bool { + // On 32-bit x86, every virtual address is valid + true + } +} + +#[cfg(test)] +mod tests { + use super::{VirtualAddress, X86Arch}; + use crate::Arch; + + #[test] + fn constants() { + assert_eq!(X86Arch::PAGE_SIZE, 4096); + assert_eq!(X86Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(X86Arch::PAGE_ADDRESS_SHIFT, 32); + assert_eq!(X86Arch::PAGE_ADDRESS_SIZE, 0x0000_0001_0000_0000); + assert_eq!(X86Arch::PAGE_ADDRESS_MASK, 0xFFFF_F000); + assert_eq!(X86Arch::PAGE_ENTRY_SIZE, 4); + assert_eq!(X86Arch::PAGE_ENTRIES, 1024); + assert_eq!(X86Arch::PAGE_ENTRY_MASK, 0x3FF); + assert_eq!(X86Arch::PAGE_NEGATIVE_MASK, 0x0000_0000_0000); + + assert_eq!(X86Arch::ENTRY_ADDRESS_SIZE, 0x0000_0000_0010_0000); + assert_eq!(X86Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF); + assert_eq!(X86Arch::ENTRY_FLAGS_MASK, 0x0000_0FFF); + + assert_eq!(X86Arch::PHYS_OFFSET, 0x8000_0000); + } +} diff --git a/rmm/src/arch/x86_64.rs b/rmm/src/arch/x86_64.rs new file mode 100644 index 0000000000..d8b44357cc --- /dev/null +++ b/rmm/src/arch/x86_64.rs @@ -0,0 +1,106 @@ +use core::arch::asm; + +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; + +#[derive(Clone, Copy, Debug)] +pub struct X8664Arch; + +impl Arch for X8664Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each + const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT + + const ENTRY_ADDRESS_WIDTH: usize = 40; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READWRITE; + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_READONLY: usize = 0; + const ENTRY_FLAG_READWRITE: usize = 1 << 1; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 2; + // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; + const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + const ENTRY_FLAG_NO_GLOBAL: usize = 0; + const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; + const ENTRY_FLAG_EXEC: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 1 << 7; + + const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1) as usize; // PML4 slot 256 and onwards + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!("X8664Arch::init unimplemented"); + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + unsafe { + asm!("invlpg [{0}]", in(reg) address.data()); + } + } + + #[inline(always)] + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { + unsafe { + let address: usize; + asm!("mov {0}, cr3", out(reg) address); + PhysicalAddress::new(address) + } + } + + #[inline(always)] + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { + unsafe { + asm!("mov cr3, {0}", in(reg) address.data()); + } + } + + fn virt_is_valid(address: VirtualAddress) -> bool { + // On x86_64, an address is valid if and only if it is canonical. It may still point to + // unmapped memory, but will always be valid once translated via the page table has + // suceeded. + let masked = address.data() & 0xFFFF_8000_0000_0000; + // TODO: 5-level paging + masked == 0xFFFF_8000_0000_0000 || masked == 0 + } +} + +#[cfg(test)] +mod tests { + use super::{VirtualAddress, X8664Arch}; + use crate::Arch; + + #[test] + fn constants() { + assert_eq!(X8664Arch::PAGE_SIZE, 4096); + assert_eq!(X8664Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(X8664Arch::PAGE_ADDRESS_SHIFT, 48); + assert_eq!(X8664Arch::PAGE_ADDRESS_SIZE, 0x0001_0000_0000_0000); + assert_eq!(X8664Arch::PAGE_ADDRESS_MASK, 0x0000_FFFF_FFFF_F000); + assert_eq!(X8664Arch::PAGE_ENTRY_SIZE, 8); + assert_eq!(X8664Arch::PAGE_ENTRIES, 512); + assert_eq!(X8664Arch::PAGE_ENTRY_MASK, 0x1FF); + assert_eq!(X8664Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); + + assert_eq!(X8664Arch::ENTRY_ADDRESS_SIZE, 0x0000_0100_0000_0000); + assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x0000_00FF_FFFF_FFFF); + assert_eq!(X8664Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + + assert_eq!(X8664Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); + } + #[test] + fn is_canonical() { + fn yes(address: usize) { + assert!(X8664Arch::virt_is_valid(VirtualAddress::new(address))); + } + fn no(address: usize) { + assert!(!X8664Arch::virt_is_valid(VirtualAddress::new(address))); + } + + yes(0xFFFF_8000_1337_1337); + yes(0xFFFF_FFFF_FFFF_FFFF); + yes(0x0000_0000_0000_0042); + yes(0x0000_7FFF_FFFF_FFFF); + no(0x1337_0000_0000_0000); + no(0x1337_8000_0000_0000); + no(0x0000_8000_0000_0000); + } +} diff --git a/rmm/src/lib.rs b/rmm/src/lib.rs new file mode 100644 index 0000000000..bb6e38cd7f --- /dev/null +++ b/rmm/src/lib.rs @@ -0,0 +1,94 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +pub use crate::{allocator::*, arch::*, page::*}; + +mod allocator; +mod arch; +mod page; + +pub const KILOBYTE: usize = 1024; +pub const MEGABYTE: usize = KILOBYTE * 1024; +pub const GIGABYTE: usize = MEGABYTE * 1024; +#[cfg(target_pointer_width = "64")] +pub const TERABYTE: usize = GIGABYTE * 1024; + +/// Specific table to be used, needed on some architectures +//TODO: Use this throughout the code +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum TableKind { + /// Userspace page table + User, + /// Kernel page table + Kernel, +} + +/// Physical memory address +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[repr(transparent)] +pub struct PhysicalAddress(usize); + +impl PhysicalAddress { + #[inline(always)] + pub const fn new(address: usize) -> Self { + Self(address) + } + + #[inline(always)] + pub fn data(&self) -> usize { + self.0 + } + + #[inline(always)] + pub fn add(self, offset: usize) -> Self { + Self(self.0 + offset) + } +} + +impl core::fmt::Debug for PhysicalAddress { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "[phys {:#0x}]", self.data()) + } +} + +/// Virtual memory address +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[repr(transparent)] +pub struct VirtualAddress(usize); + +impl VirtualAddress { + #[inline(always)] + pub const fn new(address: usize) -> Self { + Self(address) + } + + #[inline(always)] + pub fn data(&self) -> usize { + self.0 + } + + #[inline(always)] + pub fn add(self, offset: usize) -> Self { + Self(self.0 + offset) + } + + #[inline(always)] + pub fn kind(&self) -> TableKind { + if (self.0 as isize) < 0 { + TableKind::Kernel + } else { + TableKind::User + } + } +} + +impl core::fmt::Debug for VirtualAddress { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "[virt {:#0x}]", self.data()) + } +} + +#[derive(Clone, Copy, Debug)] +pub struct MemoryArea { + pub base: PhysicalAddress, + pub size: usize, +} diff --git a/rmm/src/main.rs b/rmm/src/main.rs new file mode 100644 index 0000000000..a63c24f305 --- /dev/null +++ b/rmm/src/main.rs @@ -0,0 +1,311 @@ +#![cfg(target_pointer_width = "64")] + +use rmm::{ + Arch, BuddyAllocator, BumpAllocator, EmulateArch, Flusher, FrameAllocator, FrameCount, + MemoryArea, PageFlags, PageFlushAll, PageMapper, PageTable, PhysicalAddress, TableKind, + VirtualAddress, GIGABYTE, KILOBYTE, MEGABYTE, TERABYTE, +}; +use std::marker::PhantomData; + +pub fn format_size(size: usize) -> String { + if size >= 2 * TERABYTE { + format!("{} TB", size / TERABYTE) + } else if size >= 2 * GIGABYTE { + format!("{} GB", size / GIGABYTE) + } else if size >= 2 * MEGABYTE { + format!("{} MB", size / MEGABYTE) + } else if size >= 2 * KILOBYTE { + format!("{} KB", size / KILOBYTE) + } else { + format!("{} B", size) + } +} + +#[allow(dead_code)] +unsafe fn dump_tables(table: PageTable) { + unsafe { + let level = table.level(); + for i in 0..A::PAGE_ENTRIES { + if level == 0 { + if let Some(entry) = table.entry(i) { + if entry.present() { + let base = table.entry_base(i).unwrap(); + println!( + "0x{:X}: 0x{:X}", + base.data(), + entry.address().unwrap().data() + ); + } + } + } else { + if let Some(next) = table.next(i) { + dump_tables(next); + } + } + } + } +} + +pub struct SlabNode { + next: PhysicalAddress, + count: usize, + phantom: PhantomData, +} + +impl SlabNode { + pub fn new(next: PhysicalAddress, count: usize) -> Self { + Self { + next, + count, + phantom: PhantomData, + } + } + + pub fn empty() -> Self { + Self::new(PhysicalAddress::new(0), 0) + } + + pub unsafe fn insert(&mut self, phys: PhysicalAddress) { + unsafe { + let virt = A::phys_to_virt(phys); + A::write(virt, self.next); + self.next = phys; + self.count += 1; + } + } + + pub unsafe fn remove(&mut self) -> Option { + unsafe { + if self.count > 0 { + let phys = self.next; + let virt = A::phys_to_virt(phys); + self.next = A::read(virt); + self.count -= 1; + Some(phys) + } else { + None + } + } + } +} + +pub struct SlabAllocator { + //TODO: Allow allocations up to maximum pageable size + nodes: [SlabNode; 4], + phantom: PhantomData, +} + +impl SlabAllocator { + pub unsafe fn new(areas: &'static [MemoryArea], offset: usize) -> Self { + unsafe { + let mut allocator = Self { + nodes: [ + SlabNode::empty(), + SlabNode::empty(), + SlabNode::empty(), + SlabNode::empty(), + ], + phantom: PhantomData, + }; + + // Add unused areas to free lists + let mut area_offset = offset; + for area in areas.iter() { + if area_offset < area.size { + let area_base = area.base.add(area_offset); + let area_size = area.size - area_offset; + allocator.free(area_base, area_size); + area_offset = 0; + } else { + area_offset -= area.size; + } + } + + allocator + } + } + + pub unsafe fn allocate(&mut self, size: usize) -> Option { + unsafe { + for level in 0..A::PAGE_LEVELS - 1 { + let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + let level_size = 1 << level_shift; + if size <= level_size { + if let Some(base) = self.nodes[level].remove() { + self.free(base.add(size), level_size - size); + return Some(base); + } + } + } + None + } + } + + //TODO: This causes fragmentation, since neighbors are not identified + //TODO: remainders less than PAGE_SIZE will be lost + pub unsafe fn free(&mut self, mut base: PhysicalAddress, mut size: usize) { + unsafe { + for level in (0..A::PAGE_LEVELS - 1).rev() { + let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + let level_size = 1 << level_shift; + while size >= level_size { + println!("Add {:X} {}", base.data(), format_size(level_size)); + self.nodes[level].insert(base); + base = base.add(level_size); + size -= level_size; + } + } + } + } + + pub unsafe fn remaining(&mut self) -> usize { + let mut remaining = 0; + for level in (0..A::PAGE_LEVELS - 1).rev() { + let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + let level_size = 1 << level_shift; + remaining += self.nodes[level].count * level_size; + } + remaining + } +} + +unsafe fn new_tables(areas: &'static [MemoryArea]) { + unsafe { + // First, calculate how much memory we have + let mut size = 0; + for area in areas.iter() { + size += area.size; + } + + println!("Memory: {}", format_size(size)); + + // Create a basic allocator for the first pages + let mut bump_allocator = BumpAllocator::::new(areas, 0); + + { + // Map all physical areas at PHYS_OFFSET + let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) + .expect("failed to create Mapper"); + for area in areas.iter() { + for i in 0..area.size / A::PAGE_SIZE { + let phys = area.base.add(i * A::PAGE_SIZE); + let virt = A::phys_to_virt(phys); + let flush = mapper + .map_phys(virt, phys, PageFlags::::new().write(true)) + .expect("failed to map page to frame"); + flush.ignore(); // Not the active table + } + } + + // Use the new table + mapper.make_current(); + } + + // Create the physical memory map + let offset = bump_allocator.offset(); + println!("Permanently used: {}", format_size(offset)); + + let mut allocator = BuddyAllocator::::new(bump_allocator).unwrap(); + + for i in 0..16 { + { + let phys_opt = allocator.allocate_one(); + println!("page {}: {:X?}", i, phys_opt); + if i % 3 == 0 { + if let Some(phys) = phys_opt { + println!("free {}: {:X?}", i, phys_opt); + allocator.free_one(phys); + } + } + } + + { + let phys_opt = allocator.allocate(FrameCount::new(16)); + println!("page*16 {}: {:X?}", i, phys_opt); + if i % 2 == 0 { + if let Some(phys) = phys_opt { + println!("free*16 {}: {:X?}", i, phys_opt); + allocator.free(phys, FrameCount::new(16)); + } + } + } + } + + let mut mapper = PageMapper::::current(TableKind::Kernel, &mut allocator); + let mut flush_all = PageFlushAll::new(); + for i in 0..16 { + let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); + let flush = mapper + .map(virt, PageFlags::::new().user(true).write(true)) + .expect("failed to map page"); + flush_all.consume(flush); + } + flush_all.flush(); + + let mut flush_all = PageFlushAll::new(); + for i in 0..16 { + let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); + let flush = mapper.unmap(virt, false).expect("failed to unmap page"); + flush_all.consume(flush); + } + flush_all.flush(); + + let usage = allocator.usage(); + println!("Allocator usage:"); + println!( + " Used: {}", + format_size(usage.used().data() * A::PAGE_SIZE) + ); + println!( + " Free: {}", + format_size(usage.free().data() * A::PAGE_SIZE) + ); + println!( + " Total: {}", + format_size(usage.total().data() * A::PAGE_SIZE) + ); + } +} + +unsafe fn inner() { + unsafe { + let areas = A::init(); + + // Debug table + //dump_tables(PageTable::::top()); + + new_tables::(areas); + + //dump_tables(PageTable::::top()); + + for i in &[1, 2, 4, 8, 16, 32] { + let phys = PhysicalAddress::new(i * MEGABYTE); + let virt = A::phys_to_virt(phys); + + // Test read + println!( + "0x{:X} (0x{:X}) = 0x{:X}", + virt.data(), + phys.data(), + A::read::(virt) + ); + + // Test write + A::write::(virt, 0x5A); + + // Test read + println!( + "0x{:X} (0x{:X}) = 0x{:X}", + virt.data(), + phys.data(), + A::read::(virt) + ); + } + } +} + +fn main() { + unsafe { + inner::(); + } +} diff --git a/rmm/src/page/entry.rs b/rmm/src/page/entry.rs new file mode 100644 index 0000000000..ac75f162b0 --- /dev/null +++ b/rmm/src/page/entry.rs @@ -0,0 +1,59 @@ +use core::marker::PhantomData; + +use crate::{Arch, PageFlags, PhysicalAddress}; + +#[derive(Clone, Copy, Debug)] +pub struct PageEntry { + data: usize, + phantom: PhantomData, +} + +impl PageEntry { + #[inline(always)] + pub fn new(address: usize, flags: usize) -> Self { + let data = (((address >> A::PAGE_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::ENTRY_ADDRESS_SHIFT) + | flags; + Self::from_data(data) + } + + #[inline(always)] + pub fn from_data(data: usize) -> Self { + Self { + data, + phantom: PhantomData, + } + } + + #[inline(always)] + pub fn data(&self) -> usize { + self.data + } + + #[inline(always)] + pub fn address(&self) -> Result { + let addr = PhysicalAddress( + ((self.data >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT, + ); + + if self.present() { + Ok(addr) + } else { + Err(addr) + } + } + + #[inline(always)] + pub fn flags(&self) -> PageFlags { + unsafe { PageFlags::from_data(self.data & A::ENTRY_FLAGS_MASK) } + } + #[inline(always)] + pub fn set_flags(&mut self, flags: PageFlags) { + self.data &= !A::ENTRY_FLAGS_MASK; + self.data |= flags.data(); + } + + #[inline(always)] + pub fn present(&self) -> bool { + self.data & A::ENTRY_FLAG_PRESENT != 0 + } +} diff --git a/rmm/src/page/flags.rs b/rmm/src/page/flags.rs new file mode 100644 index 0000000000..3d90691d60 --- /dev/null +++ b/rmm/src/page/flags.rs @@ -0,0 +1,145 @@ +use core::{fmt, marker::PhantomData}; + +use crate::Arch; + +#[derive(Clone, Copy)] +pub struct PageFlags { + data: usize, + arch: PhantomData, +} + +impl PageFlags { + #[inline(always)] + pub fn new() -> Self { + unsafe { + Self::from_data( + // Flags set to present, kernel space, read-only, no-execute by default + A::ENTRY_FLAG_DEFAULT_PAGE + | A::ENTRY_FLAG_READONLY + | A::ENTRY_FLAG_NO_EXEC + | A::ENTRY_FLAG_NO_GLOBAL, + ) + } + } + + #[inline(always)] + pub fn new_table() -> Self { + unsafe { + Self::from_data( + // Flags set to present, kernel space, read-only, no-execute by default + A::ENTRY_FLAG_DEFAULT_TABLE | A::ENTRY_FLAG_NO_EXEC | A::ENTRY_FLAG_NO_GLOBAL, + ) + } + } + + #[inline(always)] + pub unsafe fn from_data(data: usize) -> Self { + Self { + data, + arch: PhantomData, + } + } + + #[inline(always)] + pub fn data(&self) -> usize { + self.data + } + + #[must_use] + #[inline(always)] + pub fn custom_flag(mut self, flag: usize, value: bool) -> Self { + if value { + self.data |= flag; + } else { + self.data &= !flag; + } + self + } + + #[must_use] + #[inline(always)] + pub fn write_combining(self, value: bool) -> Self { + self.custom_flag(A::ENTRY_FLAG_WRITE_COMBINING, value) + } + + #[inline(always)] + pub fn has_flag(&self, flag: usize) -> bool { + self.data & flag == flag + } + + #[inline(always)] + pub fn has_present(&self) -> bool { + self.has_flag(A::ENTRY_FLAG_PRESENT) + } + + #[must_use] + #[inline(always)] + pub fn user(self, value: bool) -> Self { + self.custom_flag(A::ENTRY_FLAG_PAGE_USER, value) + } + + #[inline(always)] + pub fn has_user(&self) -> bool { + self.has_flag(A::ENTRY_FLAG_PAGE_USER) + } + + #[must_use] + #[inline(always)] + pub fn write(self, value: bool) -> Self { + // Architecture may use readonly or readwrite, or both, support either + if value { + self.custom_flag(A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE, false) + .custom_flag(A::ENTRY_FLAG_READWRITE, true) + } else { + self.custom_flag(A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE, false) + .custom_flag(A::ENTRY_FLAG_READONLY, true) + } + } + + #[inline(always)] + pub fn has_write(&self) -> bool { + // Architecture may use readonly or readwrite, or both, support either + self.data & (A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE) == A::ENTRY_FLAG_READWRITE + } + + #[must_use] + #[inline(always)] + pub fn execute(self, value: bool) -> Self { + //TODO: write xor execute? + // Architecture may use no exec or exec, support either + self.custom_flag(A::ENTRY_FLAG_NO_EXEC, !value) + .custom_flag(A::ENTRY_FLAG_EXEC, value) + } + + #[inline(always)] + pub fn has_execute(&self) -> bool { + // Architecture may use no exec or exec, support either + self.data & (A::ENTRY_FLAG_NO_EXEC | A::ENTRY_FLAG_EXEC) == A::ENTRY_FLAG_EXEC + } + + #[must_use] + #[inline(always)] + pub fn global(self, value: bool) -> Self { + // Architecture may use global or non global, support either + self.custom_flag(A::ENTRY_FLAG_NO_GLOBAL, !value) + .custom_flag(A::ENTRY_FLAG_GLOBAL, value) + } + + #[inline(always)] + pub fn is_global(&self) -> bool { + // Architecture may use global or non global, support either + self.data & (A::ENTRY_FLAG_GLOBAL | A::ENTRY_FLAG_NO_GLOBAL) == A::ENTRY_FLAG_GLOBAL + } +} + +impl fmt::Debug for PageFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PageFlags") + .field("present", &self.has_present()) + .field("write", &self.has_write()) + .field("executable", &self.has_execute()) + .field("user", &self.has_user()) + .field("bits", &format_args!("{:#0x}", self.data)) + .finish() + } +} diff --git a/rmm/src/page/flush.rs b/rmm/src/page/flush.rs new file mode 100644 index 0000000000..371301b901 --- /dev/null +++ b/rmm/src/page/flush.rs @@ -0,0 +1,74 @@ +use core::{marker::PhantomData, mem}; + +use crate::{Arch, VirtualAddress}; + +pub trait Flusher { + fn consume(&mut self, flush: PageFlush); +} + +#[must_use = "The page table must be flushed, or the changes unsafely ignored"] +pub struct PageFlush { + virt: VirtualAddress, + phantom: PhantomData, +} + +impl PageFlush { + pub fn new(virt: VirtualAddress) -> Self { + Self { + virt, + phantom: PhantomData, + } + } + + pub fn flush(self) { + unsafe { + A::invalidate(self.virt); + } + } + + pub unsafe fn ignore(self) { + mem::forget(self); + } +} + +// TODO: Might remove Drop and add #[must_use] again, but ergonomically I prefer being able to pass +// a flusher, and have it dropped by the end of the function it is passed to, in order to flush. +pub struct PageFlushAll { + phantom: PhantomData A>, +} + +impl PageFlushAll { + pub fn new() -> Self { + Self { + phantom: PhantomData, + } + } + + pub fn flush(self) {} + + pub unsafe fn ignore(self) { + mem::forget(self); + } +} +impl Drop for PageFlushAll { + fn drop(&mut self) { + unsafe { + A::invalidate_all(); + } + } +} +impl Flusher for PageFlushAll { + fn consume(&mut self, flush: PageFlush) { + unsafe { + flush.ignore(); + } + } +} +impl + ?Sized> Flusher for &mut T { + fn consume(&mut self, flush: PageFlush) { + >::consume(self, flush) + } +} +impl Flusher for () { + fn consume(&mut self, _: PageFlush) {} +} diff --git a/rmm/src/page/mapper.rs b/rmm/src/page/mapper.rs new file mode 100644 index 0000000000..b97d21543f --- /dev/null +++ b/rmm/src/page/mapper.rs @@ -0,0 +1,259 @@ +use core::marker::PhantomData; + +use crate::{ + Arch, FrameAllocator, PageEntry, PageFlags, PageFlush, PageTable, PhysicalAddress, TableKind, + VirtualAddress, +}; + +pub struct PageMapper { + table_kind: TableKind, + table_addr: PhysicalAddress, + allocator: F, + _phantom: PhantomData A>, +} + +impl PageMapper { + pub unsafe fn new(table_kind: TableKind, table_addr: PhysicalAddress, allocator: F) -> Self { + Self { + table_kind, + table_addr, + allocator, + _phantom: PhantomData, + } + } + + pub unsafe fn create(table_kind: TableKind, mut allocator: F) -> Option { + unsafe { + let table_addr = allocator.allocate_one()?; + Some(Self::new(table_kind, table_addr, allocator)) + } + } + + pub unsafe fn current(table_kind: TableKind, allocator: F) -> Self { + unsafe { + let table_addr = A::table(table_kind); + Self::new(table_kind, table_addr, allocator) + } + } + + pub fn is_current(&self) -> bool { + unsafe { self.table().phys() == A::table(self.table_kind) } + } + + pub unsafe fn make_current(&self) { + unsafe { + A::set_table(self.table_kind, self.table_addr); + } + } + + pub fn table(&self) -> PageTable { + // SAFETY: The only way to initialize a PageMapper is via new(), and we assume it upholds + // all necessary invariants for this to be safe. + unsafe { PageTable::new(VirtualAddress::new(0), self.table_addr, A::PAGE_LEVELS - 1) } + } + + pub fn allocator(&self) -> &F { + &self.allocator + } + + pub fn allocator_mut(&mut self) -> &mut F { + &mut self.allocator + } + + pub unsafe fn remap_with_full( + &mut self, + virt: VirtualAddress, + f: impl FnOnce(PhysicalAddress, PageFlags) -> Option<(PhysicalAddress, PageFlags)>, + ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { + unsafe { + self.visit(virt, |p1, i| { + let old_entry = p1.entry(i)?; + let old_phys = old_entry.address().ok()?; + let old_flags = old_entry.flags(); + let Some((new_phys, new_flags)) = f(old_phys, old_flags) else { + return None; + }; + // TODO: Higher-level PageEntry::new interface? + let new_entry = PageEntry::new(new_phys.data(), new_flags.data()); + p1.set_entry(i, new_entry); + Some((old_flags, old_phys, PageFlush::new(virt))) + }) + .flatten() + } + } + pub unsafe fn remap_with( + &mut self, + virt: VirtualAddress, + map_flags: impl FnOnce(PageFlags) -> PageFlags, + ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { + unsafe { + self.remap_with_full(virt, |same_phys, old_flags| { + Some((same_phys, map_flags(old_flags))) + }) + } + } + pub unsafe fn remap( + &mut self, + virt: VirtualAddress, + flags: PageFlags, + ) -> Option> { + unsafe { self.remap_with(virt, |_| flags).map(|(_, _, flush)| flush) } + } + + pub unsafe fn map( + &mut self, + virt: VirtualAddress, + flags: PageFlags, + ) -> Option> { + unsafe { + let phys = self.allocator.allocate_one()?; + self.map_phys(virt, phys, flags) + } + } + + pub unsafe fn map_phys( + &mut self, + virt: VirtualAddress, + phys: PhysicalAddress, + flags: PageFlags, + ) -> Option> { + unsafe { + //TODO: verify virt and phys are aligned + //TODO: verify flags have correct bits + let entry = PageEntry::new(phys.data(), flags.data()); + let mut table = self.table(); + loop { + let i = table.index_of(virt)?; + if table.level() == 0 { + //TODO: check for overwriting entry + table.set_entry(i, entry); + return Some(PageFlush::new(virt)); + } else { + let next_opt = table.next(i); + let next = match next_opt { + Some(some) => some, + None => { + let next_phys = self.allocator.allocate_one()?; + //TODO: correct flags? + let flags = A::ENTRY_FLAG_DEFAULT_TABLE + | if virt.kind() == TableKind::User { + A::ENTRY_FLAG_TABLE_USER + } else { + 0 + }; + table.set_entry(i, PageEntry::new(next_phys.data(), flags)); + table.next(i)? + } + }; + table = next; + } + } + } + } + pub unsafe fn map_linearly( + &mut self, + phys: PhysicalAddress, + flags: PageFlags, + ) -> Option<(VirtualAddress, PageFlush)> { + unsafe { + let virt = A::phys_to_virt(phys); + self.map_phys(virt, phys, flags).map(|flush| (virt, flush)) + } + } + fn visit( + &self, + virt: VirtualAddress, + f: impl FnOnce(&mut PageTable, usize) -> T, + ) -> Option { + let mut table = self.table(); + unsafe { + loop { + let i = table.index_of(virt)?; + if table.level() == 0 { + return Some(f(&mut table, i)); + } else { + table = table.next(i)?; + } + } + } + } + pub fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, PageFlags)> { + let entry = self.visit(virt, |p1, i| unsafe { p1.entry(i) })??; + Some((entry.address().ok()?, entry.flags())) + } + + pub unsafe fn unmap( + &mut self, + virt: VirtualAddress, + unmap_parents: bool, + ) -> Option> { + unsafe { + let (old, _, flush) = self.unmap_phys(virt, unmap_parents)?; + self.allocator.free_one(old); + Some(flush) + } + } + + pub unsafe fn unmap_phys( + &mut self, + virt: VirtualAddress, + unmap_parents: bool, + ) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { + unsafe { + //TODO: verify virt is aligned + let mut table = self.table(); + let level = table.level(); + unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator) + .map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) + } + } +} +unsafe fn unmap_phys_inner( + virt: VirtualAddress, + table: &mut PageTable, + initial_level: usize, + unmap_parents: bool, + allocator: &mut impl FrameAllocator, +) -> Option<(PhysicalAddress, PageFlags)> { + unsafe { + let i = table.index_of(virt)?; + + if table.level() == 0 { + let entry_opt = table.entry(i); + table.set_entry(i, PageEntry::new(0, 0)); + let entry = entry_opt?; + + Some((entry.address().ok()?, entry.flags())) + } else { + let mut subtable = table.next(i)?; + + let res = + unmap_phys_inner(virt, &mut subtable, initial_level, unmap_parents, allocator)?; + + //TODO: This is a bad idea for architectures where the kernel mappings are done in the process tables, + // as these mappings may become out of sync + if unmap_parents { + // TODO: Use a counter? This would reduce the remaining number of available bits, but could be + // faster (benchmark is needed). + let is_still_populated = (0..A::PAGE_ENTRIES) + .map(|j| subtable.entry(j).expect("must be within bounds")) + .any(|e| e.present()); + + if !is_still_populated { + allocator.free_one(subtable.phys()); + table.set_entry(i, PageEntry::new(0, 0)); + } + } + + Some(res) + } + } +} +impl core::fmt::Debug for PageMapper { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PageMapper") + .field("frame", &self.table_addr) + .field("allocator", &self.allocator) + .finish() + } +} diff --git a/rmm/src/page/mod.rs b/rmm/src/page/mod.rs new file mode 100644 index 0000000000..c4c4f00241 --- /dev/null +++ b/rmm/src/page/mod.rs @@ -0,0 +1,7 @@ +pub use self::{entry::*, flags::*, flush::*, mapper::*, table::*}; + +mod entry; +mod flags; +mod flush; +mod mapper; +mod table; diff --git a/rmm/src/page/table.rs b/rmm/src/page/table.rs new file mode 100644 index 0000000000..c6d377a28e --- /dev/null +++ b/rmm/src/page/table.rs @@ -0,0 +1,123 @@ +use core::marker::PhantomData; + +use super::PageEntry; +use crate::{Arch, PhysicalAddress, TableKind, VirtualAddress}; + +pub struct PageTable { + base: VirtualAddress, + phys: PhysicalAddress, + level: usize, + phantom: PhantomData, +} + +impl PageTable { + pub unsafe fn new(base: VirtualAddress, phys: PhysicalAddress, level: usize) -> Self { + Self { + base, + phys, + level, + phantom: PhantomData, + } + } + + pub unsafe fn top(table_kind: TableKind) -> Self { + unsafe { + Self::new( + VirtualAddress::new(0), + A::table(table_kind), + A::PAGE_LEVELS - 1, + ) + } + } + + pub fn base(&self) -> VirtualAddress { + self.base + } + + pub fn phys(&self) -> PhysicalAddress { + self.phys + } + + pub fn level(&self) -> usize { + self.level + } + + pub unsafe fn virt(&self) -> VirtualAddress { + unsafe { + A::phys_to_virt(self.phys) + + // Recursive mapping + // let mut addr = 0xFFFF_FFFF_FFFF_F000; + // for level in (self.level + 1 .. A::PAGE_LEVELS).rev() { + // let index = (self.base.0 >> (level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) & A::PAGE_ENTRY_MASK; + // addr <<= A::PAGE_ENTRY_SHIFT; + // addr |= index << A::PAGE_SHIFT; + // } + // VirtualAddress::new(addr) + } + } + + pub fn entry_base(&self, i: usize) -> Option { + if i < A::PAGE_ENTRIES { + let level_shift = self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + Some(self.base.add(i << level_shift)) + } else { + None + } + } + + pub unsafe fn entry_virt(&self, i: usize) -> Option { + unsafe { + if i < A::PAGE_ENTRIES { + Some(self.virt().add(i * A::PAGE_ENTRY_SIZE)) + } else { + None + } + } + } + + pub unsafe fn entry(&self, i: usize) -> Option> { + unsafe { + let addr = self.entry_virt(i)?; + Some(PageEntry::from_data(A::read::(addr))) + } + } + + pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { + unsafe { + let addr = self.entry_virt(i)?; + A::write::(addr, entry.data()); + Some(()) + } + } + + pub unsafe fn index_of(&self, address: VirtualAddress) -> Option { + // Canonicalize address first + let address = VirtualAddress::new(address.data() & A::PAGE_ADDRESS_MASK); + let level_shift = self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + // Intentionally wraps around at last-level table to get all-ones mask on architectures + // where addressable physical address space covers entire usized space (e.g. x86) + let level_mask = A::PAGE_ENTRIES + .wrapping_shl(level_shift as u32) + .wrapping_sub(1); + if address >= self.base && address <= self.base.add(level_mask) { + Some((address.data() >> level_shift) & A::PAGE_ENTRY_MASK) + } else { + None + } + } + + pub unsafe fn next(&self, i: usize) -> Option { + unsafe { + if self.level == 0 { + return None; + } + + Some(PageTable::new( + self.entry_base(i)?, + self.entry(i)?.address().ok()?, + self.level - 1, + )) + } + } +}